def test_split_event_stages(mock_app_config): # noqa: F811 impl = find_event_handler( app_config=mock_app_config, event_name='mock_shuffle_event', event_info=mock_app_config.events['mock_shuffle_event'] ) event_info = mock_app_config.events['mock_shuffle_event'] stages = split_event_stages(mock_app_config.app, event_name='mock_shuffle_event', event_info=event_info, impl=impl) assert stages == { 'mock_shuffle_event': EventDescriptor( type=EventType.GET, read_stream=event_info.read_stream, write_stream=WriteStreamDescriptor( name='mock_app.test.mock_shuffle_event.produce_messages', queues=['AUTO'], queue_strategy=StreamQueueStrategy.PROPAGATE ), auth=[] ), 'mock_shuffle_event$consume_stream': EventDescriptor( type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_app.test.mock_shuffle_event.produce_messages', consumer_group='mock_app.test.mock_shuffle_event.consume_stream', queues=['AUTO'] ), write_stream=event_info.write_stream, auth=[] ) }
def split_event_stages(app: AppDescriptor, event_name: str, event_info: EventDescriptor, impl: ModuleType) -> Dict[str, EventDescriptor]: """ Splits an event whose steps contain SHUFFLE step, in an initial event with same name as event_name plus sub_events with names `event_name.step_name' for each step after a SHUFFLE. Creates intermediate auto named write_stream, read_stream to communicate data between event and sub_events, clones event configuration from main event to sub_events, and setup write_stream property for final event to be the one specified in configuration. """ event_stages = extract_event_stages(impl) if len(event_stages) == 1: return {event_name: event_info} effective_events: Dict[str, EventDescriptor] = {} event_type = event_info.type read_stream = event_info.read_stream queues = ["AUTO"] if read_stream is None else read_stream.queues sub_event_name: Optional[str] = event_name sub_event_info = event_info intermediate_stream = None for stage in event_stages: if sub_event_name is None: sub_event_name = f"{event_name}${stage}" if read_stream is None and intermediate_stream is not None: read_stream = ReadStreamDescriptor(name=intermediate_stream, consumer_group=auto_path( app.name, app.version, *event_name.split('.'), stage), queues=queues) intermediate_stream = auto_path(app.name, app.version, *event_name.split('.'), stage) sub_event_info = EventDescriptor( type=event_type, read_stream=read_stream, connections=event_info.connections, impl=event_info.impl, write_stream=WriteStreamDescriptor( name=intermediate_stream, queue_strategy=StreamQueueStrategy.PROPAGATE)) effective_events[sub_event_name] = sub_event_info event_type = EventType.STREAM sub_event_name = None read_stream = None # Set last stage write_stream to original event write_stream sub_event_info.write_stream = event_info.write_stream return effective_events
def valid_result_app_config() -> AppConfig: return AppConfig( app=AppDescriptor(name="simple_example", version=APPS_API_VERSION), engine=AppEngineConfig(import_modules=["model"], read_stream_timeout=1, track_headers=["request_id", "correlation_id"]), env={ "fs": { "data_path": f"/tmp/simple_example.{APP_VERSION}.fs.data_path/", "app_description": f"This is simple_example version {APPS_API_VERSION}", "recursive_replacement": f"Data is in /tmp/simple_example.{APP_VERSION}.fs.data_path/. " + f"This is simple_example version {APPS_API_VERSION}" } }, events={ "query_something": EventDescriptor(type=EventType.GET), "save_something": EventDescriptor(type=EventType.POST), "streams.something_event": EventDescriptor( type=EventType.POST, write_stream=WriteStreamDescriptor( name=f'simple_example.{APP_VERSION}.streams.something_event' )), "streams.process_events": EventDescriptor( type=EventType.STREAM, read_stream=ReadStreamDescriptor( name= f'simple_example.{APP_VERSION}.streams.something_event', consumer_group= f'simple_example.{APP_VERSION}.streams.process_events'), config=EventConfig(logging=EventLoggingConfig( extra_fields=['something_id', 'path']))) })
def mock_app_config(): return AppConfig( app=AppDescriptor(name='mock_app', version='test'), plugins=[AppDescriptor('mock_plugin', 'test')], engine=AppEngineConfig(import_modules=['mock_app'], read_stream_timeout=1, read_stream_interval=5, track_headers=['session_id'], cors_origin='http://test'), env={ 'app': { 'app_value': 'test_app_value' }, 'plugin': { 'custom_value': 'test_custom_value_override' } }, settings={ "mock_stream_event": { "logging": { "extra_fields": ['value'], "stream_fields": ['msg_id'] }, "custom_setting": { "custom": "value" } }, "mock_stream_timeout": { "logging": { "extra_fields": ['value'], "stream_fields": ['msg_id'] }, "stream": { "timeout": 2 } }, "mock_write_stream_event": { "logging": { "extra_fields": ['value'], "stream_fields": ['msg_id'] }, "stream": { "target_max_len": 10 } }, "mock_service_event": { "stream": { "target_max_len": 10, "throttle_ms": 100, "batch_size": 2 } }, "mock_service_timeout": { "response_timeout": 2.0 }, "mock_spawn_event": { "stream": { "target_max_len": 10, "throttle_ms": 100, "batch_size": 2 } }, "mock_shuffle_event": { "stream": { "target_max_len": 10, "throttle_ms": 100 } }, "mock_timeout": { "response_timeout": 2.0 }, "custom_extra_settings": { "custom_setting": { "custom": "value" } } }, events={ "mock_event": EventDescriptor(type=EventType.GET, route='mock-app/test/mock-event-test', group='GROUP_A'), "mock_event_logging": EventDescriptor(type=EventType.GET, group='GROUP_A'), "mock_post_event": EventDescriptor(type=EventType.POST, route='mock-app/test/mock-event-test', group='GROUP_A'), "mock_multipart_event": EventDescriptor(type=EventType.MULTIPART, route='mock-app/test/mock-multipart-event-test', group='GROUP_B'), "mock_post_nopayload": EventDescriptor(type=EventType.POST, route='mock-app/test/mock-post-nopayload'), "mock_post_preprocess": EventDescriptor(type=EventType.POST, route='mock-app/test/mock-post-preprocess'), "mock_post_preprocess_no_datatype": EventDescriptor( type=EventType.POST, route='mock-app/test/mock-post-preprocess-no-datatype'), "mock_stream_event": EventDescriptor(type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_stream', consumer_group='mock_consumer_group'), setting_keys=["custom_extra_settings"], group='GROUP_A'), "mock_stream_timeout": EventDescriptor(type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_stream', consumer_group='mock_consumer_group'), group='GROUP_B'), "mock_write_stream_event": EventDescriptor(type=EventType.GET, write_stream=WriteStreamDescriptor( name='mock_write_stream_event')), "mock_service_event": EventDescriptor(type=EventType.SERVICE, write_stream=WriteStreamDescriptor( name='mock_write_stream_event')), "mock_service_timeout": EventDescriptor(type=EventType.SERVICE, write_stream=WriteStreamDescriptor( name='mock_write_stream_event')), "mock_spawn_event": EventDescriptor(type=EventType.GET, write_stream=WriteStreamDescriptor( name='mock_write_stream_event')), "mock_shuffle_event": EventDescriptor(type=EventType.GET, write_stream=WriteStreamDescriptor( name='mock_write_stream_event')), "mock_parallelize_event": EventDescriptor(type=EventType.GET), "mock_file_response": EventDescriptor(type=EventType.GET), "mock_stream_response": EventDescriptor(type=EventType.GET), "mock_file_response_content_type": EventDescriptor(type=EventType.GET), "mock_auth": EventDescriptor(type=EventType.GET, auth=[AuthType.BASIC]), "mock_post_auth": EventDescriptor(type=EventType.POST, auth=[AuthType.BASIC]), "mock_collector": EventDescriptor(type=EventType.POST), 'mock_timeout': EventDescriptor(type=EventType.GET), 'mock_read_write_stream': EventDescriptor(type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_read_write_stream.read', consumer_group='mock_read_write_stream'), write_stream=WriteStreamDescriptor( name='mock_read_write_stream.write')), "mock_event_custom": EventDescriptor(type=EventType.GET, impl="mock_app.mock_event_custom_impl"), "mock_event_dataobject_payload": EventDescriptor( type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_read_write_stream.read', consumer_group='mock_event_dataobject_payload'), dataobjects=["mock_app.MockData"]) }, server=ServerConfig( streams=StreamsConfig( stream_manager='mock_engine.MockStreamManager', delay_auto_start_seconds=0), logging=LoggingConfig(log_level="DEBUG", log_path="work/logs/test/"))).setup()
def mock_app_config(): return AppConfig( app=AppDescriptor( name='mock_app', version='test' ), plugins=[ AppDescriptor('mock_plugin', 'test') ], engine=AppEngineConfig( import_modules=['mock_app'], read_stream_timeout=1, read_stream_interval=5, track_headers=['session_id'], cors_origin='http://test' ), env={ 'app': { 'app_value': 'test_app_value' }, 'plugin': { 'custom_value': 'test_custom_value_override' } }, events={ "mock_event": EventDescriptor( type=EventType.GET, route='mock-app/test/mock-event-test' ), "mock_event_logging": EventDescriptor( type=EventType.GET ), "mock_post_event": EventDescriptor( type=EventType.POST, route='mock-app/test/mock-event-test' ), "mock_multipart_event": EventDescriptor( type=EventType.MULTIPART, route='mock-app/test/mock-multipart-event-test' ), "mock_post_nopayload": EventDescriptor( type=EventType.POST, route='mock-app/test/mock-post-nopayload' ), "mock_post_preprocess": EventDescriptor( type=EventType.POST, route='mock-app/test/mock-post-preprocess' ), "mock_stream_event": EventDescriptor( type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_stream', consumer_group='mock_consumer_group' ), config=EventConfig( logging=EventLoggingConfig( extra_fields=['value'], stream_fields=['msg_id'] ) ) ), "mock_stream_timeout": EventDescriptor( type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_stream', consumer_group='mock_consumer_group' ), config=EventConfig( logging=EventLoggingConfig( extra_fields=['value'], stream_fields=['msg_id'] ), stream=EventStreamConfig( timeout=2 ) ) ), "mock_write_stream_event": EventDescriptor( type=EventType.GET, write_stream=WriteStreamDescriptor( name='mock_write_stream_event' ), config=EventConfig( stream=EventStreamConfig( target_max_len=10 ) ) ), "mock_service_event": EventDescriptor( type=EventType.SERVICE, write_stream=WriteStreamDescriptor( name='mock_write_stream_event' ), config=EventConfig( stream=EventStreamConfig( target_max_len=10, throttle_ms=100, batch_size=2 ) ) ), "mock_service_timeout": EventDescriptor( type=EventType.SERVICE, write_stream=WriteStreamDescriptor( name='mock_write_stream_event' ), config=EventConfig( response_timeout=2.0 ) ), "mock_spawn_event": EventDescriptor( type=EventType.GET, write_stream=WriteStreamDescriptor( name='mock_write_stream_event' ), config=EventConfig( stream=EventStreamConfig( target_max_len=10, throttle_ms=100, batch_size=2 ) ) ), "mock_shuffle_event": EventDescriptor( type=EventType.GET, write_stream=WriteStreamDescriptor( name='mock_write_stream_event' ), config=EventConfig( stream=EventStreamConfig( target_max_len=10, throttle_ms=100 ) ) ), "mock_parallelize_event": EventDescriptor( type=EventType.GET ), "mock_file_response": EventDescriptor( type=EventType.GET ), "mock_file_response_content_type": EventDescriptor( type=EventType.GET ), "mock_auth": EventDescriptor( type=EventType.GET, auth=[AuthType.BASIC] ), "mock_post_auth": EventDescriptor( type=EventType.POST, auth=[AuthType.BASIC] ), "mock_collector": EventDescriptor( type=EventType.POST ), 'mock_timeout': EventDescriptor( type=EventType.GET, config=EventConfig(response_timeout=2.0) ), 'mock_read_write_stream': EventDescriptor( type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_read_write_stream.read', consumer_group='mock_read_write_stream' ), write_stream=WriteStreamDescriptor( name='mock_read_write_stream.write' ) ) }, server=ServerConfig( streams=StreamsConfig( stream_manager='mock_engine.MockStreamManager', delay_auto_start_seconds=0 ), logging=LoggingConfig( log_level="DEBUG", log_path="work/logs/test/") ) )