def test_split_event_stages(mock_app_config): # noqa: F811
impl = find_event_handler(
app_config=mock_app_config,
event_name='mock_shuffle_event',
event_info=mock_app_config.events['mock_shuffle_event']
)
event_info = mock_app_config.events['mock_shuffle_event']
stages = split_event_stages(mock_app_config.app,
event_name='mock_shuffle_event',
event_info=event_info,
impl=impl)
assert stages == {
'mock_shuffle_event': EventDescriptor(
type=EventType.GET,
read_stream=event_info.read_stream,
write_stream=WriteStreamDescriptor(
name='mock_app.test.mock_shuffle_event.produce_messages',
queues=['AUTO'],
queue_strategy=StreamQueueStrategy.PROPAGATE
),
auth=[]
),
'mock_shuffle_event$consume_stream': EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_app.test.mock_shuffle_event.produce_messages',
consumer_group='mock_app.test.mock_shuffle_event.consume_stream',
queues=['AUTO']
),
write_stream=event_info.write_stream,
auth=[]
)
}
def split_event_stages(app: AppDescriptor, event_name: str,
event_info: EventDescriptor,
impl: ModuleType) -> Dict[str, EventDescriptor]:
"""
Splits an event whose steps contain SHUFFLE step, in an initial event with same name as event_name
plus sub_events with names `event_name.step_name' for each step after a SHUFFLE.
Creates intermediate auto named write_stream, read_stream to communicate data between event and sub_events,
clones event configuration from main event to sub_events, and setup write_stream property for final event
to be the one specified in configuration.
"""
event_stages = extract_event_stages(impl)
if len(event_stages) == 1:
return {event_name: event_info}
effective_events: Dict[str, EventDescriptor] = {}
event_type = event_info.type
read_stream = event_info.read_stream
queues = ["AUTO"] if read_stream is None else read_stream.queues
sub_event_name: Optional[str] = event_name
sub_event_info = event_info
intermediate_stream = None
for stage in event_stages:
if sub_event_name is None:
sub_event_name = f"{event_name}${stage}"
if read_stream is None and intermediate_stream is not None:
read_stream = ReadStreamDescriptor(name=intermediate_stream,
consumer_group=auto_path(
app.name, app.version,
*event_name.split('.'),
stage),
queues=queues)
intermediate_stream = auto_path(app.name, app.version,
*event_name.split('.'), stage)
sub_event_info = EventDescriptor(
type=event_type,
read_stream=read_stream,
connections=event_info.connections,
impl=event_info.impl,
write_stream=WriteStreamDescriptor(
name=intermediate_stream,
queue_strategy=StreamQueueStrategy.PROPAGATE))
effective_events[sub_event_name] = sub_event_info
event_type = EventType.STREAM
sub_event_name = None
read_stream = None
# Set last stage write_stream to original event write_stream
sub_event_info.write_stream = event_info.write_stream
return effective_events
def mock_app_config():
return AppConfig(
app=AppDescriptor(name='test_app', version='test'),
engine=AppEngineConfig(),
env={},
events={'test': EventDescriptor(type=EventType.GET)},
server=ServerConfig(auth=AuthConfig(secrets_location='/tmp',
auth_passphrase='test',
create_keys=True)))
def _get_app_logger(monkeypatch, mock_app_config): # noqa: F811
_patch_logger(monkeypatch)
event_info = EventDescriptor(type=EventType.GET)
mock_event.logger = app_logger()
setup_app_logger(mock_event,
app_config=mock_app_config,
name='test_get_app_logger',
event_info=event_info)
return mock_event.logger
def mock_api_app_config():
return AppConfig(
app=AppDescriptor(
name='mock-app-api',
version='test'
),
plugins=[AppDescriptor(
name='mock-plugin', version='test'
)],
engine=AppEngineConfig(
import_modules=['mock_app'],
read_stream_timeout=1,
read_stream_interval=5,
track_headers=['session_id'],
cors_origin='http://test'
),
events={
"mock-app-api-get": EventDescriptor(
type=EventType.GET,
auth=[AuthType.BASIC],
route='mock-app-api/test/mock-app-api'
),
"mock-app-api-post": EventDescriptor(
type=EventType.POST,
route='mock-app-api/test/mock-app-api'
),
"mock-app-api-multipart": EventDescriptor(
type=EventType.MULTIPART,
route='mock-app-api/test/mock-app-api-multipart'
),
"mock-app-api-get-list": EventDescriptor(
type=EventType.GET,
auth=[AuthType.REFRESH]
),
"mock-app-api-query-args": EventDescriptor(
type=EventType.GET,
auth=[AuthType.REFRESH]
),
"mock-app-noapi": EventDescriptor(
type=EventType.GET
),
"mock_file_response_content_type": EventDescriptor(
type=EventType.GET
),
},
server=ServerConfig(
logging=LoggingConfig(
log_level="DEBUG", log_path="work/logs/test/"),
auth=AuthConfig(
secrets_location='/tmp',
auth_passphrase='test',
default_auth_methods=[AuthType.BEARER]
),
api=APIConfig(
docs_path='/api/docs'
)
)
)
def _get_app_extra_logger(monkeypatch, mock_app_config): # noqa: F811
_patch_logger(monkeypatch)
event_info = EventDescriptor(type=EventType.GET,
config=EventConfig(logging=EventLoggingConfig(
extra_fields=['field1', 'field2'])))
mock_event.logger, mock_event.extra = app_extra_logger()
setup_app_logger(mock_event,
app_config=mock_app_config,
name='test_get_app_logger_extra',
event_info=event_info)
return mock_event.logger, mock_event.extra
def test_context() -> EventContext:
app_config = AppConfig(
app=AppDescriptor(name='test_steps', version='test_version'),
events={'test_steps': EventDescriptor(type=EventType.POST)}
)
return EventContext(
app_config=app_config,
plugin_config=app_config,
event_name='test_steps',
track_ids={},
auth_info={}
)
def test_context() -> EventContext:
app_config = AppConfig(
app=AppDescriptor(name='test_steps', version='test_version'),
events={'test_steps': EventDescriptor(type=EventType.POST)}
).setup()
return EventContext(
app_config=app_config,
plugin_config=app_config,
event_name='test_steps',
settings=get_event_settings(app_config.effective_settings, 'test_steps'),
track_ids={},
auth_info={}
)
def valid_result_app_config() -> AppConfig:
return AppConfig(
app=AppDescriptor(name="simple_example", version=APPS_API_VERSION),
engine=AppEngineConfig(import_modules=["model"],
read_stream_timeout=1,
track_headers=["request_id", "correlation_id"]),
env={
"fs": {
"data_path":
f"/tmp/simple_example.{APP_VERSION}.fs.data_path/",
"app_description":
f"This is simple_example version {APPS_API_VERSION}",
"recursive_replacement":
f"Data is in /tmp/simple_example.{APP_VERSION}.fs.data_path/. "
+ f"This is simple_example version {APPS_API_VERSION}"
}
},
events={
"query_something":
EventDescriptor(type=EventType.GET),
"save_something":
EventDescriptor(type=EventType.POST),
"streams.something_event":
EventDescriptor(
type=EventType.POST,
write_stream=WriteStreamDescriptor(
name=f'simple_example.{APP_VERSION}.streams.something_event'
)),
"streams.process_events":
EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name=
f'simple_example.{APP_VERSION}.streams.something_event',
consumer_group=
f'simple_example.{APP_VERSION}.streams.process_events'),
config=EventConfig(logging=EventLoggingConfig(
extra_fields=['something_id', 'path'])))
})
def mock_plugin_config():
return AppConfig(app=AppDescriptor(name='mock_plugin', version='test'),
env={
'plugin': {
'plugin_value': 'test_plugin_value',
'custom_value': 'test_custom_value'
}
},
events={
'plugin_event':
EventDescriptor(type=EventType.GET,
plug_mode=EventPlugMode.ON_APP)
},
server=ServerConfig(logging=LoggingConfig(
log_level="DEBUG", log_path="work/logs/test/")))
def mock_client_app_config():
return AppConfig(
app=AppDescriptor(name='mock_client_app', version='test'),
engine=AppEngineConfig(import_modules=['mock_app'],
read_stream_timeout=1,
read_stream_interval=5),
events={
"mock_event":
EventDescriptor(type=EventType.GET,
route='mock-client-app/test/mock-event-test')
},
server=ServerConfig(
streams=StreamsConfig(
stream_manager='mock_engine.MockStreamManager',
delay_auto_start_seconds=0),
logging=LoggingConfig(log_level="DEBUG",
log_path="work/logs/test/"))).setup()
def mock_client_app_config():
return AppConfig(
app=AppDescriptor(name='mock_client_app', version='test'),
engine=AppEngineConfig(import_modules=['mock_client_app']),
app_connections={
"test_app_connection":
AppConnection(name="test_app",
version=APPS_API_VERSION,
client="hopeit.apps_client.AppsClient"),
"test_app_plugin_connection":
AppConnection(name="test_app",
version=APPS_API_VERSION,
client="hopeit.apps_client.AppsClient",
plugin_name="test_plugin",
plugin_version=APPS_API_VERSION)
},
settings={
"test_app_connection":
Payload.to_obj(
AppsClientSettings(
connection_str="http://test-host1,http://test-host2")),
"test_app_plugin_connection":
Payload.to_obj(
AppsClientSettings(
connection_str="http://test-host1,http://test-host2",
auth_strategy=ClientAuthStrategy.FORWARD_CONTEXT))
},
events={
"mock_client_event":
EventDescriptor(
type=EventType.GET,
connections=[
EventConnection(app_connection="test_app_connection",
event="test_event_get",
type=EventConnectionType.GET),
EventConnection(app_connection="test_app_connection",
event="test_event_post",
type=EventConnectionType.POST),
EventConnection(
app_connection="test_app_plugin_connection",
event="test_event_plugin",
type=EventConnectionType.GET)
])
},
server=ServerConfig(logging=LoggingConfig(
log_level="DEBUG", log_path="work/logs/test/"))).setup()
def mock_client_app_config():
return AppConfig(
app=AppDescriptor(name='mock_client_app', version='test'),
engine=AppEngineConfig(import_modules=['mock_client_app']),
app_connections={
"test_app_connection":
AppConnection(name="test_app",
version=APPS_API_VERSION,
client="mock_client_app.MockClient")
},
events={
"mock_client_event":
EventDescriptor(type=EventType.GET,
connections=[
EventConnection(
app_connection="test_app_connection",
event="test_event",
type=EventConnectionType.GET)
])
},
server=ServerConfig(logging=LoggingConfig(
log_level="DEBUG", log_path="work/logs/test/"))).setup()
def _config_effective_events(
app_config: AppConfig,
enabled_groups: List[str]) -> Dict[str, EventDescriptor]:
"""
Return effective events computed from user app config.
Effective events could be result of splitting a single event in stages,
using the "SHUFFLE" keyword, that will internaally generate 2 events.
Or for STREAMS that implements the `__service__` method, both a STREAM
and a SERVICE event will be generated.
Only events with groups defined in `enabled_groups` list will be returned.
If `enabled_groups` is empty, all events wil be considered.
"""
effective_events: Dict[str, EventDescriptor] = {}
assert app_config.server
for event_name, event_info in app_config.events.items():
if (len(enabled_groups) == 0
or event_info.group == EventDescriptor.DEFAULT_GROUP
or event_info.group in enabled_groups):
impl = find_event_handler(app_config=app_config,
event_name=event_name,
event_info=event_info)
# Add events resultant of splitting steps on SHUFFLE (stages)
splits = split_event_stages(app_config.app, event_name,
event_info, impl)
effective_events.update(**splits)
# Add associated SERVICE events to streams
if event_info.type == EventType.STREAM and hasattr(
impl, "__service__"):
effective_events[
f"{event_name}$__service__"] = EventDescriptor(
type=EventType.SERVICE,
connections=event_info.connections,
impl=event_info.impl,
)
return effective_events
def mock_app_config():
return AppConfig(
app=AppDescriptor(name='mock_app', version='test'),
plugins=[AppDescriptor('mock_plugin', 'test')],
engine=AppEngineConfig(import_modules=['mock_app'],
read_stream_timeout=1,
read_stream_interval=5,
track_headers=['session_id'],
cors_origin='http://test'),
env={
'app': {
'app_value': 'test_app_value'
},
'plugin': {
'custom_value': 'test_custom_value_override'
}
},
settings={
"mock_stream_event": {
"logging": {
"extra_fields": ['value'],
"stream_fields": ['msg_id']
},
"custom_setting": {
"custom": "value"
}
},
"mock_stream_timeout": {
"logging": {
"extra_fields": ['value'],
"stream_fields": ['msg_id']
},
"stream": {
"timeout": 2
}
},
"mock_write_stream_event": {
"logging": {
"extra_fields": ['value'],
"stream_fields": ['msg_id']
},
"stream": {
"target_max_len": 10
}
},
"mock_service_event": {
"stream": {
"target_max_len": 10,
"throttle_ms": 100,
"batch_size": 2
}
},
"mock_service_timeout": {
"response_timeout": 2.0
},
"mock_spawn_event": {
"stream": {
"target_max_len": 10,
"throttle_ms": 100,
"batch_size": 2
}
},
"mock_shuffle_event": {
"stream": {
"target_max_len": 10,
"throttle_ms": 100
}
},
"mock_timeout": {
"response_timeout": 2.0
},
"custom_extra_settings": {
"custom_setting": {
"custom": "value"
}
}
},
events={
"mock_event":
EventDescriptor(type=EventType.GET,
route='mock-app/test/mock-event-test',
group='GROUP_A'),
"mock_event_logging":
EventDescriptor(type=EventType.GET, group='GROUP_A'),
"mock_post_event":
EventDescriptor(type=EventType.POST,
route='mock-app/test/mock-event-test',
group='GROUP_A'),
"mock_multipart_event":
EventDescriptor(type=EventType.MULTIPART,
route='mock-app/test/mock-multipart-event-test',
group='GROUP_B'),
"mock_post_nopayload":
EventDescriptor(type=EventType.POST,
route='mock-app/test/mock-post-nopayload'),
"mock_post_preprocess":
EventDescriptor(type=EventType.POST,
route='mock-app/test/mock-post-preprocess'),
"mock_post_preprocess_no_datatype":
EventDescriptor(
type=EventType.POST,
route='mock-app/test/mock-post-preprocess-no-datatype'),
"mock_stream_event":
EventDescriptor(type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_stream',
consumer_group='mock_consumer_group'),
setting_keys=["custom_extra_settings"],
group='GROUP_A'),
"mock_stream_timeout":
EventDescriptor(type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_stream',
consumer_group='mock_consumer_group'),
group='GROUP_B'),
"mock_write_stream_event":
EventDescriptor(type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_service_event":
EventDescriptor(type=EventType.SERVICE,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_service_timeout":
EventDescriptor(type=EventType.SERVICE,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_spawn_event":
EventDescriptor(type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_shuffle_event":
EventDescriptor(type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_parallelize_event":
EventDescriptor(type=EventType.GET),
"mock_file_response":
EventDescriptor(type=EventType.GET),
"mock_stream_response":
EventDescriptor(type=EventType.GET),
"mock_file_response_content_type":
EventDescriptor(type=EventType.GET),
"mock_auth":
EventDescriptor(type=EventType.GET, auth=[AuthType.BASIC]),
"mock_post_auth":
EventDescriptor(type=EventType.POST, auth=[AuthType.BASIC]),
"mock_collector":
EventDescriptor(type=EventType.POST),
'mock_timeout':
EventDescriptor(type=EventType.GET),
'mock_read_write_stream':
EventDescriptor(type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_read_write_stream.read',
consumer_group='mock_read_write_stream'),
write_stream=WriteStreamDescriptor(
name='mock_read_write_stream.write')),
"mock_event_custom":
EventDescriptor(type=EventType.GET,
impl="mock_app.mock_event_custom_impl"),
"mock_event_dataobject_payload":
EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_read_write_stream.read',
consumer_group='mock_event_dataobject_payload'),
dataobjects=["mock_app.MockData"])
},
server=ServerConfig(
streams=StreamsConfig(
stream_manager='mock_engine.MockStreamManager',
delay_auto_start_seconds=0),
logging=LoggingConfig(log_level="DEBUG",
log_path="work/logs/test/"))).setup()
def mock_app_config():
return AppConfig(
app=AppDescriptor(
name='mock_app',
version='test'
),
plugins=[
AppDescriptor('mock_plugin', 'test')
],
engine=AppEngineConfig(
import_modules=['mock_app'],
read_stream_timeout=1,
read_stream_interval=5,
track_headers=['session_id'],
cors_origin='http://test'
),
env={
'app': {
'app_value': 'test_app_value'
},
'plugin': {
'custom_value': 'test_custom_value_override'
}
},
events={
"mock_event": EventDescriptor(
type=EventType.GET,
route='mock-app/test/mock-event-test'
),
"mock_event_logging": EventDescriptor(
type=EventType.GET
),
"mock_post_event": EventDescriptor(
type=EventType.POST,
route='mock-app/test/mock-event-test'
),
"mock_multipart_event": EventDescriptor(
type=EventType.MULTIPART,
route='mock-app/test/mock-multipart-event-test'
),
"mock_post_nopayload": EventDescriptor(
type=EventType.POST,
route='mock-app/test/mock-post-nopayload'
),
"mock_post_preprocess": EventDescriptor(
type=EventType.POST,
route='mock-app/test/mock-post-preprocess'
),
"mock_stream_event": EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_stream',
consumer_group='mock_consumer_group'
),
config=EventConfig(
logging=EventLoggingConfig(
extra_fields=['value'],
stream_fields=['msg_id']
)
)
),
"mock_stream_timeout": EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_stream',
consumer_group='mock_consumer_group'
),
config=EventConfig(
logging=EventLoggingConfig(
extra_fields=['value'],
stream_fields=['msg_id']
),
stream=EventStreamConfig(
timeout=2
)
)
),
"mock_write_stream_event": EventDescriptor(
type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
stream=EventStreamConfig(
target_max_len=10
)
)
),
"mock_service_event": EventDescriptor(
type=EventType.SERVICE,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
stream=EventStreamConfig(
target_max_len=10,
throttle_ms=100,
batch_size=2
)
)
),
"mock_service_timeout": EventDescriptor(
type=EventType.SERVICE,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
response_timeout=2.0
)
),
"mock_spawn_event": EventDescriptor(
type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
stream=EventStreamConfig(
target_max_len=10,
throttle_ms=100,
batch_size=2
)
)
),
"mock_shuffle_event": EventDescriptor(
type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
stream=EventStreamConfig(
target_max_len=10,
throttle_ms=100
)
)
),
"mock_parallelize_event": EventDescriptor(
type=EventType.GET
),
"mock_file_response": EventDescriptor(
type=EventType.GET
),
"mock_file_response_content_type": EventDescriptor(
type=EventType.GET
),
"mock_auth": EventDescriptor(
type=EventType.GET,
auth=[AuthType.BASIC]
),
"mock_post_auth": EventDescriptor(
type=EventType.POST,
auth=[AuthType.BASIC]
),
"mock_collector": EventDescriptor(
type=EventType.POST
),
'mock_timeout': EventDescriptor(
type=EventType.GET,
config=EventConfig(response_timeout=2.0)
),
'mock_read_write_stream': EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_read_write_stream.read',
consumer_group='mock_read_write_stream'
),
write_stream=WriteStreamDescriptor(
name='mock_read_write_stream.write'
)
)
},
server=ServerConfig(
streams=StreamsConfig(
stream_manager='mock_engine.MockStreamManager',
delay_auto_start_seconds=0
),
logging=LoggingConfig(
log_level="DEBUG", log_path="work/logs/test/")
)
)