def _setup_client_context(app_config: AppConfig, server_app_config: AppConfig,
register_client_key: bool = True) -> EventContext:
_init_engine_logger(app_config)
assert app_config.server
assert server_app_config.server
app_config.server.auth = AuthConfig(
secrets_location=f"/tmp/{uuid.uuid4()}",
auth_passphrase='test_passphrase',
enabled=True,
create_keys=True
)
auth.init(app_config.app_key(), app_config.server.auth)
if register_client_key:
os.rename(
pathlib.Path(app_config.server.auth.secrets_location) / 'public' / f'{app_config.app_key()}_pub.pem',
pathlib.Path(server_app_config.server.auth.secrets_location) / 'public' / f'{app_config.app_key()}_pub.pem'
)
return EventContext(
app_config=app_config,
plugin_config=app_config,
event_name='mock_event',
settings=get_event_settings(app_config.effective_settings, 'mock_event'), # type: ignore
track_ids={},
auth_info={}
)
def __init__(self,
*,
app_config: AppConfig,
plugins: List[AppConfig],
enabled_groups: List[str],
streams_enabled: bool = True):
"""
Creates an instance of the AppEngine
:param app_config: AppConfig, Hopeit application configuration as specified in config module
:param plugins: List of AppConfig, Hopeit application configurations for enabled plugins
:enabled_groups: List of str, list of enabled event groups
:streams_enabled: bool, for testing, set to False to disable automatic starting streams
"""
self.app_config = app_config
self.app_key = app_config.app_key()
self.effective_events = self._config_effective_events(
app_config, enabled_groups)
self.plugins = plugins
self.settings = get_runtime_settings(app_config, plugins)
self.event_handler: Optional[EventHandler] = None
self.streams_enabled = streams_enabled
self.stream_manager: Optional[StreamManager] = None
self._running: Dict[str, asyncio.Lock] = {
event_name: asyncio.Lock()
for event_name, event_info in self.effective_events.items()
if event_info.type in (EventType.STREAM, EventType.SERVICE)
}
logger.init_app(app_config, plugins)
def __init__(self, app_config: AppConfig, app_connection: str):
self.app_key = app_config.app_key()
self.app_conn_key = app_connection
self.app_connection = app_config.app_connections[app_connection]
settings_key = self.app_connection.settings or app_connection
self.settings = Payload.from_obj(
app_config.settings.get(settings_key, {}), AppsClientSettings)
self.event_connections = {
event_name: {
conn.event: conn
for conn in event_info.connections
if conn.app_connection == app_connection
}
for event_name, event_info in app_config.events.items()
}
self.routes = {
conn.event: self._get_route(conn.event)
for event_info in app_config.events.values()
for conn in event_info.connections
if conn.app_connection == app_connection
}
self.conn_state: Optional[AppConnectionState] = None
self.session: Optional[Any] = None
self.token: Optional[str] = None
self.token_expire: int = 0
def _update_api_paths(app_config: AppConfig, plugin: Optional[AppConfig] = None):
"""
Populates paths section of spec based on __api__ specified in implemented events
"""
assert spec is not None
events = {
k: v for k, v in app_config.events.items() if v.plug_mode == EventPlugMode.STANDALONE
} if plugin is None else {
k: v for k, v in plugin.events.items() if v.plug_mode == EventPlugMode.ON_APP
}
plugin_app = None if plugin is None else plugin.app
paths = spec.get('paths', {})
for event_name, event_info in events.items():
route = app_route_name(app_config.app, event_name=event_name, plugin=plugin_app,
override_route_name=event_info.route)
method = METHOD_MAPPING.get(event_info.type)
if method is None:
continue
event_api_spec = _extract_event_api_spec(
app_config if plugin is None else plugin, event_name, event_info
)
if event_api_spec is None:
event_api_spec = paths.get(route, {}).get(method)
if event_api_spec is None and _options.get('generate_mode'):
event_api_spec = {"description": f"<<<{event_name}>>>", "parameters": [], "responses": {}}
if event_api_spec is not None:
event_api_spec['tags'] = [app_config.app_key()]
_set_optional_fixed_headers(event_api_spec)
_set_track_headers(event_api_spec, app_config)
_set_path_security(event_api_spec, app_config, event_info)
route_path = paths.get(route, {})
route_path[method] = event_api_spec
paths[route] = route_path
spec['paths'] = paths
def mock_api_app_config():
return AppConfig(
app=AppDescriptor(
name='mock-app-api',
version='test'
),
plugins=[AppDescriptor(
name='mock-plugin', version='test'
)],
engine=AppEngineConfig(
import_modules=['mock_app'],
read_stream_timeout=1,
read_stream_interval=5,
track_headers=['session_id'],
cors_origin='http://test'
),
events={
"mock-app-api-get": EventDescriptor(
type=EventType.GET,
auth=[AuthType.BASIC],
route='mock-app-api/test/mock-app-api'
),
"mock-app-api-post": EventDescriptor(
type=EventType.POST,
route='mock-app-api/test/mock-app-api'
),
"mock-app-api-multipart": EventDescriptor(
type=EventType.MULTIPART,
route='mock-app-api/test/mock-app-api-multipart'
),
"mock-app-api-get-list": EventDescriptor(
type=EventType.GET,
auth=[AuthType.REFRESH]
),
"mock-app-api-query-args": EventDescriptor(
type=EventType.GET,
auth=[AuthType.REFRESH]
),
"mock-app-noapi": EventDescriptor(
type=EventType.GET
),
"mock_file_response_content_type": EventDescriptor(
type=EventType.GET
),
},
server=ServerConfig(
logging=LoggingConfig(
log_level="DEBUG", log_path="work/logs/test/"),
auth=AuthConfig(
secrets_location='/tmp',
auth_passphrase='test',
default_auth_methods=[AuthType.BEARER]
),
api=APIConfig(
docs_path='/api/docs'
)
)
)
def mock_app_config():
return AppConfig(
app=AppDescriptor(name='test_app', version='test'),
engine=AppEngineConfig(),
env={},
events={'test': EventDescriptor(type=EventType.GET)},
server=ServerConfig(auth=AuthConfig(secrets_location='/tmp',
auth_passphrase='test',
create_keys=True)))
async def register_app_connections(app_config: AppConfig):
"""
Used by the engine to initialize and keep client connections
"""
_registered_clients[app_config.app_key()] = {
app_connection: await Client.create(app_config,
app_connection).start()
for app_connection in app_config.app_connections.keys()
}
def test_context() -> EventContext:
app_config = AppConfig(
app=AppDescriptor(name='test_steps', version='test_version'),
events={'test_steps': EventDescriptor(type=EventType.POST)}
)
return EventContext(
app_config=app_config,
plugin_config=app_config,
event_name='test_steps',
track_ids={},
auth_info={}
)
def test_context() -> EventContext:
app_config = AppConfig(
app=AppDescriptor(name='test_steps', version='test_version'),
events={'test_steps': EventDescriptor(type=EventType.POST)}
).setup()
return EventContext(
app_config=app_config,
plugin_config=app_config,
event_name='test_steps',
settings=get_event_settings(app_config.effective_settings, 'test_steps'),
track_ids={},
auth_info={}
)
async def start_app(self, app_config: AppConfig):
"""
Starts and register a Hopeit App into this engine instance
:param app_config: AppConfog, app configuration as specified in config module
"""
logger.info(__name__, f"Starting app={app_config.app_key()}...")
plugins = [
self.app_engine(app_key=plugin.app_key()).app_config
for plugin in app_config.plugins
]
app_engine = await AppEngine(app_config=app_config,
plugins=plugins).start()
self.app_engines[app_config.app_key()] = app_engine
return app_engine
def mock_plugin_config():
return AppConfig(app=AppDescriptor(name='mock_plugin', version='test'),
env={
'plugin': {
'plugin_value': 'test_plugin_value',
'custom_value': 'test_custom_value'
}
},
events={
'plugin_event':
EventDescriptor(type=EventType.GET,
plug_mode=EventPlugMode.ON_APP)
},
server=ServerConfig(logging=LoggingConfig(
log_level="DEBUG", log_path="work/logs/test/")))
def mock_client_app_config():
return AppConfig(
app=AppDescriptor(name='mock_client_app', version='test'),
engine=AppEngineConfig(import_modules=['mock_client_app']),
app_connections={
"test_app_connection":
AppConnection(name="test_app",
version=APPS_API_VERSION,
client="hopeit.apps_client.AppsClient"),
"test_app_plugin_connection":
AppConnection(name="test_app",
version=APPS_API_VERSION,
client="hopeit.apps_client.AppsClient",
plugin_name="test_plugin",
plugin_version=APPS_API_VERSION)
},
settings={
"test_app_connection":
Payload.to_obj(
AppsClientSettings(
connection_str="http://test-host1,http://test-host2")),
"test_app_plugin_connection":
Payload.to_obj(
AppsClientSettings(
connection_str="http://test-host1,http://test-host2",
auth_strategy=ClientAuthStrategy.FORWARD_CONTEXT))
},
events={
"mock_client_event":
EventDescriptor(
type=EventType.GET,
connections=[
EventConnection(app_connection="test_app_connection",
event="test_event_get",
type=EventConnectionType.GET),
EventConnection(app_connection="test_app_connection",
event="test_event_post",
type=EventConnectionType.POST),
EventConnection(
app_connection="test_app_plugin_connection",
event="test_event_plugin",
type=EventConnectionType.GET)
])
},
server=ServerConfig(logging=LoggingConfig(
log_level="DEBUG", log_path="work/logs/test/"))).setup()
def mock_client_app_config():
return AppConfig(
app=AppDescriptor(name='mock_client_app', version='test'),
engine=AppEngineConfig(import_modules=['mock_app'],
read_stream_timeout=1,
read_stream_interval=5),
events={
"mock_event":
EventDescriptor(type=EventType.GET,
route='mock-client-app/test/mock-event-test')
},
server=ServerConfig(
streams=StreamsConfig(
stream_manager='mock_engine.MockStreamManager',
delay_auto_start_seconds=0),
logging=LoggingConfig(log_level="DEBUG",
log_path="work/logs/test/"))).setup()
def _setup_server_context(app_config: AppConfig) -> EventContext:
_init_engine_logger(app_config)
assert app_config.server
app_config.server.auth = AuthConfig(
secrets_location=f"/tmp/{uuid.uuid4()}",
auth_passphrase='test_passphrase',
enabled=True,
create_keys=True
)
auth.init(app_config.app_key(), app_config.server.auth)
return EventContext(
app_config=app_config,
plugin_config=app_config,
event_name='mock_event',
settings=get_event_settings(app_config.effective_settings, 'mock_event'), # type: ignore
track_ids={},
auth_info={}
)
def _set_path_security(event_api_spec: dict, app_config: AppConfig, event_info: EventDescriptor):
"""
Setup security schemes allowed for each path
"""
assert spec is not None
security: list = []
for auth in event_info.auth:
if auth == AuthType.REFRESH:
_update_auth_refresh_method(app_config.app_key())
auth_str = f"{app_config.app_key()}.refresh"
security.append({auth_str: []})
elif auth != AuthType.UNSECURED:
auth_str = f"auth.{auth.value.lower()}"
security.append({auth_str: []})
if len(security) == 0 and AuthType.UNSECURED not in event_info.auth:
security = spec['security']
if len(security) > 0:
event_api_spec['security'] = security
async def stream_startup_hook(app_config: AppConfig, *args, **kwargs):
"""
Start all stream event types configured in app.
:param app_key: already started app_key
"""
app_engine = runtime.server.app_engines[app_config.app_key()]
for event_name, event_info in app_engine.effective_events.items():
if event_info.type == EventType.STREAM:
assert event_info.read_stream
logger.info(
__name__,
f"STREAM start event_name={event_name} read_stream={event_info.read_stream.name}"
)
asyncio.create_task(app_engine.read_stream(event_name=event_name))
elif event_info.type == EventType.SERVICE:
logger.info(__name__, f"SERVICE start event_name={event_name}")
asyncio.create_task(app_engine.service_loop(event_name=event_name))
def __init__(self, *,
app_config: AppConfig,
plugin_config: AppConfig,
event_name: str,
track_ids: Dict[str, str],
auth_info: Dict[str, Any]):
self.app_key: str = app_config.app_key()
self.app: AppDescriptor = app_config.app
self.env: Env = {**plugin_config.env, **app_config.env}
self.event_name = event_name
base_event = event_name.split('$')[0]
self.event_info: EventDescriptor = plugin_config.events[base_event]
self.creation_ts: datetime = datetime.now().astimezone(tz=timezone.utc)
self.auth_info = auth_info
track_fields = [
'track.operation_id',
*app_config.engine.track_headers,
*(self.event_info.config.logging.stream_fields
if self.event_info.type == EventType.STREAM
else [])
]
self.track_ids = {k: track_ids.get(k) or '' for k in track_fields}
def mock_client_app_config():
return AppConfig(
app=AppDescriptor(name='mock_client_app', version='test'),
engine=AppEngineConfig(import_modules=['mock_client_app']),
app_connections={
"test_app_connection":
AppConnection(name="test_app",
version=APPS_API_VERSION,
client="mock_client_app.MockClient")
},
events={
"mock_client_event":
EventDescriptor(type=EventType.GET,
connections=[
EventConnection(
app_connection="test_app_connection",
event="test_event",
type=EventConnectionType.GET)
])
},
server=ServerConfig(logging=LoggingConfig(
log_level="DEBUG", log_path="work/logs/test/"))).setup()
def valid_result_app_config() -> AppConfig:
return AppConfig(
app=AppDescriptor(name="simple_example", version=APPS_API_VERSION),
engine=AppEngineConfig(import_modules=["model"],
read_stream_timeout=1,
track_headers=["request_id", "correlation_id"]),
env={
"fs": {
"data_path":
f"/tmp/simple_example.{APP_VERSION}.fs.data_path/",
"app_description":
f"This is simple_example version {APPS_API_VERSION}",
"recursive_replacement":
f"Data is in /tmp/simple_example.{APP_VERSION}.fs.data_path/. "
+ f"This is simple_example version {APPS_API_VERSION}"
}
},
events={
"query_something":
EventDescriptor(type=EventType.GET),
"save_something":
EventDescriptor(type=EventType.POST),
"streams.something_event":
EventDescriptor(
type=EventType.POST,
write_stream=WriteStreamDescriptor(
name=f'simple_example.{APP_VERSION}.streams.something_event'
)),
"streams.process_events":
EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name=
f'simple_example.{APP_VERSION}.streams.something_event',
consumer_group=
f'simple_example.{APP_VERSION}.streams.process_events'),
config=EventConfig(logging=EventLoggingConfig(
extra_fields=['something_id', 'path'])))
})
from hopeit.app.config import AppConfig
from hopeit.server.config import ServerConfig
import json
with open('app-config-schema-draftv6.json', 'w') as fb:
fb.write(json.dumps(AppConfig.json_schema(), indent=2))
with open('server-config-schema-draftv6.json', 'w') as fb:
fb.write(json.dumps(ServerConfig.json_schema(), indent=2))
def mock_app_config():
return AppConfig(
app=AppDescriptor(name='mock_app', version='test'),
plugins=[AppDescriptor('mock_plugin', 'test')],
engine=AppEngineConfig(import_modules=['mock_app'],
read_stream_timeout=1,
read_stream_interval=5,
track_headers=['session_id'],
cors_origin='http://test'),
env={
'app': {
'app_value': 'test_app_value'
},
'plugin': {
'custom_value': 'test_custom_value_override'
}
},
settings={
"mock_stream_event": {
"logging": {
"extra_fields": ['value'],
"stream_fields": ['msg_id']
},
"custom_setting": {
"custom": "value"
}
},
"mock_stream_timeout": {
"logging": {
"extra_fields": ['value'],
"stream_fields": ['msg_id']
},
"stream": {
"timeout": 2
}
},
"mock_write_stream_event": {
"logging": {
"extra_fields": ['value'],
"stream_fields": ['msg_id']
},
"stream": {
"target_max_len": 10
}
},
"mock_service_event": {
"stream": {
"target_max_len": 10,
"throttle_ms": 100,
"batch_size": 2
}
},
"mock_service_timeout": {
"response_timeout": 2.0
},
"mock_spawn_event": {
"stream": {
"target_max_len": 10,
"throttle_ms": 100,
"batch_size": 2
}
},
"mock_shuffle_event": {
"stream": {
"target_max_len": 10,
"throttle_ms": 100
}
},
"mock_timeout": {
"response_timeout": 2.0
},
"custom_extra_settings": {
"custom_setting": {
"custom": "value"
}
}
},
events={
"mock_event":
EventDescriptor(type=EventType.GET,
route='mock-app/test/mock-event-test',
group='GROUP_A'),
"mock_event_logging":
EventDescriptor(type=EventType.GET, group='GROUP_A'),
"mock_post_event":
EventDescriptor(type=EventType.POST,
route='mock-app/test/mock-event-test',
group='GROUP_A'),
"mock_multipart_event":
EventDescriptor(type=EventType.MULTIPART,
route='mock-app/test/mock-multipart-event-test',
group='GROUP_B'),
"mock_post_nopayload":
EventDescriptor(type=EventType.POST,
route='mock-app/test/mock-post-nopayload'),
"mock_post_preprocess":
EventDescriptor(type=EventType.POST,
route='mock-app/test/mock-post-preprocess'),
"mock_post_preprocess_no_datatype":
EventDescriptor(
type=EventType.POST,
route='mock-app/test/mock-post-preprocess-no-datatype'),
"mock_stream_event":
EventDescriptor(type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_stream',
consumer_group='mock_consumer_group'),
setting_keys=["custom_extra_settings"],
group='GROUP_A'),
"mock_stream_timeout":
EventDescriptor(type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_stream',
consumer_group='mock_consumer_group'),
group='GROUP_B'),
"mock_write_stream_event":
EventDescriptor(type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_service_event":
EventDescriptor(type=EventType.SERVICE,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_service_timeout":
EventDescriptor(type=EventType.SERVICE,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_spawn_event":
EventDescriptor(type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_shuffle_event":
EventDescriptor(type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_parallelize_event":
EventDescriptor(type=EventType.GET),
"mock_file_response":
EventDescriptor(type=EventType.GET),
"mock_stream_response":
EventDescriptor(type=EventType.GET),
"mock_file_response_content_type":
EventDescriptor(type=EventType.GET),
"mock_auth":
EventDescriptor(type=EventType.GET, auth=[AuthType.BASIC]),
"mock_post_auth":
EventDescriptor(type=EventType.POST, auth=[AuthType.BASIC]),
"mock_collector":
EventDescriptor(type=EventType.POST),
'mock_timeout':
EventDescriptor(type=EventType.GET),
'mock_read_write_stream':
EventDescriptor(type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_read_write_stream.read',
consumer_group='mock_read_write_stream'),
write_stream=WriteStreamDescriptor(
name='mock_read_write_stream.write')),
"mock_event_custom":
EventDescriptor(type=EventType.GET,
impl="mock_app.mock_event_custom_impl"),
"mock_event_dataobject_payload":
EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_read_write_stream.read',
consumer_group='mock_event_dataobject_payload'),
dataobjects=["mock_app.MockData"])
},
server=ServerConfig(
streams=StreamsConfig(
stream_manager='mock_engine.MockStreamManager',
delay_auto_start_seconds=0),
logging=LoggingConfig(log_level="DEBUG",
log_path="work/logs/test/"))).setup()
def mock_app_config():
return AppConfig(
app=AppDescriptor(
name='mock_app',
version='test'
),
plugins=[
AppDescriptor('mock_plugin', 'test')
],
engine=AppEngineConfig(
import_modules=['mock_app'],
read_stream_timeout=1,
read_stream_interval=5,
track_headers=['session_id'],
cors_origin='http://test'
),
env={
'app': {
'app_value': 'test_app_value'
},
'plugin': {
'custom_value': 'test_custom_value_override'
}
},
events={
"mock_event": EventDescriptor(
type=EventType.GET,
route='mock-app/test/mock-event-test'
),
"mock_event_logging": EventDescriptor(
type=EventType.GET
),
"mock_post_event": EventDescriptor(
type=EventType.POST,
route='mock-app/test/mock-event-test'
),
"mock_multipart_event": EventDescriptor(
type=EventType.MULTIPART,
route='mock-app/test/mock-multipart-event-test'
),
"mock_post_nopayload": EventDescriptor(
type=EventType.POST,
route='mock-app/test/mock-post-nopayload'
),
"mock_post_preprocess": EventDescriptor(
type=EventType.POST,
route='mock-app/test/mock-post-preprocess'
),
"mock_stream_event": EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_stream',
consumer_group='mock_consumer_group'
),
config=EventConfig(
logging=EventLoggingConfig(
extra_fields=['value'],
stream_fields=['msg_id']
)
)
),
"mock_stream_timeout": EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_stream',
consumer_group='mock_consumer_group'
),
config=EventConfig(
logging=EventLoggingConfig(
extra_fields=['value'],
stream_fields=['msg_id']
),
stream=EventStreamConfig(
timeout=2
)
)
),
"mock_write_stream_event": EventDescriptor(
type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
stream=EventStreamConfig(
target_max_len=10
)
)
),
"mock_service_event": EventDescriptor(
type=EventType.SERVICE,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
stream=EventStreamConfig(
target_max_len=10,
throttle_ms=100,
batch_size=2
)
)
),
"mock_service_timeout": EventDescriptor(
type=EventType.SERVICE,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
response_timeout=2.0
)
),
"mock_spawn_event": EventDescriptor(
type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
stream=EventStreamConfig(
target_max_len=10,
throttle_ms=100,
batch_size=2
)
)
),
"mock_shuffle_event": EventDescriptor(
type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
stream=EventStreamConfig(
target_max_len=10,
throttle_ms=100
)
)
),
"mock_parallelize_event": EventDescriptor(
type=EventType.GET
),
"mock_file_response": EventDescriptor(
type=EventType.GET
),
"mock_file_response_content_type": EventDescriptor(
type=EventType.GET
),
"mock_auth": EventDescriptor(
type=EventType.GET,
auth=[AuthType.BASIC]
),
"mock_post_auth": EventDescriptor(
type=EventType.POST,
auth=[AuthType.BASIC]
),
"mock_collector": EventDescriptor(
type=EventType.POST
),
'mock_timeout': EventDescriptor(
type=EventType.GET,
config=EventConfig(response_timeout=2.0)
),
'mock_read_write_stream': EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_read_write_stream.read',
consumer_group='mock_read_write_stream'
),
write_stream=WriteStreamDescriptor(
name='mock_read_write_stream.write'
)
)
},
server=ServerConfig(
streams=StreamsConfig(
stream_manager='mock_engine.MockStreamManager',
delay_auto_start_seconds=0
),
logging=LoggingConfig(
log_level="DEBUG", log_path="work/logs/test/")
)
)