def _setup_client_context(app_config: AppConfig, server_app_config: AppConfig, register_client_key: bool = True) -> EventContext: _init_engine_logger(app_config) assert app_config.server assert server_app_config.server app_config.server.auth = AuthConfig( secrets_location=f"/tmp/{uuid.uuid4()}", auth_passphrase='test_passphrase', enabled=True, create_keys=True ) auth.init(app_config.app_key(), app_config.server.auth) if register_client_key: os.rename( pathlib.Path(app_config.server.auth.secrets_location) / 'public' / f'{app_config.app_key()}_pub.pem', pathlib.Path(server_app_config.server.auth.secrets_location) / 'public' / f'{app_config.app_key()}_pub.pem' ) return EventContext( app_config=app_config, plugin_config=app_config, event_name='mock_event', settings=get_event_settings(app_config.effective_settings, 'mock_event'), # type: ignore track_ids={}, auth_info={} )
def __init__(self, *, app_config: AppConfig, plugins: List[AppConfig], enabled_groups: List[str], streams_enabled: bool = True): """ Creates an instance of the AppEngine :param app_config: AppConfig, Hopeit application configuration as specified in config module :param plugins: List of AppConfig, Hopeit application configurations for enabled plugins :enabled_groups: List of str, list of enabled event groups :streams_enabled: bool, for testing, set to False to disable automatic starting streams """ self.app_config = app_config self.app_key = app_config.app_key() self.effective_events = self._config_effective_events( app_config, enabled_groups) self.plugins = plugins self.settings = get_runtime_settings(app_config, plugins) self.event_handler: Optional[EventHandler] = None self.streams_enabled = streams_enabled self.stream_manager: Optional[StreamManager] = None self._running: Dict[str, asyncio.Lock] = { event_name: asyncio.Lock() for event_name, event_info in self.effective_events.items() if event_info.type in (EventType.STREAM, EventType.SERVICE) } logger.init_app(app_config, plugins)
def __init__(self, app_config: AppConfig, app_connection: str): self.app_key = app_config.app_key() self.app_conn_key = app_connection self.app_connection = app_config.app_connections[app_connection] settings_key = self.app_connection.settings or app_connection self.settings = Payload.from_obj( app_config.settings.get(settings_key, {}), AppsClientSettings) self.event_connections = { event_name: { conn.event: conn for conn in event_info.connections if conn.app_connection == app_connection } for event_name, event_info in app_config.events.items() } self.routes = { conn.event: self._get_route(conn.event) for event_info in app_config.events.values() for conn in event_info.connections if conn.app_connection == app_connection } self.conn_state: Optional[AppConnectionState] = None self.session: Optional[Any] = None self.token: Optional[str] = None self.token_expire: int = 0
def _update_api_paths(app_config: AppConfig, plugin: Optional[AppConfig] = None): """ Populates paths section of spec based on __api__ specified in implemented events """ assert spec is not None events = { k: v for k, v in app_config.events.items() if v.plug_mode == EventPlugMode.STANDALONE } if plugin is None else { k: v for k, v in plugin.events.items() if v.plug_mode == EventPlugMode.ON_APP } plugin_app = None if plugin is None else plugin.app paths = spec.get('paths', {}) for event_name, event_info in events.items(): route = app_route_name(app_config.app, event_name=event_name, plugin=plugin_app, override_route_name=event_info.route) method = METHOD_MAPPING.get(event_info.type) if method is None: continue event_api_spec = _extract_event_api_spec( app_config if plugin is None else plugin, event_name, event_info ) if event_api_spec is None: event_api_spec = paths.get(route, {}).get(method) if event_api_spec is None and _options.get('generate_mode'): event_api_spec = {"description": f"<<<{event_name}>>>", "parameters": [], "responses": {}} if event_api_spec is not None: event_api_spec['tags'] = [app_config.app_key()] _set_optional_fixed_headers(event_api_spec) _set_track_headers(event_api_spec, app_config) _set_path_security(event_api_spec, app_config, event_info) route_path = paths.get(route, {}) route_path[method] = event_api_spec paths[route] = route_path spec['paths'] = paths
def mock_api_app_config(): return AppConfig( app=AppDescriptor( name='mock-app-api', version='test' ), plugins=[AppDescriptor( name='mock-plugin', version='test' )], engine=AppEngineConfig( import_modules=['mock_app'], read_stream_timeout=1, read_stream_interval=5, track_headers=['session_id'], cors_origin='http://test' ), events={ "mock-app-api-get": EventDescriptor( type=EventType.GET, auth=[AuthType.BASIC], route='mock-app-api/test/mock-app-api' ), "mock-app-api-post": EventDescriptor( type=EventType.POST, route='mock-app-api/test/mock-app-api' ), "mock-app-api-multipart": EventDescriptor( type=EventType.MULTIPART, route='mock-app-api/test/mock-app-api-multipart' ), "mock-app-api-get-list": EventDescriptor( type=EventType.GET, auth=[AuthType.REFRESH] ), "mock-app-api-query-args": EventDescriptor( type=EventType.GET, auth=[AuthType.REFRESH] ), "mock-app-noapi": EventDescriptor( type=EventType.GET ), "mock_file_response_content_type": EventDescriptor( type=EventType.GET ), }, server=ServerConfig( logging=LoggingConfig( log_level="DEBUG", log_path="work/logs/test/"), auth=AuthConfig( secrets_location='/tmp', auth_passphrase='test', default_auth_methods=[AuthType.BEARER] ), api=APIConfig( docs_path='/api/docs' ) ) )
def mock_app_config(): return AppConfig( app=AppDescriptor(name='test_app', version='test'), engine=AppEngineConfig(), env={}, events={'test': EventDescriptor(type=EventType.GET)}, server=ServerConfig(auth=AuthConfig(secrets_location='/tmp', auth_passphrase='test', create_keys=True)))
async def register_app_connections(app_config: AppConfig): """ Used by the engine to initialize and keep client connections """ _registered_clients[app_config.app_key()] = { app_connection: await Client.create(app_config, app_connection).start() for app_connection in app_config.app_connections.keys() }
def test_context() -> EventContext: app_config = AppConfig( app=AppDescriptor(name='test_steps', version='test_version'), events={'test_steps': EventDescriptor(type=EventType.POST)} ) return EventContext( app_config=app_config, plugin_config=app_config, event_name='test_steps', track_ids={}, auth_info={} )
def test_context() -> EventContext: app_config = AppConfig( app=AppDescriptor(name='test_steps', version='test_version'), events={'test_steps': EventDescriptor(type=EventType.POST)} ).setup() return EventContext( app_config=app_config, plugin_config=app_config, event_name='test_steps', settings=get_event_settings(app_config.effective_settings, 'test_steps'), track_ids={}, auth_info={} )
async def start_app(self, app_config: AppConfig): """ Starts and register a Hopeit App into this engine instance :param app_config: AppConfog, app configuration as specified in config module """ logger.info(__name__, f"Starting app={app_config.app_key()}...") plugins = [ self.app_engine(app_key=plugin.app_key()).app_config for plugin in app_config.plugins ] app_engine = await AppEngine(app_config=app_config, plugins=plugins).start() self.app_engines[app_config.app_key()] = app_engine return app_engine
def mock_plugin_config(): return AppConfig(app=AppDescriptor(name='mock_plugin', version='test'), env={ 'plugin': { 'plugin_value': 'test_plugin_value', 'custom_value': 'test_custom_value' } }, events={ 'plugin_event': EventDescriptor(type=EventType.GET, plug_mode=EventPlugMode.ON_APP) }, server=ServerConfig(logging=LoggingConfig( log_level="DEBUG", log_path="work/logs/test/")))
def mock_client_app_config(): return AppConfig( app=AppDescriptor(name='mock_client_app', version='test'), engine=AppEngineConfig(import_modules=['mock_client_app']), app_connections={ "test_app_connection": AppConnection(name="test_app", version=APPS_API_VERSION, client="hopeit.apps_client.AppsClient"), "test_app_plugin_connection": AppConnection(name="test_app", version=APPS_API_VERSION, client="hopeit.apps_client.AppsClient", plugin_name="test_plugin", plugin_version=APPS_API_VERSION) }, settings={ "test_app_connection": Payload.to_obj( AppsClientSettings( connection_str="http://test-host1,http://test-host2")), "test_app_plugin_connection": Payload.to_obj( AppsClientSettings( connection_str="http://test-host1,http://test-host2", auth_strategy=ClientAuthStrategy.FORWARD_CONTEXT)) }, events={ "mock_client_event": EventDescriptor( type=EventType.GET, connections=[ EventConnection(app_connection="test_app_connection", event="test_event_get", type=EventConnectionType.GET), EventConnection(app_connection="test_app_connection", event="test_event_post", type=EventConnectionType.POST), EventConnection( app_connection="test_app_plugin_connection", event="test_event_plugin", type=EventConnectionType.GET) ]) }, server=ServerConfig(logging=LoggingConfig( log_level="DEBUG", log_path="work/logs/test/"))).setup()
def mock_client_app_config(): return AppConfig( app=AppDescriptor(name='mock_client_app', version='test'), engine=AppEngineConfig(import_modules=['mock_app'], read_stream_timeout=1, read_stream_interval=5), events={ "mock_event": EventDescriptor(type=EventType.GET, route='mock-client-app/test/mock-event-test') }, server=ServerConfig( streams=StreamsConfig( stream_manager='mock_engine.MockStreamManager', delay_auto_start_seconds=0), logging=LoggingConfig(log_level="DEBUG", log_path="work/logs/test/"))).setup()
def _setup_server_context(app_config: AppConfig) -> EventContext: _init_engine_logger(app_config) assert app_config.server app_config.server.auth = AuthConfig( secrets_location=f"/tmp/{uuid.uuid4()}", auth_passphrase='test_passphrase', enabled=True, create_keys=True ) auth.init(app_config.app_key(), app_config.server.auth) return EventContext( app_config=app_config, plugin_config=app_config, event_name='mock_event', settings=get_event_settings(app_config.effective_settings, 'mock_event'), # type: ignore track_ids={}, auth_info={} )
def _set_path_security(event_api_spec: dict, app_config: AppConfig, event_info: EventDescriptor): """ Setup security schemes allowed for each path """ assert spec is not None security: list = [] for auth in event_info.auth: if auth == AuthType.REFRESH: _update_auth_refresh_method(app_config.app_key()) auth_str = f"{app_config.app_key()}.refresh" security.append({auth_str: []}) elif auth != AuthType.UNSECURED: auth_str = f"auth.{auth.value.lower()}" security.append({auth_str: []}) if len(security) == 0 and AuthType.UNSECURED not in event_info.auth: security = spec['security'] if len(security) > 0: event_api_spec['security'] = security
async def stream_startup_hook(app_config: AppConfig, *args, **kwargs): """ Start all stream event types configured in app. :param app_key: already started app_key """ app_engine = runtime.server.app_engines[app_config.app_key()] for event_name, event_info in app_engine.effective_events.items(): if event_info.type == EventType.STREAM: assert event_info.read_stream logger.info( __name__, f"STREAM start event_name={event_name} read_stream={event_info.read_stream.name}" ) asyncio.create_task(app_engine.read_stream(event_name=event_name)) elif event_info.type == EventType.SERVICE: logger.info(__name__, f"SERVICE start event_name={event_name}") asyncio.create_task(app_engine.service_loop(event_name=event_name))
def __init__(self, *, app_config: AppConfig, plugin_config: AppConfig, event_name: str, track_ids: Dict[str, str], auth_info: Dict[str, Any]): self.app_key: str = app_config.app_key() self.app: AppDescriptor = app_config.app self.env: Env = {**plugin_config.env, **app_config.env} self.event_name = event_name base_event = event_name.split('$')[0] self.event_info: EventDescriptor = plugin_config.events[base_event] self.creation_ts: datetime = datetime.now().astimezone(tz=timezone.utc) self.auth_info = auth_info track_fields = [ 'track.operation_id', *app_config.engine.track_headers, *(self.event_info.config.logging.stream_fields if self.event_info.type == EventType.STREAM else []) ] self.track_ids = {k: track_ids.get(k) or '' for k in track_fields}
def mock_client_app_config(): return AppConfig( app=AppDescriptor(name='mock_client_app', version='test'), engine=AppEngineConfig(import_modules=['mock_client_app']), app_connections={ "test_app_connection": AppConnection(name="test_app", version=APPS_API_VERSION, client="mock_client_app.MockClient") }, events={ "mock_client_event": EventDescriptor(type=EventType.GET, connections=[ EventConnection( app_connection="test_app_connection", event="test_event", type=EventConnectionType.GET) ]) }, server=ServerConfig(logging=LoggingConfig( log_level="DEBUG", log_path="work/logs/test/"))).setup()
def valid_result_app_config() -> AppConfig: return AppConfig( app=AppDescriptor(name="simple_example", version=APPS_API_VERSION), engine=AppEngineConfig(import_modules=["model"], read_stream_timeout=1, track_headers=["request_id", "correlation_id"]), env={ "fs": { "data_path": f"/tmp/simple_example.{APP_VERSION}.fs.data_path/", "app_description": f"This is simple_example version {APPS_API_VERSION}", "recursive_replacement": f"Data is in /tmp/simple_example.{APP_VERSION}.fs.data_path/. " + f"This is simple_example version {APPS_API_VERSION}" } }, events={ "query_something": EventDescriptor(type=EventType.GET), "save_something": EventDescriptor(type=EventType.POST), "streams.something_event": EventDescriptor( type=EventType.POST, write_stream=WriteStreamDescriptor( name=f'simple_example.{APP_VERSION}.streams.something_event' )), "streams.process_events": EventDescriptor( type=EventType.STREAM, read_stream=ReadStreamDescriptor( name= f'simple_example.{APP_VERSION}.streams.something_event', consumer_group= f'simple_example.{APP_VERSION}.streams.process_events'), config=EventConfig(logging=EventLoggingConfig( extra_fields=['something_id', 'path']))) })
from hopeit.app.config import AppConfig from hopeit.server.config import ServerConfig import json with open('app-config-schema-draftv6.json', 'w') as fb: fb.write(json.dumps(AppConfig.json_schema(), indent=2)) with open('server-config-schema-draftv6.json', 'w') as fb: fb.write(json.dumps(ServerConfig.json_schema(), indent=2))
def mock_app_config(): return AppConfig( app=AppDescriptor(name='mock_app', version='test'), plugins=[AppDescriptor('mock_plugin', 'test')], engine=AppEngineConfig(import_modules=['mock_app'], read_stream_timeout=1, read_stream_interval=5, track_headers=['session_id'], cors_origin='http://test'), env={ 'app': { 'app_value': 'test_app_value' }, 'plugin': { 'custom_value': 'test_custom_value_override' } }, settings={ "mock_stream_event": { "logging": { "extra_fields": ['value'], "stream_fields": ['msg_id'] }, "custom_setting": { "custom": "value" } }, "mock_stream_timeout": { "logging": { "extra_fields": ['value'], "stream_fields": ['msg_id'] }, "stream": { "timeout": 2 } }, "mock_write_stream_event": { "logging": { "extra_fields": ['value'], "stream_fields": ['msg_id'] }, "stream": { "target_max_len": 10 } }, "mock_service_event": { "stream": { "target_max_len": 10, "throttle_ms": 100, "batch_size": 2 } }, "mock_service_timeout": { "response_timeout": 2.0 }, "mock_spawn_event": { "stream": { "target_max_len": 10, "throttle_ms": 100, "batch_size": 2 } }, "mock_shuffle_event": { "stream": { "target_max_len": 10, "throttle_ms": 100 } }, "mock_timeout": { "response_timeout": 2.0 }, "custom_extra_settings": { "custom_setting": { "custom": "value" } } }, events={ "mock_event": EventDescriptor(type=EventType.GET, route='mock-app/test/mock-event-test', group='GROUP_A'), "mock_event_logging": EventDescriptor(type=EventType.GET, group='GROUP_A'), "mock_post_event": EventDescriptor(type=EventType.POST, route='mock-app/test/mock-event-test', group='GROUP_A'), "mock_multipart_event": EventDescriptor(type=EventType.MULTIPART, route='mock-app/test/mock-multipart-event-test', group='GROUP_B'), "mock_post_nopayload": EventDescriptor(type=EventType.POST, route='mock-app/test/mock-post-nopayload'), "mock_post_preprocess": EventDescriptor(type=EventType.POST, route='mock-app/test/mock-post-preprocess'), "mock_post_preprocess_no_datatype": EventDescriptor( type=EventType.POST, route='mock-app/test/mock-post-preprocess-no-datatype'), "mock_stream_event": EventDescriptor(type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_stream', consumer_group='mock_consumer_group'), setting_keys=["custom_extra_settings"], group='GROUP_A'), "mock_stream_timeout": EventDescriptor(type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_stream', consumer_group='mock_consumer_group'), group='GROUP_B'), "mock_write_stream_event": EventDescriptor(type=EventType.GET, write_stream=WriteStreamDescriptor( name='mock_write_stream_event')), "mock_service_event": EventDescriptor(type=EventType.SERVICE, write_stream=WriteStreamDescriptor( name='mock_write_stream_event')), "mock_service_timeout": EventDescriptor(type=EventType.SERVICE, write_stream=WriteStreamDescriptor( name='mock_write_stream_event')), "mock_spawn_event": EventDescriptor(type=EventType.GET, write_stream=WriteStreamDescriptor( name='mock_write_stream_event')), "mock_shuffle_event": EventDescriptor(type=EventType.GET, write_stream=WriteStreamDescriptor( name='mock_write_stream_event')), "mock_parallelize_event": EventDescriptor(type=EventType.GET), "mock_file_response": EventDescriptor(type=EventType.GET), "mock_stream_response": EventDescriptor(type=EventType.GET), "mock_file_response_content_type": EventDescriptor(type=EventType.GET), "mock_auth": EventDescriptor(type=EventType.GET, auth=[AuthType.BASIC]), "mock_post_auth": EventDescriptor(type=EventType.POST, auth=[AuthType.BASIC]), "mock_collector": EventDescriptor(type=EventType.POST), 'mock_timeout': EventDescriptor(type=EventType.GET), 'mock_read_write_stream': EventDescriptor(type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_read_write_stream.read', consumer_group='mock_read_write_stream'), write_stream=WriteStreamDescriptor( name='mock_read_write_stream.write')), "mock_event_custom": EventDescriptor(type=EventType.GET, impl="mock_app.mock_event_custom_impl"), "mock_event_dataobject_payload": EventDescriptor( type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_read_write_stream.read', consumer_group='mock_event_dataobject_payload'), dataobjects=["mock_app.MockData"]) }, server=ServerConfig( streams=StreamsConfig( stream_manager='mock_engine.MockStreamManager', delay_auto_start_seconds=0), logging=LoggingConfig(log_level="DEBUG", log_path="work/logs/test/"))).setup()
def mock_app_config(): return AppConfig( app=AppDescriptor( name='mock_app', version='test' ), plugins=[ AppDescriptor('mock_plugin', 'test') ], engine=AppEngineConfig( import_modules=['mock_app'], read_stream_timeout=1, read_stream_interval=5, track_headers=['session_id'], cors_origin='http://test' ), env={ 'app': { 'app_value': 'test_app_value' }, 'plugin': { 'custom_value': 'test_custom_value_override' } }, events={ "mock_event": EventDescriptor( type=EventType.GET, route='mock-app/test/mock-event-test' ), "mock_event_logging": EventDescriptor( type=EventType.GET ), "mock_post_event": EventDescriptor( type=EventType.POST, route='mock-app/test/mock-event-test' ), "mock_multipart_event": EventDescriptor( type=EventType.MULTIPART, route='mock-app/test/mock-multipart-event-test' ), "mock_post_nopayload": EventDescriptor( type=EventType.POST, route='mock-app/test/mock-post-nopayload' ), "mock_post_preprocess": EventDescriptor( type=EventType.POST, route='mock-app/test/mock-post-preprocess' ), "mock_stream_event": EventDescriptor( type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_stream', consumer_group='mock_consumer_group' ), config=EventConfig( logging=EventLoggingConfig( extra_fields=['value'], stream_fields=['msg_id'] ) ) ), "mock_stream_timeout": EventDescriptor( type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_stream', consumer_group='mock_consumer_group' ), config=EventConfig( logging=EventLoggingConfig( extra_fields=['value'], stream_fields=['msg_id'] ), stream=EventStreamConfig( timeout=2 ) ) ), "mock_write_stream_event": EventDescriptor( type=EventType.GET, write_stream=WriteStreamDescriptor( name='mock_write_stream_event' ), config=EventConfig( stream=EventStreamConfig( target_max_len=10 ) ) ), "mock_service_event": EventDescriptor( type=EventType.SERVICE, write_stream=WriteStreamDescriptor( name='mock_write_stream_event' ), config=EventConfig( stream=EventStreamConfig( target_max_len=10, throttle_ms=100, batch_size=2 ) ) ), "mock_service_timeout": EventDescriptor( type=EventType.SERVICE, write_stream=WriteStreamDescriptor( name='mock_write_stream_event' ), config=EventConfig( response_timeout=2.0 ) ), "mock_spawn_event": EventDescriptor( type=EventType.GET, write_stream=WriteStreamDescriptor( name='mock_write_stream_event' ), config=EventConfig( stream=EventStreamConfig( target_max_len=10, throttle_ms=100, batch_size=2 ) ) ), "mock_shuffle_event": EventDescriptor( type=EventType.GET, write_stream=WriteStreamDescriptor( name='mock_write_stream_event' ), config=EventConfig( stream=EventStreamConfig( target_max_len=10, throttle_ms=100 ) ) ), "mock_parallelize_event": EventDescriptor( type=EventType.GET ), "mock_file_response": EventDescriptor( type=EventType.GET ), "mock_file_response_content_type": EventDescriptor( type=EventType.GET ), "mock_auth": EventDescriptor( type=EventType.GET, auth=[AuthType.BASIC] ), "mock_post_auth": EventDescriptor( type=EventType.POST, auth=[AuthType.BASIC] ), "mock_collector": EventDescriptor( type=EventType.POST ), 'mock_timeout': EventDescriptor( type=EventType.GET, config=EventConfig(response_timeout=2.0) ), 'mock_read_write_stream': EventDescriptor( type=EventType.STREAM, read_stream=ReadStreamDescriptor( name='mock_read_write_stream.read', consumer_group='mock_read_write_stream' ), write_stream=WriteStreamDescriptor( name='mock_read_write_stream.write' ) ) }, server=ServerConfig( streams=StreamsConfig( stream_manager='mock_engine.MockStreamManager', delay_auto_start_seconds=0 ), logging=LoggingConfig( log_level="DEBUG", log_path="work/logs/test/") ) )