def app_mock(mock_env_devel_environment): cfg = {"director": {"enabled": True}} app = create_safe_application(cfg) setup_settings(app) assert setup_director(app) return app
def create(settings: Settings) -> web.Application: log.debug( "Initializing app with settings:\n%s", settings.json(indent=2, sort_keys=True), ) # TODO: tmp using {} until webserver is also pydantic-compatible app = create_safe_application(None) app[APP_CONFIG_KEY] = settings if settings.STORAGE_TRACING.enabled: setup_tracing( app, "simcore_service_storage", settings.STORAGE_HOST, settings.STORAGE_PORT, settings.STORAGE_TRACING.dict(), ) setup_db(app) # -> postgres service setup_s3(app) # -> minio service setup_dsm(app) # core subsystem. Needs s3 and db setups done setup_rest(app) # lastly, we expose API to the world if settings.STORAGE_MONITORING_ENABLED: setup_monitoring(app, "simcore_service_storage") return app
def client(loop, aiohttp_unused_port, aiohttp_client, api_version_prefix): app = create_safe_application() MAX_DELAY_SECS_ALLOWED = 1 # secs async def slow_handler(request: web.Request): import time time.sleep(MAX_DELAY_SECS_ALLOWED * 1.1) raise web.HTTPOk() server_kwargs = {"port": aiohttp_unused_port(), "host": "localhost"} # fake config app[APP_CONFIG_KEY] = { "main": server_kwargs, "rest": { "enabled": True, "version": api_version_prefix }, } # activates only security+restAPI sub-modules setup_settings(app) setup_security(app) setup_rest(app) app.router.add_get("/slow", slow_handler) cli = loop.run_until_complete( aiohttp_client(app, server_kwargs=server_kwargs)) return cli
def client( loop, aiohttp_client, app_config, ## waits until swarm with *_services are up rabbit_service: RabbitConfig, ## waits until rabbit is responsive postgres_db: sa.engine.Engine, ): assert app_config["rest"]["version"] == API_VERSION app_config["storage"]["enabled"] = False # fake config app = create_safe_application() app[APP_CONFIG_KEY] = app_config setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_login(app) setup_projects(app) setup_computation(app) setup_director_v2(app) setup_socketio(app) setup_resource_manager(app) yield loop.run_until_complete( aiohttp_client( app, server_kwargs={ "port": app_config["main"]["port"], "host": app_config["main"]["host"], }, ))
def client( loop, aiohttp_client, app_config, ## waits until swarm with *_services are up ): assert app_config["rest"]["version"] == API_VERSION app_config["storage"]["enabled"] = False app_config["main"]["testing"] = True # fake config app = create_safe_application(app_config) setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_login(app) setup_users(app) setup_socketio(app) setup_projects(app) setup_computation(app) setup_director_v2(app) setup_resource_manager(app) yield loop.run_until_complete( aiohttp_client( app, server_kwargs={ "port": app_config["main"]["port"], "host": app_config["main"]["host"], }, ))
def client( event_loop: asyncio.AbstractEventLoop, unused_tcp_port_factory, aiohttp_client, api_version_prefix, mock_env_devel_environment, ): app = create_safe_application() MAX_DELAY_SECS_ALLOWED = 1 # secs async def slow_handler(request: web.Request): import time time.sleep(MAX_DELAY_SECS_ALLOWED * 1.1) raise web.HTTPOk() server_kwargs = {"port": unused_tcp_port_factory(), "host": "localhost"} # activates only security+restAPI sub-modules setup_settings(app) setup_security(app) setup_rest(app) app.router.add_get("/slow", slow_handler) cli = event_loop.run_until_complete( aiohttp_client(app, server_kwargs=server_kwargs)) return cli
def client(loop, aiohttp_client: TestClient): app = create_safe_application() app[APP_OPENAPI_SPECS_KEY] = load_openapi_specs() setup_catalog.__wrapped__(app) yield loop.run_until_complete(aiohttp_client(app))
def create(settings: Settings) -> web.Application: log.debug( "Initializing app with settings:\n%s", settings.json(indent=2, sort_keys=True), ) # TODO: tmp using {} until webserver is also pydantic-compatible app = create_safe_application(None) app[APP_CONFIG_KEY] = settings if settings.STORAGE_TRACING: setup_tracing( app, service_name="simcore_service_storage", host=settings.STORAGE_HOST, port=settings.STORAGE_PORT, jaeger_base_url= f"{settings.STORAGE_TRACING.TRACING_ZIPKIN_ENDPOINT}", skip_routes=None, ) setup_db(app) # -> postgres service setup_s3(app) # -> minio service setup_dsm(app) # core subsystem. Needs s3 and db setups done setup_rest(app) # lastly, we expose API to the world if settings.STORAGE_MONITORING_ENABLED: setup_monitoring(app, app_name, version=f"{version}") return app
def client( event_loop, aiohttp_client, app_cfg, postgres_db, monkeypatch_setenv_from_app_config: Callable, ): cfg = deepcopy(app_cfg) port = cfg["main"]["port"] assert cfg["rest"]["version"] == API_VERSION monkeypatch_setenv_from_app_config(cfg) # fake config app = create_safe_application(cfg) assert setup_settings(app) setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_login(app) setup_users(app) setup_groups(app) client = event_loop.run_until_complete( aiohttp_client(app, server_kwargs={ "port": port, "host": "localhost" })) return client
def client( event_loop: asyncio.AbstractEventLoop, aiohttp_client: Callable[..., Awaitable[TestClient]], ): app = create_safe_application() app[APP_OPENAPI_SPECS_KEY] = load_openapi_specs() setup_catalog.__wrapped__(app) yield event_loop.run_until_complete(aiohttp_client(app))
def client( loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, app_cfg: Dict[str, Any], postgres_db: sa.engine.Engine, mock_orphaned_services, redis_client: Redis, monkeypatch_setenv_from_app_config: Callable, ) -> TestClient: cfg = deepcopy(app_cfg) assert cfg["rest"]["version"] == API_VTAG assert cfg["rest"]["enabled"] cfg["projects"]["enabled"] = True cfg["director"]["enabled"] = True # sets TTL of a resource after logout cfg["resource_manager"][ "resource_deletion_timeout_seconds"] = SERVICE_DELETION_DELAY monkeypatch_setenv_from_app_config(cfg) app = create_safe_application(cfg) # activates only security+restAPI sub-modules assert setup_settings(app) setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_login(app) setup_users(app) setup_socketio(app) setup_projects(app) setup_director(app) setup_director_v2(app) assert setup_resource_manager(app) assert is_setup_completed("simcore_service_webserver.resource_manager", app) # NOTE: garbage_collector is disabled and instead explicitly called using # garbage_collector_core.collect_garbage assert not is_setup_completed( "simcore_service_webserver.garbage_collector", app) return loop.run_until_complete( aiohttp_client( app, server_kwargs={ "port": cfg["main"]["port"], "host": cfg["main"]["host"] }, ))
def client( loop: asyncio.AbstractEventLoop, postgres_session: sa.orm.session.Session, rabbit_service: RabbitConfig, redis_service: RedisConfig, simcore_services_ready: None, aiohttp_client: Callable, app_config: Dict[str, Any], ## waits until swarm with *_services are up mocker: MockerFixture, monkeypatch_setenv_from_app_config: Callable, ) -> TestClient: cfg = deepcopy(app_config) assert cfg["rest"]["version"] == API_VTAG cfg["storage"]["enabled"] = False cfg["main"]["testing"] = True # fake config monkeypatch_setenv_from_app_config(cfg) app = create_safe_application(app_config) setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_diagnostics(app) setup_login(app) setup_users(app) setup_socketio(app) setup_projects(app) setup_computation(app) setup_director_v2(app) # GC not included in this test-suite, mocker.patch( "simcore_service_webserver.resource_manager.module_setup.setup_garbage_collector", side_effect=lambda app: print( f"PATCH @{__name__}:" "Garbage collector disabled." "Mock bypasses setup_garbage_collector to skip initializing the GC" ), ) setup_resource_manager(app) return loop.run_until_complete( aiohttp_client( app, server_kwargs={ "port": app_config["main"]["port"], "host": app_config["main"]["host"], }, ))
def client(mocked_celery_client, loop, aiohttp_client, app_config, mock_orphaned_services): app = create_safe_application(app_config) setup_session(app) setup_security(app) setup_rest(app) setup_activity(app) cli = loop.run_until_complete(aiohttp_client(app)) return cli
def client(loop, aiohttp_client, monkeypatch): monkeypatch.setenv("WEBSERVER_DEV_FEATURES_ENABLED", "1") cfg = load_default_config() app = create_safe_application(cfg) app[APP_OPENAPI_SPECS_KEY] = load_openapi_specs() # __wrapped__ avoids app modules dependency checks setup_catalog.__wrapped__(app, disable_auth=True) # needs to start application ... yield loop.run_until_complete(aiohttp_client(app))
def client( loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, app_config: Dict, postgres_with_template_db: aiopg.sa.engine.Engine, mock_orphaned_services: mock.Mock, monkeypatch_setenv_from_app_config: Callable, ): # test config & env vars ---------------------- cfg = deepcopy(app_config) assert cfg["rest"]["version"] == API_VERSION assert cfg["rest"]["enabled"] cfg["projects"]["enabled"] = True cfg["director"]["enabled"] = True cfg["exporter"]["enabled"] = True monkeypatch_setenv_from_app_config(cfg) # app setup ---------------------------------- app = create_safe_application(cfg) # activates only security+restAPI sub-modules setup_settings(app) assert get_exporter_settings(app) is not None, "Should capture defaults" setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_login(app) setup_users(app) setup_socketio(app) setup_projects(app) setup_director(app) setup_director_v2(app) setup_exporter(app) # <---- under test setup_storage(app) setup_products(app) setup_catalog(app) setup_scicrunch_submodule(app) assert setup_resource_manager(app) yield loop.run_until_complete( aiohttp_client( app, server_kwargs={ "port": cfg["main"]["port"], "host": cfg["main"]["host"] }, ))
async def fake_app(mock_env_devel_environment, loop): # By using .env-devel we ensure all needed variables are at # least defined there print("app's environment variables", format(mock_env_devel_environment)) app = create_safe_application() setup_settings(app) setup_scicrunch(app) yield app client = get_client_session(app) await client.close()
def client( loop, aiohttp_client, app_config, postgres_with_template_db, mock_orphaned_services, monkeypatch_setenv_from_app_config: Callable, ): cfg = deepcopy(app_config) assert cfg["rest"]["version"] == API_VERSION assert cfg["rest"]["enabled"] cfg["projects"]["enabled"] = True cfg["director"]["enabled"] = True cfg["resource_manager"].update({ "garbage_collection_interval_seconds": GARBAGE_COLLECTOR_INTERVAL, # increase speed of garbage collection "resource_deletion_timeout_seconds": SERVICE_DELETION_DELAY, # reduce deletion delay }) monkeypatch_setenv_from_app_config(cfg) app = create_safe_application(cfg) # activates only security+restAPI sub-modules assert setup_settings(app) setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_login(app) setup_users(app) setup_socketio(app) setup_projects(app) setup_director(app) setup_director_v2(app) assert setup_resource_manager(app) setup_garbage_collector(app) yield loop.run_until_complete( aiohttp_client( app, server_kwargs={ "port": cfg["main"]["port"], "host": cfg["main"]["host"] }, ))
def client( event_loop, aiohttp_client, app_cfg, postgres_db, mocked_director_v2_api, mock_orphaned_services, redis_client, monkeypatch_setenv_from_app_config: Callable, ): # config app cfg = deepcopy(app_cfg) port = cfg["main"]["port"] cfg["projects"]["enabled"] = True cfg["director"]["enabled"] = True cfg["resource_manager"][ "garbage_collection_interval_seconds" ] = DEFAULT_GARBAGE_COLLECTOR_INTERVAL_SECONDS # increase speed of garbage collection cfg["resource_manager"][ "resource_deletion_timeout_seconds" ] = DEFAULT_GARBAGE_COLLECTOR_DELETION_TIMEOUT_SECONDS # reduce deletion delay monkeypatch_setenv_from_app_config(cfg) app = create_safe_application(cfg) # setup app assert setup_settings(app) setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_login(app) # needed for login_utils fixtures setup_resource_manager(app) setup_garbage_collector(app) setup_socketio(app) setup_director(app) setup_director_v2(app) setup_tags(app) assert setup_projects(app) setup_products(app) # server and client yield event_loop.run_until_complete( aiohttp_client(app, server_kwargs={"port": port, "host": "localhost"}) )
def client( event_loop: asyncio.AbstractEventLoop, postgres_session: sa.orm.session.Session, rabbit_service: RabbitSettings, redis_settings: RedisSettings, simcore_services_ready: None, aiohttp_client: Callable, app_config: Dict[str, Any], ## waits until swarm with *_services are up mocker: MockerFixture, monkeypatch_setenv_from_app_config: Callable, ) -> TestClient: cfg = deepcopy(app_config) assert cfg["rest"]["version"] == API_VTAG cfg["storage"]["enabled"] = False cfg["main"]["testing"] = True # fake config monkeypatch_setenv_from_app_config(cfg) app = create_safe_application(app_config) assert setup_settings(app) setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_diagnostics(app) setup_login(app) setup_users(app) setup_socketio(app) setup_projects(app) setup_computation(app) setup_director_v2(app) setup_resource_manager(app) # no garbage collector return event_loop.run_until_complete( aiohttp_client( app, server_kwargs={ "port": app_config["main"]["port"], "host": app_config["main"]["host"], }, ))
def client( mock_garbage_collector_task, loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, app_cfg: Dict[str, Any], postgres_db: sa.engine.Engine, mock_orphaned_services, redis_client: Redis, monkeypatch_setenv_from_app_config: Callable, ) -> TestClient: cfg = deepcopy(app_cfg) assert cfg["rest"]["version"] == API_VERSION assert cfg["rest"]["enabled"] cfg["projects"]["enabled"] = True cfg["director"]["enabled"] = True cfg[config.CONFIG_SECTION_NAME][ "garbage_collection_interval_seconds"] = GARBAGE_COLLECTOR_INTERVAL # increase speed of garbage collection # fake config monkeypatch_setenv_from_app_config(cfg) app = create_safe_application(cfg) # activates only security+restAPI sub-modules setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_login(app) setup_users(app) setup_socketio(app) setup_projects(app) setup_director(app) setup_director_v2(app) assert setup_resource_manager(app) return loop.run_until_complete( aiohttp_client( app, server_kwargs={ "port": cfg["main"]["port"], "host": cfg["main"]["host"] }, ))
def client( loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, app_config: Dict[str, Any], ## waits until swarm with *_services are up rabbit_service: RabbitConfig, ## waits until rabbit is responsive and set env vars postgres_db: sa.engine.Engine, mocker: MockerFixture, monkeypatch_setenv_from_app_config: Callable, ): app_config["storage"]["enabled"] = False monkeypatch_setenv_from_app_config(app_config) app = create_safe_application(app_config) setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_diagnostics(app) setup_login(app) setup_projects(app) setup_computation(app) setup_director_v2(app) setup_socketio(app) # GC not relevant for these test-suite, mocker.patch( "simcore_service_webserver.resource_manager.module_setup.setup_garbage_collector", side_effect=lambda app: print( f"PATCH @{__name__}:" "Garbage collector disabled." "Mock bypasses setup_garbage_collector to skip initializing the GC" ), ) setup_resource_manager(app) return loop.run_until_complete( aiohttp_client( app, server_kwargs={ "port": app_config["main"]["port"], "host": app_config["main"]["host"], }, ))
def create_application(config: Dict[str, Any]) -> web.Application: """ Initializes service """ log.debug( "Initializing app with config:\n%s", json.dumps(config, indent=2, sort_keys=True), ) app = create_safe_application(config) setup_settings(app) # TODO: create dependency mechanism # and compute setup order https://github.com/ITISFoundation/osparc-simcore/issues/1142 setup_app_tracing(app) setup_statics(app) setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_diagnostics(app) setup_email(app) setup_computation(app) setup_socketio(app) setup_login(app) setup_director(app) setup_director_v2(app) setup_storage(app) setup_users(app) setup_groups(app) setup_projects(app) setup_version_control(app) setup_activity(app) setup_resource_manager(app) setup_tags(app) setup_catalog(app) setup_publications(app) setup_products(app) setup_studies_access(app) setup_studies_dispatcher(app) setup_exporter(app) setup_clusters(app) return app
def client( loop, aiohttp_client, app_config, mock_orphaned_services, monkeypatch_setenv_from_app_config: Callable, ): monkeypatch_setenv_from_app_config(app_config) app = create_safe_application(app_config) setup_session(app) setup_security(app) setup_rest(app) setup_activity(app) cli = loop.run_until_complete(aiohttp_client(app)) return cli
def client( loop, mock_orphaned_services, aiohttp_client, app_config, # waits until swarm with *_services are up monkeypatch_setenv_from_app_config: Callable, ): assert app_config["rest"]["version"] == API_VERSION app_config["main"]["testing"] = True app_config["storage"]["enabled"] = False app_config["computation"]["enabled"] = False monkeypatch_setenv_from_app_config(app_config) app = create_safe_application(app_config) assert setup_settings(app) setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_login(app) setup_resource_manager(app) setup_garbage_collector(app) assert setup_projects(app) setup_catalog(app) setup_products(app) setup_director_v2(app) yield loop.run_until_complete( aiohttp_client( app, server_kwargs={ "port": app_config["main"]["port"], "host": app_config["main"]["host"], }, ) )
def client( event_loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, app_config: Dict[str, Any], ## waits until swarm with *_services are up rabbit_service: RabbitSettings, ## waits until rabbit is responsive and set env vars postgres_db: sa.engine.Engine, mocker: MockerFixture, monkeypatch_setenv_from_app_config: Callable, ): app_config["storage"]["enabled"] = False monkeypatch_setenv_from_app_config(app_config) app = create_safe_application(app_config) assert setup_settings(app) assert app[APP_SETTINGS_KEY].WEBSERVER_COMPUTATION setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_diagnostics(app) setup_login(app) setup_projects(app) setup_computation(app) setup_director_v2(app) setup_socketio(app) setup_resource_manager(app) # GC not relevant for these test-suite, return event_loop.run_until_complete( aiohttp_client( app, server_kwargs={ "port": app_config["main"]["port"], "host": app_config["main"]["host"], }, ) )
def redis_enabled_app(redis_client: aioredis.Redis, mocker, mock_env_devel_environment) -> web.Application: # app.cleanup_ctx.append(redis_client) in setup_redis would create a client and connect # to a real redis service. Instead, we mock the get_redis_client access mocker.patch("simcore_service_webserver.redis.get_redis_client", return_value=redis_client) mocker.patch( "simcore_service_webserver.resource_manager.registry.get_redis_client", return_value=redis_client, ) # ------------------ app = create_safe_application() assert setup_settings(app) assert setup_resource_manager(app) assert is_setup_completed("simcore_service_webserver.redis", app) assert get_plugin_settings(app).RESOURCE_MANAGER_RESOURCE_TTL_S == 3 assert get_registry(app) return app
def client(loop, aiohttp_client, app_config, postgres_with_template_db, mock_orphaned_services): cfg = deepcopy(app_config) assert cfg["rest"]["version"] == API_VERSION assert cfg["rest"]["enabled"] cfg["projects"]["enabled"] = True cfg["director"]["enabled"] = True # fake config app = create_safe_application(cfg) # activates only security+restAPI sub-modules setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_login(app) setup_users(app) setup_socketio(app) setup_projects(app) setup_director(app) setup_director_v2(app) setup_exporter(app) setup_storage(app) setup_products(app) setup_catalog(app) setup_scicrunch_submodule(app) assert setup_resource_manager(app) yield loop.run_until_complete( aiohttp_client( app, server_kwargs={ "port": cfg["main"]["port"], "host": cfg["main"]["host"] }, ))
def client(loop, aiohttp_client, app_cfg, postgres_db): cfg = deepcopy(app_cfg) port = cfg["main"]["port"] assert cfg["rest"]["version"] == API_VERSION # fake config app = create_safe_application(cfg) setup_db(app) setup_session(app) setup_security(app) setup_rest(app) setup_login(app) setup_users(app) setup_groups(app) client = loop.run_until_complete( aiohttp_client(app, server_kwargs={ "port": port, "host": "localhost" })) return client
def moduleless_app(loop, aiohttp_server) -> web.Application: app: web.Application = create_safe_application() # creates a dummy server server = loop.run_until_complete(aiohttp_server(app)) # server is destroyed on exit https://docs.aiohttp.org/en/stable/testing.html#pytest_aiohttp.aiohttp_server return app
def client(loop, aiohttp_unused_port, aiohttp_client, api_version_prefix, monkeypatch): SLOW_HANDLER_DELAY_SECS = 2.0 # secs # pylint:disable=unused-variable routes = web.RouteTableDef() @routes.get("/error") async def unexpected_error(request: web.Request): raise Exception("boom shall produce 500") @routes.get(r"/fail") async def expected_failure(request: web.Request): raise web.HTTPServiceUnavailable() @routes.get(r"/slow") async def blocking_slow(request: web.Request): time.sleep(SLOW_HANDLER_DELAY_SECS * 1.1) return web.json_response({"data": True, "error": None}) @routes.get(r"/cancel") async def cancelled_task(request: web.Request): task: asyncio.Task = request.app.loop.create_task(asyncio.sleep(10)) task.cancel() # raise CancelledError @routes.get(r"/timeout/{secs}") async def time_out(request: web.Request): secs = float(request.match_info.get("secs", 0)) await asyncio.wait_for(asyncio.sleep(10 * secs), timeout=secs) # raises TimeOutError @routes.get(r"/delay/{secs}") async def delay_response(request: web.Request): secs = float(request.match_info.get("secs", 0)) await asyncio.sleep(secs) # non-blocking slow return web.json_response({"data": True, "error": None}) # ----- app = create_safe_application() main = {"port": aiohttp_unused_port(), "host": "localhost"} app[APP_CONFIG_KEY] = { "main": main, "rest": { "enabled": True, "version": api_version_prefix }, "diagnostics": { "enabled": True }, } # activates some sub-modules setup_security(app) setup_rest(app) monkeypatch.setenv("AIODEBUG_SLOW_DURATION_SECS", f"{SLOW_HANDLER_DELAY_SECS / 10}") monkeypatch.setenv("DIAGNOSTICS_MAX_TASK_DELAY", f"{SLOW_HANDLER_DELAY_SECS}") monkeypatch.setenv("DIAGNOSTICS_MAX_AVG_LATENCY", f"{2.0}") monkeypatch.setenv("DIAGNOSTICS_START_SENSING_DELAY", f"{0}") # inmidiately setup_diagnostics(app, ) assert app[kMAX_AVG_RESP_LATENCY] == 2.0 app.router.add_routes(routes) cli = loop.run_until_complete( aiohttp_client( app, server_kwargs={key: main[key] for key in ("host", "port")})) return cli