def init( config: pyramid.config.Configurator, master_prefix: str, slave_prefix: Optional[str] = None, force_master: Optional[Iterable[str]] = None, force_slave: Optional[Iterable[str]] = None, ) -> SessionFactory: """ Initialize the database for a Pyramid app. Arguments: config: The pyramid Configuration object master_prefix: The prefix for the master connection configuration entries in the application \ settings slave_prefix: The prefix for the slave connection configuration entries in the application \ settings force_master: The method/paths that needs to use the master force_slave: The method/paths that needs to use the slave Returns: The SQLAlchemy session """ settings = config.get_settings() settings["tm.manager_hook"] = "pyramid_tm.explicit_manager" # hook to share the dbengine fixture in testing dbengine = settings.get("dbengine") if not dbengine: rw_engine = get_engine(settings, master_prefix + ".") rw_engine.c2c_name = master_prefix # Setup a slave DB connection and add a tween to use it. if slave_prefix and settings[master_prefix + ".url"] != settings.get( slave_prefix + ".url"): LOG.info("Using a slave DB for reading %s", master_prefix) ro_engine = get_engine(config.get_settings(), slave_prefix + ".") ro_engine.c2c_name = slave_prefix else: ro_engine = rw_engine else: ro_engine = rw_engine = dbengine session_factory = SessionFactory(force_master, force_slave, ro_engine, rw_engine) config.registry["dbsession_factory"] = session_factory # make request.dbsession available for use in Pyramid def dbsession(request: pyramid.request.Request) -> sqlalchemy.orm.Session: # hook to share the dbsession fixture in testing dbsession = request.environ.get("app.dbsession") if dbsession is None: # request.tm is the transaction manager used by pyramid_tm dbsession = get_tm_session_pyramid(session_factory, request.tm, request=request) return dbsession config.add_request_method(dbsession, reify=True) return session_factory
def env_or_config(config: Optional[pyramid.config.Configurator], env_name: Optional[str] = None, config_name: Optional[str] = None, default: Any = None, type_: Callable[[str], Any] = str) -> Any: return env_or_settings(config.get_settings() if config is not None else {}, env_name, config_name, default, type_)
def includeme(config: Optional[pyramid.config.Configurator] = None) -> None: """ Initialize the broadcaster with Redis, if configured. Otherwise, fall back to a fake local implementation. """ global _broadcaster broadcast_prefix = config_utils.env_or_config(config, BROADCAST_ENV_KEY, BROADCAST_CONFIG_KEY, "broadcast_api_") master, slave, _ = redis_utils.get( config.get_settings() if config else None) if _broadcaster is None: if master is not None and slave is not None: _broadcaster = redis.RedisBroadcaster(broadcast_prefix, master, slave) else: _broadcaster = local.LocalBroadcaster() LOG.info("Broadcast service setup using local implementation") elif isinstance(_broadcaster, local.LocalBroadcaster ) and master is not None and slave is not None: LOG.info("Switching from a local broadcaster to a redis broadcaster") prev_broadcaster = _broadcaster _broadcaster = redis.RedisBroadcaster(broadcast_prefix, master, slave) _broadcaster.copy_local_subscriptions(prev_broadcaster)
def setup_session( config: pyramid.config.Configurator, master_prefix: str, slave_prefix: Optional[str] = None, force_master: Optional[Iterable[str]] = None, force_slave: Optional[Iterable[str]] = None, ) -> Tuple[Union[sqlalchemy.orm.Session, sqlalchemy.orm.scoped_session], sqlalchemy.engine.Engine, sqlalchemy.engine.Engine, ]: """ Create a SQLAlchemy session. With an accompanying tween that switches between the master and the slave DB connection. Uses prefixed entries in the application's settings. The slave DB will be used for anything that is GET and OPTIONS queries. The master DB will be used for all the other queries. You can tweak this behavior with the force_master and force_slave parameters. Those parameters are lists of regex that are going to be matched against "{VERB} {PATH}". Warning, the path includes the route_prefix. Arguments: config: The pyramid Configuration object master_prefix: The prefix for the master connection configuration entries in the application \ settings slave_prefix: The prefix for the slave connection configuration entries in the application \ settings force_master: The method/paths that needs to use the master force_slave: The method/paths that needs to use the slave Returns: The SQLAlchemy session, the R/W engine and the R/O engine """ warnings.warn( "setup_session function is deprecated; use init and request.dbsession instead" ) if slave_prefix is None: slave_prefix = master_prefix settings = config.registry.settings rw_engine = sqlalchemy.engine_from_config(settings, master_prefix + ".") rw_engine.c2c_name = master_prefix factory = sqlalchemy.orm.sessionmaker(bind=rw_engine) register(factory) db_session = sqlalchemy.orm.scoped_session(factory) # Setup a slave DB connection and add a tween to use it. if settings[master_prefix + ".url"] != settings.get(slave_prefix + ".url"): LOG.info("Using a slave DB for reading %s", master_prefix) ro_engine = sqlalchemy.engine_from_config(config.get_settings(), slave_prefix + ".") ro_engine.c2c_name = slave_prefix tween_name = master_prefix.replace(".", "_") _add_tween(config, tween_name, db_session, force_master, force_slave) else: ro_engine = rw_engine db_session.c2c_rw_bind = rw_engine db_session.c2c_ro_bind = ro_engine return db_session, rw_engine, ro_engine
def _init_domain(config: pyramid.config.Configurator) -> None: settings: Final = config.get_settings() domain_factory: Final = sample.bootstrap.domain_factory(settings) def request_method( request: pyramid.request.Request) -> sample.domain.Domain: return domain_factory(request.repository) config.add_request_method(request_method, 'domain', reify=True)
def env_or_config( config: Optional[pyramid.config.Configurator], env_name: Optional[str] = None, config_name: Optional[str] = None, default: Any = None, type_: Callable[[str], Any] = str, ) -> Any: """Get the setting from the environment or from the config file.""" return env_or_settings(config.get_settings() if config is not None else {}, env_name, config_name, default, type_)
def init(config: pyramid.config.Configurator, health_check: c2cwsgiutils.health_check.HealthCheck) -> None: """Initialize the checkers.""" global_settings = config.get_settings() if "checker" not in global_settings: return settings = global_settings["checker"] _routes(settings, health_check) _pdf3(settings, health_check) _fts(settings, health_check) _themes_errors(settings, health_check) _lang_files(global_settings, settings, health_check) _phantomjs(settings, health_check)
def add_getitfixed(config: pyramid.config.Configurator) -> None: if config.get_settings()["getitfixed"].get("enabled", False): for route_name, pattern in ( ("getitfixed_add_ending_slash", "/getitfixed"), ("getitfixed_admin_add_ending_slash", "/getitfixed_admin"), ): config.add_view(c2cgeoportal_geoportal.views.add_ending_slash, route_name=route_name) config.add_route(route_name, pattern, request_method="GET") config.include("getitfixed") # Register admin and getitfixed search paths together Form.set_zpt_renderer(c2cgeoform.default_search_paths, translator=translator)
def add_admin_interface(config: pyramid.config.Configurator) -> None: if config.get_settings().get("enable_admin_interface", False): config.add_request_method( lambda request: c2cgeoportal_commons.models.DBSession(), "dbsession", reify=True, ) config.add_view(c2cgeoportal_geoportal.views.add_ending_slash, route_name="admin_add_ending_slash") config.add_route("admin_add_ending_slash", "/admin", request_method="GET") config.include("c2cgeoportal_admin")
def init(config: pyramid.config.Configurator) -> None: """ Initialize the whole stats module. :param config: The Pyramid config """ stats.init_backends(config.get_settings()) if stats.BACKENDS: # pragma: nocover if 'memory' in stats.BACKENDS: # pragma: nocover from . import _views _views.init(config) from . import _pyramid_spy _pyramid_spy.init(config) init_db_spy()
def _init_repository(config: pyramid.config.Configurator) -> None: settings: Final = config.get_settings() settings['tm.manager_hook'] = 'pyramid_tm.explicit_manager' config.include('pyramid_tm') session_factory = sample.bootstrap.sqlalchemy_session_factory_factory() def request_method( request: pyramid.request.Request) -> sample.repository.Repository: session: Final = session_factory() zope.sqlalchemy.register(session, transaction_manager=request.tm) return sample.repository.Repository(session) config.add_request_method(request_method, 'repository', reify=True)
def init(config: pyramid.config.Configurator, health_check: c2cwsgiutils.health_check.HealthCheck) -> None: """ Initialize the check collector. Add him in the c2cwsgichecks. """ global_settings = config.get_settings() if "check_collector" not in global_settings: return settings = global_settings["check_collector"] c2c_base = global_settings.get("c2c.base_path", "") max_level = settings["max_level"] for host in settings["hosts"]: class Check: def __init__(self, host: Dict[str, Any]): self.host = host def __call__( self, request: pyramid.request.Request ) -> Optional[Dict[str, Any]]: params = request.params display = self.host["display"] if "host" not in params or display == params["host"]: url_headers = build_url( "check_collector", f"{self.host['url'].rstrip('/')}/{c2c_base.strip('/')}/health_check", request, ) r = requests.get( params={ "max_level": str(self.host.get("max_level", max_level)) }, timeout=120, **url_headers, # type: ignore ) r.raise_for_status() return cast(Dict[str, Any], r.json()) return None health_check.add_custom_check(name="check_collector_" + host["display"], check_cb=Check(host), level=settings["level"])
def includeme(config): """Set up standard configurator registrations. Use via: .. code-block:: python config = Configurator() config.include('pyramid_stripe') """ config.set_request_property(request.add_stripe_event, "stripe", reify=True) config.set_request_property(request.add_stripe_event_raw, "stripe_raw", reify=True) config.include(add_routes) config.scan("pyramid_stripe.views") if not stripe.api_key: stripe.api_key = config.get_settings()["stripe.api_key"]
def includeme(config: pyramid.config.Configurator) -> None: """Initialize the index page.""" base_path = config_utils.get_base_path(config) if base_path != "": config.add_route("c2c_index", base_path, request_method=("GET", "POST")) config.add_view(_index, route_name="c2c_index", http_cache=0) config.add_route("c2c_index_slash", base_path + "/", request_method=("GET", "POST")) config.add_view(_index, route_name="c2c_index_slash", http_cache=0) settings = config.get_settings() auth_type_ = auth_type(settings) if auth_type_ == AuthenticationType.SECRET: LOG.warning( "It is recommended to use OAuth2 with GitHub login instead of the `C2C_SECRET` because it " "protects from brute force attacks and the access grant is personal and can be revoked." ) if auth_type_ == AuthenticationType.GITHUB: config.add_route("c2c_github_login", base_path + "/github-login", request_method=("GET", )) config.add_view(_github_login, route_name="c2c_github_login", http_cache=0) config.add_route("c2c_github_callback", base_path + "/github-callback", request_method=("GET", )) config.add_view(_github_login_callback, route_name="c2c_github_callback", http_cache=0, renderer="fast_json") config.add_route("c2c_github_logout", base_path + "/github-logout", request_method=("GET", )) config.add_view(_github_logout, route_name="c2c_github_logout", http_cache=0)
def includeme(config: pyramid.config.Configurator) -> None: """ This function returns a Pyramid WSGI application. """ settings = config.get_settings() config.include("c2cgeoportal_commons") if "available_locale_names" not in settings: settings["available_locale_names"] = available_locale_names() call_hook(settings, "after_settings", settings) get_user_from_request = create_get_user_from_request(settings) config.add_request_method(get_user_from_request, name="user", property=True) config.add_request_method(get_user_from_request, name="get_user") # Configure 'locale' dir as the translation dir for c2cgeoportal app config.add_translation_dirs("c2cgeoportal_geoportal:locale/") config.include("c2cwsgiutils.pyramid.includeme") health_check = HealthCheck(config) config.registry["health_check"] = health_check metrics_config = config.registry.settings["metrics"] if metrics_config["memory_maps_rss"]: add_provider(MemoryMapProvider("rss")) if metrics_config["memory_maps_size"]: add_provider(MemoryMapProvider("size")) if metrics_config["memory_cache"]: add_provider( MemoryCacheSizeProvider( metrics_config.get("memory_cache_all", False))) if metrics_config["raster_data"]: add_provider(RasterDataSizeProvider()) if metrics_config["total_python_object_memory"]: add_provider(TotalPythonObjectMemoryProvider()) # Initialise DBSessions init_dbsessions(settings, config, health_check) checker.init(config, health_check) check_collector.init(config, health_check) # dogpile.cache configuration if "cache" in settings: register_backend("c2cgeoportal.hybrid", "c2cgeoportal_geoportal.lib.caching", "HybridRedisBackend") register_backend("c2cgeoportal.hybridsentinel", "c2cgeoportal_geoportal.lib.caching", "HybridRedisSentinelBackend") for name, cache_config in settings["cache"].items(): caching.init_region(cache_config, name) @zope.event.classhandler.handler(InvalidateCacheEvent) def handle(event: InvalidateCacheEvent) -> None: # pylint: disable=unused-variable del event caching.invalidate_region() if caching.MEMORY_CACHE_DICT: caching.get_region("std").delete_multi( caching.MEMORY_CACHE_DICT.keys()) caching.MEMORY_CACHE_DICT.clear() # Register a tween to get back the cache buster path. if "cache_path" not in config.get_settings(): config.get_settings()["cache_path"] = ["static"] config.add_tween( "c2cgeoportal_geoportal.lib.cacheversion.CachebusterTween") config.add_tween("c2cgeoportal_geoportal.lib.headers.HeadersTween") # Bind the mako renderer to other file extensions add_mako_renderer(config, ".html") add_mako_renderer(config, ".js") # Add the "geojson" renderer config.add_renderer("geojson", GeoJSON()) # Add the "xsd" renderer config.add_renderer("xsd", XSD(include_foreign_keys=True)) # Add the set_user_validator directive, and set a default user validator config.add_directive("set_user_validator", set_user_validator) config.set_user_validator(default_user_validator) config.add_route("dynamic", "/dynamic.json", request_method="GET") # Add routes to the mapserver proxy config.add_route_predicate("mapserverproxy", MapserverproxyRoutePredicate) config.add_route( "mapserverproxy", "/mapserv_proxy", mapserverproxy=True, pregenerator=C2CPregenerator(role=True), request_method="GET", ) config.add_route( "mapserverproxy_post", "/mapserv_proxy", mapserverproxy=True, pregenerator=C2CPregenerator(role=True), request_method="POST", ) add_cors_route(config, "/mapserv_proxy", "mapserver") # Add route to the tinyows proxy config.add_route("tinyowsproxy", "/tinyows_proxy", pregenerator=C2CPregenerator(role=True)) # Add routes to the entry view class config.add_route("base", "/", static=True) config.add_route("loginform", "/login.html", request_method="GET") add_cors_route(config, "/login", "login") config.add_route("login", "/login", request_method="POST") add_cors_route(config, "/logout", "login") config.add_route("logout", "/logout", request_method="GET") add_cors_route(config, "/loginchangepassword", "login") config.add_route("change_password", "/loginchangepassword", request_method="POST") add_cors_route(config, "/loginresetpassword", "login") config.add_route("loginresetpassword", "/loginresetpassword", request_method="POST") add_cors_route(config, "/loginuser", "login") config.add_route("loginuser", "/loginuser", request_method="GET") config.add_route("testi18n", "/testi18n.html", request_method="GET") config.add_renderer(".map", AssetRendererFactory) config.add_renderer(".css", AssetRendererFactory) config.add_renderer(".ico", AssetRendererFactory) config.add_route("localejson", "/locale.json", request_method="GET") def add_static_route(name: str, attr: str, path: str, renderer: str) -> None: config.add_route(name, path, request_method="GET") config.add_view(Entry, attr=attr, route_name=name, renderer=renderer) add_static_route("favicon", "favicon", "/favicon.ico", "/etc/geomapfish/static/images/favicon.ico") add_static_route("robot.txt", "robot_txt", "/robot.txt", "/etc/geomapfish/static/robot.txt") add_static_route("apijs", "apijs", "/api.js", "/etc/static-ngeo/api.js") add_static_route("apijsmap", "apijsmap", "/api.js.map", "/etc/static-ngeo/api.js.map") add_static_route("apicss", "apicss", "/api.css", "/etc/static-ngeo/api.css") add_static_route("apihelp", "apihelp", "/apihelp/index.html", "/etc/geomapfish/static/apihelp/index.html") c2cgeoportal_geoportal.views.add_redirect(config, "apihelp_redirect", "/apihelp.html", "apihelp") config.add_route("themes", "/themes", request_method="GET", pregenerator=C2CPregenerator(role=True)) config.add_route("invalidate", "/invalidate", request_method="GET") # Print proxy routes config.add_route("printproxy", "/printproxy", request_method="HEAD") add_cors_route(config, "/printproxy/*all", "print") config.add_route( "printproxy_capabilities", "/printproxy/capabilities.json", request_method="GET", pregenerator=C2CPregenerator(role=True), ) config.add_route( "printproxy_report_create", "/printproxy/report.{format}", request_method="POST", header=JSON_CONTENT_TYPE, ) config.add_route("printproxy_status", "/printproxy/status/{ref}.json", request_method="GET") config.add_route("printproxy_cancel", "/printproxy/cancel/{ref}", request_method="DELETE") config.add_route("printproxy_report_get", "/printproxy/report/{ref}", request_method="GET") # Full-text search routes add_cors_route(config, "/search", "fulltextsearch") config.add_route("fulltextsearch", "/search", request_method="GET") # Access to raster data add_cors_route(config, "/raster", "raster") config.add_route("raster", "/raster", request_method="GET") add_cors_route(config, "/profile.json", "profile") config.add_route("profile.json", "/profile.json", request_method="POST") # Shortener add_cors_route(config, "/short/create", "shortener") config.add_route("shortener_create", "/short/create", request_method="POST") config.add_route("shortener_get", "/s/{ref}", request_method="GET") # Geometry processing config.add_route("difference", "/difference", request_method="POST") # PDF report tool config.add_route("pdfreport", "/pdfreport/{layername}/{ids}", request_method="GET") # Add routes for the "layers" web service add_cors_route(config, "/layers/*all", "layers") config.add_route("layers_count", "/layers/{layer_id:\\d+}/count", request_method="GET") config.add_route( "layers_metadata", "/layers/{layer_id:\\d+}/md.xsd", request_method="GET", pregenerator=C2CPregenerator(role=True), ) config.add_route("layers_read_many", "/layers/{layer_id:\\d+,?(\\d+,)*\\d*$}", request_method="GET") # supports URLs like /layers/1,2,3 config.add_route("layers_read_one", "/layers/{layer_id:\\d+}/{feature_id}", request_method="GET") config.add_route("layers_create", "/layers/{layer_id:\\d+}", request_method="POST", header=GEOJSON_CONTENT_TYPE) config.add_route( "layers_update", "/layers/{layer_id:\\d+}/{feature_id}", request_method="PUT", header=GEOJSON_CONTENT_TYPE, ) config.add_route("layers_delete", "/layers/{layer_id:\\d+}/{feature_id}", request_method="DELETE") config.add_route( "layers_enumerate_attribute_values", "/layers/{layer_name}/values/{field_name}", request_method="GET", pregenerator=C2CPregenerator(), ) # There is no view corresponding to that route, it is to be used from # mako templates to get the root of the "layers" web service config.add_route("layers_root", "/layers", request_method="HEAD") # Resource proxy (load external url, useful when loading non https content) config.add_route("resourceproxy", "/resourceproxy", request_method="GET") # Dev config.add_route("dev", "/dev/*path", request_method="GET") # Used memory in caches config.add_route("memory", "/memory", request_method="GET") # Scan view decorator for adding routes config.scan(ignore=[ "c2cgeoportal_geoportal.lib", "c2cgeoportal_geoportal.scaffolds", "c2cgeoportal_geoportal.scripts", ]) add_admin_interface(config) add_getitfixed(config) # Add the project static view with cache buster config.add_static_view( name="static", path="/etc/geomapfish/static", cache_max_age=int(config.get_settings()["default_max_age"]), ) config.add_cache_buster("/etc/geomapfish/static", version_cache_buster) # Add the project static view without cache buster config.add_static_view( name="static-ngeo", path="/etc/static-ngeo", cache_max_age=int(config.get_settings()["default_max_age"]), ) # Handles the other HTTP errors raised by the views. Without that, # the client receives a status=200 without content. config.add_view(error_handler, context=HTTPException) c2cwsgiutils.index.additional_title = ( '<div class="row"><div class="col-lg-3"><h2>GeoMapFish</h2></div><div class="col-lg">' ) c2cwsgiutils.index.additional_auth.extend([ '<a href="../tiles/admin/">TileCloud chain admin</a><br>', '<a href="../tiles/c2c/">TileCloud chain c2c tools</a><br>', '<a href="../invalidate">Invalidate the cache</a><br>', '<a href="../memory">Memory status</a><br>', ]) if config.get_settings().get("enable_admin_interface", False): c2cwsgiutils.index.additional_noauth.append( '<a href="../admin/">Admin</a><br>') c2cwsgiutils.index.additional_noauth.append( '</div></div><div class="row"><div class="col-lg-3"><h3>Interfaces</h3></div><div class="col-lg">' ) c2cwsgiutils.index.additional_noauth.append( '<a href="../">Default</a><br>') for interface in config.get_settings().get("interfaces", []): if not interface.get("default", False): c2cwsgiutils.index.additional_noauth.append( '<a href="../{interface}">{interface}</a><br>'.format( interface=interface["name"])) c2cwsgiutils.index.additional_noauth.append( '<a href="../apihelp/index.html">API help</a><br>') c2cwsgiutils.index.additional_noauth.append("</div></div><hr>")