def init(
config: pyramid.config.Configurator,
master_prefix: str,
slave_prefix: Optional[str] = None,
force_master: Optional[Iterable[str]] = None,
force_slave: Optional[Iterable[str]] = None,
) -> SessionFactory:
"""
Initialize the database for a Pyramid app.
Arguments:
config: The pyramid Configuration object
master_prefix: The prefix for the master connection configuration entries in the application \
settings
slave_prefix: The prefix for the slave connection configuration entries in the application \
settings
force_master: The method/paths that needs to use the master
force_slave: The method/paths that needs to use the slave
Returns: The SQLAlchemy session
"""
settings = config.get_settings()
settings["tm.manager_hook"] = "pyramid_tm.explicit_manager"
# hook to share the dbengine fixture in testing
dbengine = settings.get("dbengine")
if not dbengine:
rw_engine = get_engine(settings, master_prefix + ".")
rw_engine.c2c_name = master_prefix
# Setup a slave DB connection and add a tween to use it.
if slave_prefix and settings[master_prefix + ".url"] != settings.get(
slave_prefix + ".url"):
LOG.info("Using a slave DB for reading %s", master_prefix)
ro_engine = get_engine(config.get_settings(), slave_prefix + ".")
ro_engine.c2c_name = slave_prefix
else:
ro_engine = rw_engine
else:
ro_engine = rw_engine = dbengine
session_factory = SessionFactory(force_master, force_slave, ro_engine,
rw_engine)
config.registry["dbsession_factory"] = session_factory
# make request.dbsession available for use in Pyramid
def dbsession(request: pyramid.request.Request) -> sqlalchemy.orm.Session:
# hook to share the dbsession fixture in testing
dbsession = request.environ.get("app.dbsession")
if dbsession is None:
# request.tm is the transaction manager used by pyramid_tm
dbsession = get_tm_session_pyramid(session_factory,
request.tm,
request=request)
return dbsession
config.add_request_method(dbsession, reify=True)
return session_factory
def env_or_config(config: Optional[pyramid.config.Configurator],
env_name: Optional[str] = None,
config_name: Optional[str] = None,
default: Any = None,
type_: Callable[[str], Any] = str) -> Any:
return env_or_settings(config.get_settings() if config is not None else {},
env_name, config_name, default, type_)
def includeme(config: Optional[pyramid.config.Configurator] = None) -> None:
"""
Initialize the broadcaster with Redis, if configured.
Otherwise, fall back to a fake local implementation.
"""
global _broadcaster
broadcast_prefix = config_utils.env_or_config(config, BROADCAST_ENV_KEY,
BROADCAST_CONFIG_KEY,
"broadcast_api_")
master, slave, _ = redis_utils.get(
config.get_settings() if config else None)
if _broadcaster is None:
if master is not None and slave is not None:
_broadcaster = redis.RedisBroadcaster(broadcast_prefix, master,
slave)
else:
_broadcaster = local.LocalBroadcaster()
LOG.info("Broadcast service setup using local implementation")
elif isinstance(_broadcaster, local.LocalBroadcaster
) and master is not None and slave is not None:
LOG.info("Switching from a local broadcaster to a redis broadcaster")
prev_broadcaster = _broadcaster
_broadcaster = redis.RedisBroadcaster(broadcast_prefix, master, slave)
_broadcaster.copy_local_subscriptions(prev_broadcaster)
def setup_session(
config: pyramid.config.Configurator,
master_prefix: str,
slave_prefix: Optional[str] = None,
force_master: Optional[Iterable[str]] = None,
force_slave: Optional[Iterable[str]] = None,
) -> Tuple[Union[sqlalchemy.orm.Session, sqlalchemy.orm.scoped_session],
sqlalchemy.engine.Engine, sqlalchemy.engine.Engine, ]:
"""
Create a SQLAlchemy session.
With an accompanying tween that switches between the master and the slave DB
connection. Uses prefixed entries in the application's settings.
The slave DB will be used for anything that is GET and OPTIONS queries. The master DB will be used for
all the other queries. You can tweak this behavior with the force_master and force_slave parameters.
Those parameters are lists of regex that are going to be matched against "{VERB} {PATH}". Warning, the
path includes the route_prefix.
Arguments:
config: The pyramid Configuration object
master_prefix: The prefix for the master connection configuration entries in the application \
settings
slave_prefix: The prefix for the slave connection configuration entries in the application \
settings
force_master: The method/paths that needs to use the master
force_slave: The method/paths that needs to use the slave
Returns: The SQLAlchemy session, the R/W engine and the R/O engine
"""
warnings.warn(
"setup_session function is deprecated; use init and request.dbsession instead"
)
if slave_prefix is None:
slave_prefix = master_prefix
settings = config.registry.settings
rw_engine = sqlalchemy.engine_from_config(settings, master_prefix + ".")
rw_engine.c2c_name = master_prefix
factory = sqlalchemy.orm.sessionmaker(bind=rw_engine)
register(factory)
db_session = sqlalchemy.orm.scoped_session(factory)
# Setup a slave DB connection and add a tween to use it.
if settings[master_prefix + ".url"] != settings.get(slave_prefix + ".url"):
LOG.info("Using a slave DB for reading %s", master_prefix)
ro_engine = sqlalchemy.engine_from_config(config.get_settings(),
slave_prefix + ".")
ro_engine.c2c_name = slave_prefix
tween_name = master_prefix.replace(".", "_")
_add_tween(config, tween_name, db_session, force_master, force_slave)
else:
ro_engine = rw_engine
db_session.c2c_rw_bind = rw_engine
db_session.c2c_ro_bind = ro_engine
return db_session, rw_engine, ro_engine
def _init_domain(config: pyramid.config.Configurator) -> None:
settings: Final = config.get_settings()
domain_factory: Final = sample.bootstrap.domain_factory(settings)
def request_method(
request: pyramid.request.Request) -> sample.domain.Domain:
return domain_factory(request.repository)
config.add_request_method(request_method, 'domain', reify=True)
def env_or_config(
config: Optional[pyramid.config.Configurator],
env_name: Optional[str] = None,
config_name: Optional[str] = None,
default: Any = None,
type_: Callable[[str], Any] = str,
) -> Any:
"""Get the setting from the environment or from the config file."""
return env_or_settings(config.get_settings() if config is not None else {},
env_name, config_name, default, type_)
def init(config: pyramid.config.Configurator,
health_check: c2cwsgiutils.health_check.HealthCheck) -> None:
"""Initialize the checkers."""
global_settings = config.get_settings()
if "checker" not in global_settings:
return
settings = global_settings["checker"]
_routes(settings, health_check)
_pdf3(settings, health_check)
_fts(settings, health_check)
_themes_errors(settings, health_check)
_lang_files(global_settings, settings, health_check)
_phantomjs(settings, health_check)
def add_getitfixed(config: pyramid.config.Configurator) -> None:
if config.get_settings()["getitfixed"].get("enabled", False):
for route_name, pattern in (
("getitfixed_add_ending_slash", "/getitfixed"),
("getitfixed_admin_add_ending_slash", "/getitfixed_admin"),
):
config.add_view(c2cgeoportal_geoportal.views.add_ending_slash,
route_name=route_name)
config.add_route(route_name, pattern, request_method="GET")
config.include("getitfixed")
# Register admin and getitfixed search paths together
Form.set_zpt_renderer(c2cgeoform.default_search_paths,
translator=translator)
def add_admin_interface(config: pyramid.config.Configurator) -> None:
if config.get_settings().get("enable_admin_interface", False):
config.add_request_method(
lambda request: c2cgeoportal_commons.models.DBSession(),
"dbsession",
reify=True,
)
config.add_view(c2cgeoportal_geoportal.views.add_ending_slash,
route_name="admin_add_ending_slash")
config.add_route("admin_add_ending_slash",
"/admin",
request_method="GET")
config.include("c2cgeoportal_admin")
def init(config: pyramid.config.Configurator) -> None:
"""
Initialize the whole stats module.
:param config: The Pyramid config
"""
stats.init_backends(config.get_settings())
if stats.BACKENDS: # pragma: nocover
if 'memory' in stats.BACKENDS: # pragma: nocover
from . import _views
_views.init(config)
from . import _pyramid_spy
_pyramid_spy.init(config)
init_db_spy()
def _init_repository(config: pyramid.config.Configurator) -> None:
settings: Final = config.get_settings()
settings['tm.manager_hook'] = 'pyramid_tm.explicit_manager'
config.include('pyramid_tm')
session_factory = sample.bootstrap.sqlalchemy_session_factory_factory()
def request_method(
request: pyramid.request.Request) -> sample.repository.Repository:
session: Final = session_factory()
zope.sqlalchemy.register(session, transaction_manager=request.tm)
return sample.repository.Repository(session)
config.add_request_method(request_method, 'repository', reify=True)
def init(config: pyramid.config.Configurator,
health_check: c2cwsgiutils.health_check.HealthCheck) -> None:
"""
Initialize the check collector.
Add him in the c2cwsgichecks.
"""
global_settings = config.get_settings()
if "check_collector" not in global_settings:
return
settings = global_settings["check_collector"]
c2c_base = global_settings.get("c2c.base_path", "")
max_level = settings["max_level"]
for host in settings["hosts"]:
class Check:
def __init__(self, host: Dict[str, Any]):
self.host = host
def __call__(
self, request: pyramid.request.Request
) -> Optional[Dict[str, Any]]:
params = request.params
display = self.host["display"]
if "host" not in params or display == params["host"]:
url_headers = build_url(
"check_collector",
f"{self.host['url'].rstrip('/')}/{c2c_base.strip('/')}/health_check",
request,
)
r = requests.get(
params={
"max_level":
str(self.host.get("max_level", max_level))
},
timeout=120,
**url_headers, # type: ignore
)
r.raise_for_status()
return cast(Dict[str, Any], r.json())
return None
health_check.add_custom_check(name="check_collector_" +
host["display"],
check_cb=Check(host),
level=settings["level"])
def includeme(config):
"""Set up standard configurator registrations. Use via:
.. code-block:: python
config = Configurator()
config.include('pyramid_stripe')
"""
config.set_request_property(request.add_stripe_event, "stripe", reify=True)
config.set_request_property(request.add_stripe_event_raw,
"stripe_raw",
reify=True)
config.include(add_routes)
config.scan("pyramid_stripe.views")
if not stripe.api_key:
stripe.api_key = config.get_settings()["stripe.api_key"]
def includeme(config):
"""Set up standard configurator registrations. Use via:
.. code-block:: python
config = Configurator()
config.include('pyramid_stripe')
"""
config.set_request_property(request.add_stripe_event, "stripe",
reify=True)
config.set_request_property(request.add_stripe_event_raw, "stripe_raw",
reify=True)
config.include(add_routes)
config.scan("pyramid_stripe.views")
if not stripe.api_key:
stripe.api_key = config.get_settings()["stripe.api_key"]
def includeme(config: pyramid.config.Configurator) -> None:
"""Initialize the index page."""
base_path = config_utils.get_base_path(config)
if base_path != "":
config.add_route("c2c_index",
base_path,
request_method=("GET", "POST"))
config.add_view(_index, route_name="c2c_index", http_cache=0)
config.add_route("c2c_index_slash",
base_path + "/",
request_method=("GET", "POST"))
config.add_view(_index, route_name="c2c_index_slash", http_cache=0)
settings = config.get_settings()
auth_type_ = auth_type(settings)
if auth_type_ == AuthenticationType.SECRET:
LOG.warning(
"It is recommended to use OAuth2 with GitHub login instead of the `C2C_SECRET` because it "
"protects from brute force attacks and the access grant is personal and can be revoked."
)
if auth_type_ == AuthenticationType.GITHUB:
config.add_route("c2c_github_login",
base_path + "/github-login",
request_method=("GET", ))
config.add_view(_github_login,
route_name="c2c_github_login",
http_cache=0)
config.add_route("c2c_github_callback",
base_path + "/github-callback",
request_method=("GET", ))
config.add_view(_github_login_callback,
route_name="c2c_github_callback",
http_cache=0,
renderer="fast_json")
config.add_route("c2c_github_logout",
base_path + "/github-logout",
request_method=("GET", ))
config.add_view(_github_logout,
route_name="c2c_github_logout",
http_cache=0)
def includeme(config: pyramid.config.Configurator) -> None:
"""
This function returns a Pyramid WSGI application.
"""
settings = config.get_settings()
config.include("c2cgeoportal_commons")
if "available_locale_names" not in settings:
settings["available_locale_names"] = available_locale_names()
call_hook(settings, "after_settings", settings)
get_user_from_request = create_get_user_from_request(settings)
config.add_request_method(get_user_from_request,
name="user",
property=True)
config.add_request_method(get_user_from_request, name="get_user")
# Configure 'locale' dir as the translation dir for c2cgeoportal app
config.add_translation_dirs("c2cgeoportal_geoportal:locale/")
config.include("c2cwsgiutils.pyramid.includeme")
health_check = HealthCheck(config)
config.registry["health_check"] = health_check
metrics_config = config.registry.settings["metrics"]
if metrics_config["memory_maps_rss"]:
add_provider(MemoryMapProvider("rss"))
if metrics_config["memory_maps_size"]:
add_provider(MemoryMapProvider("size"))
if metrics_config["memory_cache"]:
add_provider(
MemoryCacheSizeProvider(
metrics_config.get("memory_cache_all", False)))
if metrics_config["raster_data"]:
add_provider(RasterDataSizeProvider())
if metrics_config["total_python_object_memory"]:
add_provider(TotalPythonObjectMemoryProvider())
# Initialise DBSessions
init_dbsessions(settings, config, health_check)
checker.init(config, health_check)
check_collector.init(config, health_check)
# dogpile.cache configuration
if "cache" in settings:
register_backend("c2cgeoportal.hybrid",
"c2cgeoportal_geoportal.lib.caching",
"HybridRedisBackend")
register_backend("c2cgeoportal.hybridsentinel",
"c2cgeoportal_geoportal.lib.caching",
"HybridRedisSentinelBackend")
for name, cache_config in settings["cache"].items():
caching.init_region(cache_config, name)
@zope.event.classhandler.handler(InvalidateCacheEvent)
def handle(event: InvalidateCacheEvent) -> None: # pylint: disable=unused-variable
del event
caching.invalidate_region()
if caching.MEMORY_CACHE_DICT:
caching.get_region("std").delete_multi(
caching.MEMORY_CACHE_DICT.keys())
caching.MEMORY_CACHE_DICT.clear()
# Register a tween to get back the cache buster path.
if "cache_path" not in config.get_settings():
config.get_settings()["cache_path"] = ["static"]
config.add_tween(
"c2cgeoportal_geoportal.lib.cacheversion.CachebusterTween")
config.add_tween("c2cgeoportal_geoportal.lib.headers.HeadersTween")
# Bind the mako renderer to other file extensions
add_mako_renderer(config, ".html")
add_mako_renderer(config, ".js")
# Add the "geojson" renderer
config.add_renderer("geojson", GeoJSON())
# Add the "xsd" renderer
config.add_renderer("xsd", XSD(include_foreign_keys=True))
# Add the set_user_validator directive, and set a default user validator
config.add_directive("set_user_validator", set_user_validator)
config.set_user_validator(default_user_validator)
config.add_route("dynamic", "/dynamic.json", request_method="GET")
# Add routes to the mapserver proxy
config.add_route_predicate("mapserverproxy", MapserverproxyRoutePredicate)
config.add_route(
"mapserverproxy",
"/mapserv_proxy",
mapserverproxy=True,
pregenerator=C2CPregenerator(role=True),
request_method="GET",
)
config.add_route(
"mapserverproxy_post",
"/mapserv_proxy",
mapserverproxy=True,
pregenerator=C2CPregenerator(role=True),
request_method="POST",
)
add_cors_route(config, "/mapserv_proxy", "mapserver")
# Add route to the tinyows proxy
config.add_route("tinyowsproxy",
"/tinyows_proxy",
pregenerator=C2CPregenerator(role=True))
# Add routes to the entry view class
config.add_route("base", "/", static=True)
config.add_route("loginform", "/login.html", request_method="GET")
add_cors_route(config, "/login", "login")
config.add_route("login", "/login", request_method="POST")
add_cors_route(config, "/logout", "login")
config.add_route("logout", "/logout", request_method="GET")
add_cors_route(config, "/loginchangepassword", "login")
config.add_route("change_password",
"/loginchangepassword",
request_method="POST")
add_cors_route(config, "/loginresetpassword", "login")
config.add_route("loginresetpassword",
"/loginresetpassword",
request_method="POST")
add_cors_route(config, "/loginuser", "login")
config.add_route("loginuser", "/loginuser", request_method="GET")
config.add_route("testi18n", "/testi18n.html", request_method="GET")
config.add_renderer(".map", AssetRendererFactory)
config.add_renderer(".css", AssetRendererFactory)
config.add_renderer(".ico", AssetRendererFactory)
config.add_route("localejson", "/locale.json", request_method="GET")
def add_static_route(name: str, attr: str, path: str,
renderer: str) -> None:
config.add_route(name, path, request_method="GET")
config.add_view(Entry, attr=attr, route_name=name, renderer=renderer)
add_static_route("favicon", "favicon", "/favicon.ico",
"/etc/geomapfish/static/images/favicon.ico")
add_static_route("robot.txt", "robot_txt", "/robot.txt",
"/etc/geomapfish/static/robot.txt")
add_static_route("apijs", "apijs", "/api.js", "/etc/static-ngeo/api.js")
add_static_route("apijsmap", "apijsmap", "/api.js.map",
"/etc/static-ngeo/api.js.map")
add_static_route("apicss", "apicss", "/api.css",
"/etc/static-ngeo/api.css")
add_static_route("apihelp", "apihelp", "/apihelp/index.html",
"/etc/geomapfish/static/apihelp/index.html")
c2cgeoportal_geoportal.views.add_redirect(config, "apihelp_redirect",
"/apihelp.html", "apihelp")
config.add_route("themes",
"/themes",
request_method="GET",
pregenerator=C2CPregenerator(role=True))
config.add_route("invalidate", "/invalidate", request_method="GET")
# Print proxy routes
config.add_route("printproxy", "/printproxy", request_method="HEAD")
add_cors_route(config, "/printproxy/*all", "print")
config.add_route(
"printproxy_capabilities",
"/printproxy/capabilities.json",
request_method="GET",
pregenerator=C2CPregenerator(role=True),
)
config.add_route(
"printproxy_report_create",
"/printproxy/report.{format}",
request_method="POST",
header=JSON_CONTENT_TYPE,
)
config.add_route("printproxy_status",
"/printproxy/status/{ref}.json",
request_method="GET")
config.add_route("printproxy_cancel",
"/printproxy/cancel/{ref}",
request_method="DELETE")
config.add_route("printproxy_report_get",
"/printproxy/report/{ref}",
request_method="GET")
# Full-text search routes
add_cors_route(config, "/search", "fulltextsearch")
config.add_route("fulltextsearch", "/search", request_method="GET")
# Access to raster data
add_cors_route(config, "/raster", "raster")
config.add_route("raster", "/raster", request_method="GET")
add_cors_route(config, "/profile.json", "profile")
config.add_route("profile.json", "/profile.json", request_method="POST")
# Shortener
add_cors_route(config, "/short/create", "shortener")
config.add_route("shortener_create",
"/short/create",
request_method="POST")
config.add_route("shortener_get", "/s/{ref}", request_method="GET")
# Geometry processing
config.add_route("difference", "/difference", request_method="POST")
# PDF report tool
config.add_route("pdfreport",
"/pdfreport/{layername}/{ids}",
request_method="GET")
# Add routes for the "layers" web service
add_cors_route(config, "/layers/*all", "layers")
config.add_route("layers_count",
"/layers/{layer_id:\\d+}/count",
request_method="GET")
config.add_route(
"layers_metadata",
"/layers/{layer_id:\\d+}/md.xsd",
request_method="GET",
pregenerator=C2CPregenerator(role=True),
)
config.add_route("layers_read_many",
"/layers/{layer_id:\\d+,?(\\d+,)*\\d*$}",
request_method="GET") # supports URLs like /layers/1,2,3
config.add_route("layers_read_one",
"/layers/{layer_id:\\d+}/{feature_id}",
request_method="GET")
config.add_route("layers_create",
"/layers/{layer_id:\\d+}",
request_method="POST",
header=GEOJSON_CONTENT_TYPE)
config.add_route(
"layers_update",
"/layers/{layer_id:\\d+}/{feature_id}",
request_method="PUT",
header=GEOJSON_CONTENT_TYPE,
)
config.add_route("layers_delete",
"/layers/{layer_id:\\d+}/{feature_id}",
request_method="DELETE")
config.add_route(
"layers_enumerate_attribute_values",
"/layers/{layer_name}/values/{field_name}",
request_method="GET",
pregenerator=C2CPregenerator(),
)
# There is no view corresponding to that route, it is to be used from
# mako templates to get the root of the "layers" web service
config.add_route("layers_root", "/layers", request_method="HEAD")
# Resource proxy (load external url, useful when loading non https content)
config.add_route("resourceproxy", "/resourceproxy", request_method="GET")
# Dev
config.add_route("dev", "/dev/*path", request_method="GET")
# Used memory in caches
config.add_route("memory", "/memory", request_method="GET")
# Scan view decorator for adding routes
config.scan(ignore=[
"c2cgeoportal_geoportal.lib",
"c2cgeoportal_geoportal.scaffolds",
"c2cgeoportal_geoportal.scripts",
])
add_admin_interface(config)
add_getitfixed(config)
# Add the project static view with cache buster
config.add_static_view(
name="static",
path="/etc/geomapfish/static",
cache_max_age=int(config.get_settings()["default_max_age"]),
)
config.add_cache_buster("/etc/geomapfish/static", version_cache_buster)
# Add the project static view without cache buster
config.add_static_view(
name="static-ngeo",
path="/etc/static-ngeo",
cache_max_age=int(config.get_settings()["default_max_age"]),
)
# Handles the other HTTP errors raised by the views. Without that,
# the client receives a status=200 without content.
config.add_view(error_handler, context=HTTPException)
c2cwsgiutils.index.additional_title = (
'<div class="row"><div class="col-lg-3"><h2>GeoMapFish</h2></div><div class="col-lg">'
)
c2cwsgiutils.index.additional_auth.extend([
'<a href="../tiles/admin/">TileCloud chain admin</a><br>',
'<a href="../tiles/c2c/">TileCloud chain c2c tools</a><br>',
'<a href="../invalidate">Invalidate the cache</a><br>',
'<a href="../memory">Memory status</a><br>',
])
if config.get_settings().get("enable_admin_interface", False):
c2cwsgiutils.index.additional_noauth.append(
'<a href="../admin/">Admin</a><br>')
c2cwsgiutils.index.additional_noauth.append(
'</div></div><div class="row"><div class="col-lg-3"><h3>Interfaces</h3></div><div class="col-lg">'
)
c2cwsgiutils.index.additional_noauth.append(
'<a href="../">Default</a><br>')
for interface in config.get_settings().get("interfaces", []):
if not interface.get("default", False):
c2cwsgiutils.index.additional_noauth.append(
'<a href="../{interface}">{interface}</a><br>'.format(
interface=interface["name"]))
c2cwsgiutils.index.additional_noauth.append(
'<a href="../apihelp/index.html">API help</a><br>')
c2cwsgiutils.index.additional_noauth.append("</div></div><hr>")
