def test_returns_when_invalid_and_not_strict(self):
cb = ManifestCacheBuster("warehouse:static/dist/manifest.json",
strict=False)
cb._manifest = {}
result = cb(None, "/the/path/style.css", {"keyword": "arg"})
assert result == ("/the/path/style.css", {"keyword": "arg"})
def test_returns_when_invalid_and_not_strict(self):
cb = ManifestCacheBuster(
"warehouse:static/dist/manifest.json",
strict=False,
)
cb._manifest = {}
result = cb(None, "/the/path/style.css", {"keyword": "arg"})
assert result == ("/the/path/style.css", {"keyword": "arg"})
def includeme(config):
from warehouse.accounts.views import login, logout
sponsorlogos_storage_class = config.maybe_dotted(
config.registry.settings["sponsorlogos.backend"])
config.register_service_factory(sponsorlogos_storage_class.create_service,
ISponsorLogoStorage)
# Setup Jinja2 Rendering for the Admin application
config.add_jinja2_search_path("templates", name=".html")
# Setup our static assets
prevent_http_cache = config.get_settings().get(
"pyramid.prevent_http_cache", False)
config.add_static_view(
"admin/static",
"warehouse.admin:static/dist",
# Don't cache at all if prevent_http_cache is true, else we'll cache
# the files for 10 years.
cache_max_age=0 if prevent_http_cache else 10 * 365 * 24 * 60 * 60,
)
config.add_cache_buster(
"warehouse.admin:static/dist/",
ManifestCacheBuster(
"warehouse.admin:static/dist/manifest.json",
reload=config.registry.settings["pyramid.reload_assets"],
strict=not prevent_http_cache,
),
)
config.whitenoise_add_files("warehouse.admin:static/dist/",
prefix="/admin/static/")
config.whitenoise_add_manifest("warehouse.admin:static/dist/manifest.json",
prefix="/admin/static/")
# Add our routes
config.include(".routes")
# Add our flags
config.include(".flags")
config.add_view(
login,
route_name="admin.login",
renderer="admin/login.html",
uses_session=True,
require_csrf=True,
require_methods=False,
has_translations=True,
)
config.add_view(
logout,
route_name="admin.logout",
renderer="admin/logout.html",
uses_session=True,
require_csrf=True,
require_methods=False,
has_translations=True,
)
def test_returns_when_valid(self, monkeypatch):
monkeypatch.setattr(
ManifestCacheBuster,
"get_manifest",
lambda x: {"/the/path/style.css": "/the/busted/path/style.css"},
)
cb = ManifestCacheBuster("warehouse:static/dist/manifest.json")
result = cb(None, "/the/path/style.css", {"keyword": "arg"})
assert result == ("/the/busted/path/style.css", {"keyword": "arg"})
def configure(settings=None):
if settings is None:
settings = {}
# Add information about the current copy of the code.
settings.setdefault("warehouse.commit", __commit__)
# Set the environment from an environment variable, if one hasn't already
# been set.
maybe_set(
settings,
"warehouse.env",
"WAREHOUSE_ENV",
Environment,
default=Environment.production,
)
# Pull in default configuration from the environment.
maybe_set(settings, "warehouse.token", "WAREHOUSE_TOKEN")
maybe_set(settings, "warehouse.num_proxies", "WAREHOUSE_NUM_PROXIES", int)
maybe_set(settings, "warehouse.theme", "WAREHOUSE_THEME")
maybe_set(settings, "warehouse.domain", "WAREHOUSE_DOMAIN")
maybe_set(settings, "forklift.domain", "FORKLIFT_DOMAIN")
maybe_set(settings, "warehouse.legacy_domain", "WAREHOUSE_LEGACY_DOMAIN")
maybe_set(settings, "site.name", "SITE_NAME", default="Warehouse")
maybe_set(settings, "aws.key_id", "AWS_ACCESS_KEY_ID")
maybe_set(settings, "aws.secret_key", "AWS_SECRET_ACCESS_KEY")
maybe_set(settings, "aws.region", "AWS_REGION")
maybe_set(settings, "gcloud.credentials", "GCLOUD_CREDENTIALS")
maybe_set(settings, "gcloud.project", "GCLOUD_PROJECT")
maybe_set(settings, "warehouse.trending_table", "WAREHOUSE_TRENDING_TABLE")
maybe_set(settings, "celery.broker_url", "AMQP_URL")
maybe_set(settings, "celery.result_url", "REDIS_URL")
maybe_set(settings, "celery.scheduler_url", "REDIS_URL")
maybe_set(settings, "database.url", "DATABASE_URL")
maybe_set(settings, "elasticsearch.url", "ELASTICSEARCH_URL")
maybe_set(settings, "sentry.dsn", "SENTRY_DSN")
maybe_set(settings, "sentry.transport", "SENTRY_TRANSPORT")
maybe_set(settings, "sessions.url", "REDIS_URL")
maybe_set(settings, "download_stats.url", "REDIS_URL")
maybe_set(settings, "ratelimit.url", "REDIS_URL")
maybe_set(settings, "recaptcha.site_key", "RECAPTCHA_SITE_KEY")
maybe_set(settings, "recaptcha.secret_key", "RECAPTCHA_SECRET_KEY")
maybe_set(settings, "sessions.secret", "SESSION_SECRET")
maybe_set(settings, "camo.url", "CAMO_URL")
maybe_set(settings, "camo.key", "CAMO_KEY")
maybe_set(settings, "docs.url", "DOCS_URL")
maybe_set(settings, "mail.host", "MAIL_HOST")
maybe_set(settings, "mail.port", "MAIL_PORT")
maybe_set(settings, "mail.username", "MAIL_USERNAME")
maybe_set(settings, "mail.password", "MAIL_PASSWORD")
maybe_set(settings, "mail.sender", "MAIL_SENDER")
maybe_set(settings, "mail.ssl", "MAIL_SSL", default=True)
maybe_set(settings, "ga.tracking_id", "GA_TRACKING_ID")
maybe_set(settings, "statuspage.url", "STATUSPAGE_URL")
maybe_set(settings, "token.password.secret", "TOKEN_PASSWORD_SECRET")
maybe_set(settings, "token.email.secret", "TOKEN_EMAIL_SECRET")
maybe_set(
settings,
"token.password.max_age",
"TOKEN_PASSWORD_MAX_AGE",
coercer=int,
)
maybe_set(
settings,
"token.email.max_age",
"TOKEN_EMAIL_MAX_AGE",
coercer=int,
)
maybe_set(
settings,
"token.default.max_age",
"TOKEN_DEFAULT_MAX_AGE",
coercer=int,
default=21600, # 6 hours
)
maybe_set_compound(settings, "files", "backend", "FILES_BACKEND")
maybe_set_compound(settings, "origin_cache", "backend", "ORIGIN_CACHE")
# Add the settings we use when the environment is set to development.
if settings["warehouse.env"] == Environment.development:
settings.setdefault("enforce_https", False)
settings.setdefault("pyramid.reload_assets", True)
settings.setdefault("pyramid.reload_templates", True)
settings.setdefault("pyramid.prevent_http_cache", True)
settings.setdefault("debugtoolbar.hosts", ["0.0.0.0/0"])
settings.setdefault(
"debugtoolbar.panels",
[
".".join(["pyramid_debugtoolbar.panels", panel]) for panel in [
"versions.VersionDebugPanel",
"settings.SettingsDebugPanel",
"headers.HeaderDebugPanel",
"request_vars.RequestVarsDebugPanel",
"renderings.RenderingsDebugPanel",
"logger.LoggingPanel",
"performance.PerformanceDebugPanel",
"routes.RoutesDebugPanel",
"sqla.SQLADebugPanel",
"tweens.TweensDebugPanel",
"introspection.IntrospectionDebugPanel",
]
],
)
# Actually setup our Pyramid Configurator with the values pulled in from
# the environment as well as the ones passed in to the configure function.
config = Configurator(settings=settings)
config.set_root_factory(RootFactory)
# Register our CSRF support. We do this here, immediately after we've
# created the Configurator instance so that we ensure to get our defaults
# set ASAP before anything else has a chance to set them and possibly call
# Configurator().commit()
config.include(".csrf")
# Include anything needed by the development environment.
if config.registry.settings["warehouse.env"] == Environment.development:
config.include("pyramid_debugtoolbar")
# Register our logging support
config.include(".logging")
# We'll want to use Jinja2 as our template system.
config.include("pyramid_jinja2")
# Including pyramid_mailer for sending emails through SMTP.
config.include("pyramid_mailer")
# We want to use newstyle gettext
config.add_settings({"jinja2.newstyle": True})
# We also want to use Jinja2 for .html templates as well, because we just
# assume that all templates will be using Jinja.
config.add_jinja2_renderer(".html")
# Sometimes our files are .txt files and we still want to use Jinja2 to
# render them.
config.add_jinja2_renderer(".txt")
# Anytime we want to render a .xml template, we'll also use Jinja.
config.add_jinja2_renderer(".xml")
# We need to enable our Client Side Include extension
config.get_settings().setdefault(
"jinja2.extensions",
["warehouse.utils.html.ClientSideIncludeExtension"],
)
# We'll want to configure some filters for Jinja2 as well.
filters = config.get_settings().setdefault("jinja2.filters", {})
filters.setdefault(
"format_classifiers",
"warehouse.filters:format_classifiers",
)
filters.setdefault("format_tags", "warehouse.filters:format_tags")
filters.setdefault("json", "warehouse.filters:tojson")
filters.setdefault("readme", "warehouse.filters:readme")
filters.setdefault("shorten_number", "warehouse.filters:shorten_number")
filters.setdefault("urlparse", "warehouse.filters:urlparse")
filters.setdefault("contains_valid_uris",
"warehouse.filters:contains_valid_uris")
filters.setdefault("format_package_type",
"warehouse.filters:format_package_type")
filters.setdefault("parse_version", "warehouse.filters:parse_version")
# We also want to register some global functions for Jinja
jglobals = config.get_settings().setdefault("jinja2.globals", {})
jglobals.setdefault("is_valid_uri", "warehouse.utils.http:is_valid_uri")
jglobals.setdefault("gravatar", "warehouse.utils.gravatar:gravatar")
jglobals.setdefault("gravatar_profile", "warehouse.utils.gravatar:profile")
jglobals.setdefault("now", "warehouse.utils:now")
# We'll store all of our templates in one location, warehouse/templates
# so we'll go ahead and add that to the Jinja2 search path.
config.add_jinja2_search_path("warehouse:templates", name=".html")
config.add_jinja2_search_path("warehouse:templates", name=".txt")
config.add_jinja2_search_path("warehouse:templates", name=".xml")
# We want to configure our JSON renderer to sort the keys, and also to use
# an ultra compact serialization format.
config.add_renderer(
"json",
renderers.JSON(sort_keys=True, separators=(",", ":")),
)
# Configure retry support.
config.add_settings({"retry.attempts": 3})
config.include("pyramid_retry")
# Configure our transaction handling so that each request gets its own
# transaction handler and the lifetime of the transaction is tied to the
# lifetime of the request.
config.add_settings({
"tm.manager_hook":
lambda request: transaction.TransactionManager(),
"tm.activate_hook":
activate_hook,
"tm.annotate_user":
False,
})
config.include("pyramid_tm")
# Register support for services
config.include("pyramid_services")
# Register support for XMLRPC and override it's renderer to allow
# specifying custom dumps arguments.
config.include("pyramid_rpc.xmlrpc")
config.add_renderer("xmlrpc", XMLRPCRenderer(allow_none=True))
# Register support for our legacy action URLs
config.include(".legacy.action_routing")
# Register support for our domain predicates
config.include(".domain")
# Register support for template views.
config.add_directive("add_template_view", template_view, action_wrap=False)
# Register support for internationalization and localization
config.include(".i18n")
# Register the configuration for the PostgreSQL database.
config.include(".db")
# Register support for our rate limiting mechanisms
config.include(".rate_limiting")
config.include(".static")
config.include(".policy")
config.include(".search")
# Register the support for AWS and Google Cloud
config.include(".aws")
config.include(".gcloud")
# Register the support for Celery Tasks
config.include(".tasks")
# Register our session support
config.include(".sessions")
# Register our support for http and origin caching
config.include(".cache.http")
config.include(".cache.origin")
# Register our authentication support.
config.include(".accounts")
# Register logged-in views
config.include(".manage")
# Allow the packaging app to register any services it has.
config.include(".packaging")
# Configure redirection support
config.include(".redirects")
# Register all our URL routes for Warehouse.
config.include(".routes")
# Include our admin application
config.include(".admin")
# Register forklift, at least until we split it out into it's own project.
config.include(".forklift")
# Block non HTTPS requests for the legacy ?:action= routes when they are
# sent via POST.
config.add_tween("warehouse.config.require_https_tween_factory")
# Enable compression of our HTTP responses
config.add_tween(
"warehouse.utils.compression.compression_tween_factory",
over=[
"warehouse.cache.http.conditional_http_tween_factory",
"pyramid_debugtoolbar.toolbar_tween_factory",
"warehouse.raven.raven_tween_factory",
EXCVIEW,
],
)
# Enable Warehouse to serve our static files
prevent_http_cache = \
config.get_settings().get("pyramid.prevent_http_cache", False)
config.add_static_view(
"static",
"warehouse:static/dist/",
# Don't cache at all if prevent_http_cache is true, else we'll cache
# the files for 10 years.
cache_max_age=0 if prevent_http_cache else 10 * 365 * 24 * 60 * 60,
)
config.add_cache_buster(
"warehouse:static/dist/",
ManifestCacheBuster(
"warehouse:static/dist/manifest.json",
reload=config.registry.settings["pyramid.reload_assets"],
strict=not prevent_http_cache,
),
)
config.whitenoise_serve_static(
autorefresh=prevent_http_cache,
max_age=0 if prevent_http_cache else 10 * 365 * 24 * 60 * 60,
manifest="warehouse:static/dist/manifest.json",
)
config.whitenoise_add_files("warehouse:static/dist/", prefix="/static/")
# Enable Warehouse to serve our locale files
config.add_static_view("locales", "warehouse:locales/")
# Enable support of passing certain values like remote host, client
# address, and protocol support in from an outer proxy to the application.
config.add_wsgi_middleware(
ProxyFixer,
token=config.registry.settings["warehouse.token"],
num_proxies=config.registry.settings.get("warehouse.num_proxies", 1),
)
# Protect against cache poisoning via the X-Vhm-Root headers.
config.add_wsgi_middleware(VhmRootRemover)
# Fix our host header when getting sent upload.pypi.io as a HOST.
# TODO: Remove this, this is at the wrong layer.
config.add_wsgi_middleware(HostRewrite)
# We want Raven to be the last things we add here so that it's the outer
# most WSGI middleware.
config.include(".raven")
# Register Content-Security-Policy service
config.include(".csp")
# Register Referrer-Policy service
config.include(".referrer_policy")
# Register recaptcha service
config.include(".recaptcha")
config.add_settings({
"http": {
"verify": "/etc/ssl/certs/",
},
})
config.include(".http")
# Add our theme if one was configured
if config.get_settings().get("warehouse.theme"):
config.include(config.get_settings()["warehouse.theme"])
# Scan everything for configuration
config.scan(ignore=[
"warehouse.migrations.env",
"warehouse.celery",
"warehouse.wsgi",
], )
# Finally, commit all of our changes
config.commit()
return config
def test_raises_when_invalid(self):
cb = ManifestCacheBuster("warehouse:static/dist/manifest.json")
cb._manifest = {}
with pytest.raises(ValueError):
cb(None, "/the/path/style.css", {"keyword": "arg"})
def test_returns_when_valid(self):
cb = ManifestCacheBuster("warehouse:static/dist/manifest.json")
cb._manifest = {"/the/path/style.css": "/the/busted/path/style.css"}
result = cb(None, "/the/path/style.css", {"keyword": "arg"})
assert result == ("/the/busted/path/style.css", {"keyword": "arg"})
def test_returns_when_invalid_and_not_strict(self, monkeypatch):
monkeypatch.setattr(ManifestCacheBuster, "get_manifest", lambda x: {})
cb = ManifestCacheBuster("warehouse:static/dist/manifest.json", strict=False)
result = cb(None, "/the/path/style.css", {"keyword": "arg"})
assert result == ("/the/path/style.css", {"keyword": "arg"})
def test_raises_when_invalid(self, monkeypatch):
monkeypatch.setattr(ManifestCacheBuster, "get_manifest", lambda x: {})
cb = ManifestCacheBuster("warehouse:static/dist/manifest.json")
with pytest.raises(ValueError):
cb(None, "/the/path/style.css", {"keyword": "arg"})
def configure(settings=None):
if settings is None:
settings = {}
# Add information about the current copy of the code.
maybe_set(settings, "warehouse.commit", "SOURCE_COMMIT", default="null")
# Set the environment from an environment variable, if one hasn't already
# been set.
maybe_set(
settings,
"warehouse.env",
"WAREHOUSE_ENV",
Environment,
default=Environment.production,
)
# Pull in default configuration from the environment.
maybe_set(settings, "warehouse.token", "WAREHOUSE_TOKEN")
maybe_set(settings, "warehouse.num_proxies", "WAREHOUSE_NUM_PROXIES", int)
maybe_set(settings, "warehouse.domain", "WAREHOUSE_DOMAIN")
maybe_set(settings, "forklift.domain", "FORKLIFT_DOMAIN")
maybe_set(settings, "warehouse.legacy_domain", "WAREHOUSE_LEGACY_DOMAIN")
maybe_set(settings, "site.name", "SITE_NAME", default="Warehouse")
maybe_set(settings, "aws.key_id", "AWS_ACCESS_KEY_ID")
maybe_set(settings, "aws.secret_key", "AWS_SECRET_ACCESS_KEY")
maybe_set(settings, "aws.region", "AWS_REGION")
maybe_set(settings, "gcloud.credentials", "GCLOUD_CREDENTIALS")
maybe_set(settings, "gcloud.project", "GCLOUD_PROJECT")
maybe_set(settings, "warehouse.release_files_table",
"WAREHOUSE_RELEASE_FILES_TABLE")
maybe_set(settings, "github.token", "GITHUB_TOKEN")
maybe_set(
settings,
"github.token_scanning_meta_api.url",
"GITHUB_TOKEN_SCANNING_META_API_URL",
default="https://api.github.com/meta/public_keys/token_scanning",
)
maybe_set(settings, "warehouse.trending_table", "WAREHOUSE_TRENDING_TABLE")
maybe_set(settings, "celery.broker_url", "BROKER_URL")
maybe_set(settings, "celery.result_url", "REDIS_URL")
maybe_set(settings, "celery.scheduler_url", "REDIS_URL")
maybe_set(settings, "oidc.jwk_cache_url", "REDIS_URL")
maybe_set(settings, "database.url", "DATABASE_URL")
maybe_set(settings, "elasticsearch.url", "ELASTICSEARCH_URL")
maybe_set(settings, "elasticsearch.url", "ELASTICSEARCH_SIX_URL")
maybe_set(settings, "sentry.dsn", "SENTRY_DSN")
maybe_set(settings, "sentry.transport", "SENTRY_TRANSPORT")
maybe_set(settings, "sessions.url", "REDIS_URL")
maybe_set(settings, "ratelimit.url", "REDIS_URL")
maybe_set(settings, "sessions.secret", "SESSION_SECRET")
maybe_set(settings, "camo.url", "CAMO_URL")
maybe_set(settings, "camo.key", "CAMO_KEY")
maybe_set(settings, "docs.url", "DOCS_URL")
maybe_set(settings, "ga.tracking_id", "GA_TRACKING_ID")
maybe_set(settings, "statuspage.url", "STATUSPAGE_URL")
maybe_set(settings, "token.password.secret", "TOKEN_PASSWORD_SECRET")
maybe_set(settings, "token.email.secret", "TOKEN_EMAIL_SECRET")
maybe_set(settings, "token.two_factor.secret", "TOKEN_TWO_FACTOR_SECRET")
maybe_set(
settings,
"warehouse.xmlrpc.search.enabled",
"WAREHOUSE_XMLRPC_SEARCH",
coercer=distutils.util.strtobool,
default=True,
)
maybe_set(settings, "warehouse.xmlrpc.cache.url", "REDIS_URL")
maybe_set(
settings,
"warehouse.xmlrpc.client.ratelimit_string",
"XMLRPC_RATELIMIT_STRING",
default="3600 per hour",
)
maybe_set(settings,
"token.password.max_age",
"TOKEN_PASSWORD_MAX_AGE",
coercer=int)
maybe_set(settings,
"token.email.max_age",
"TOKEN_EMAIL_MAX_AGE",
coercer=int)
maybe_set(
settings,
"token.two_factor.max_age",
"TOKEN_TWO_FACTOR_MAX_AGE",
coercer=int,
default=300,
)
maybe_set(
settings,
"token.default.max_age",
"TOKEN_DEFAULT_MAX_AGE",
coercer=int,
default=21600, # 6 hours
)
maybe_set_compound(settings, "files", "backend", "FILES_BACKEND")
maybe_set_compound(settings, "simple", "backend", "SIMPLE_BACKEND")
maybe_set_compound(settings, "docs", "backend", "DOCS_BACKEND")
maybe_set_compound(settings, "sponsorlogos", "backend",
"SPONSORLOGOS_BACKEND")
maybe_set_compound(settings, "origin_cache", "backend", "ORIGIN_CACHE")
maybe_set_compound(settings, "mail", "backend", "MAIL_BACKEND")
maybe_set_compound(settings, "metrics", "backend", "METRICS_BACKEND")
maybe_set_compound(settings, "breached_passwords", "backend",
"BREACHED_PASSWORDS")
maybe_set_compound(settings, "malware_check", "backend",
"MALWARE_CHECK_BACKEND")
# Pythondotorg integration settings
maybe_set(settings,
"pythondotorg.host",
"PYTHONDOTORG_HOST",
default="python.org")
maybe_set(settings, "pythondotorg.api_token", "PYTHONDOTORG_API_TOKEN")
# Configure our ratelimiters
maybe_set(
settings,
"warehouse.account.user_login_ratelimit_string",
"USER_LOGIN_RATELIMIT_STRING",
default="10 per 5 minutes",
)
maybe_set(
settings,
"warehouse.account.ip_login_ratelimit_string",
"IP_LOGIN_RATELIMIT_STRING",
default="10 per 5 minutes",
)
maybe_set(
settings,
"warehouse.account.global_login_ratelimit_string",
"GLOBAL_LOGIN_RATELIMIT_STRING",
default="1000 per 5 minutes",
)
maybe_set(
settings,
"warehouse.account.email_add_ratelimit_string",
"EMAIL_ADD_RATELIMIT_STRING",
default="2 per day",
)
maybe_set(
settings,
"warehouse.account.password_reset_ratelimit_string",
"PASSWORD_RESET_RATELIMIT_STRING",
default="5 per day",
)
maybe_set(
settings,
"warehouse.manage.oidc.user_registration_ratelimit_string",
"USER_OIDC_REGISTRATION_RATELIMIT_STRING",
default="20 per day",
)
maybe_set(
settings,
"warehouse.manage.oidc.ip_registration_ratelimit_string",
"IP_OIDC_REGISTRATION_RATELIMIT_STRING",
default="20 per day",
)
# 2FA feature flags
maybe_set(
settings,
"warehouse.two_factor_requirement.enabled",
"TWOFACTORREQUIREMENT_ENABLED",
coercer=distutils.util.strtobool,
default=False,
)
maybe_set(
settings,
"warehouse.two_factor_mandate.available",
"TWOFACTORMANDATE_AVAILABLE",
coercer=distutils.util.strtobool,
default=False,
)
maybe_set(
settings,
"warehouse.two_factor_mandate.enabled",
"TWOFACTORMANDATE_ENABLED",
coercer=distutils.util.strtobool,
default=False,
)
# OIDC feature flags
maybe_set(
settings,
"warehouse.oidc.enabled",
"OIDC_ENABLED",
coercer=distutils.util.strtobool,
default=False,
)
# Add the settings we use when the environment is set to development.
if settings["warehouse.env"] == Environment.development:
settings.setdefault("enforce_https", False)
settings.setdefault("pyramid.reload_assets", True)
settings.setdefault("pyramid.reload_templates", True)
settings.setdefault("pyramid.prevent_http_cache", True)
settings.setdefault("debugtoolbar.hosts", ["0.0.0.0/0"])
settings.setdefault(
"debugtoolbar.panels",
[
".".join(["pyramid_debugtoolbar.panels", panel]) for panel in [
"versions.VersionDebugPanel",
"settings.SettingsDebugPanel",
"headers.HeaderDebugPanel",
"request_vars.RequestVarsDebugPanel",
"renderings.RenderingsDebugPanel",
"logger.LoggingPanel",
"performance.PerformanceDebugPanel",
"routes.RoutesDebugPanel",
"sqla.SQLADebugPanel",
"tweens.TweensDebugPanel",
"introspection.IntrospectionDebugPanel",
]
],
)
# Actually setup our Pyramid Configurator with the values pulled in from
# the environment as well as the ones passed in to the configure function.
config = Configurator(settings=settings)
config.set_root_factory(RootFactory)
# Register support for services
config.include("pyramid_services")
# Register metrics
config.include(".metrics")
# Register our CSRF support. We do this here, immediately after we've
# created the Configurator instance so that we ensure to get our defaults
# set ASAP before anything else has a chance to set them and possibly call
# Configurator().commit()
config.include(".csrf")
# Include anything needed by the development environment.
if config.registry.settings["warehouse.env"] == Environment.development:
config.include("pyramid_debugtoolbar")
# Register our logging support
config.include(".logging")
# We'll want to use Jinja2 as our template system.
config.include("pyramid_jinja2")
# Include our filters
config.include(".filters")
# Including pyramid_mailer for sending emails through SMTP.
config.include("pyramid_mailer")
# We want to use newstyle gettext
config.add_settings({"jinja2.newstyle": True})
# Our translation strings are all in the "messages" domain
config.add_settings({"jinja2.i18n.domain": "messages"})
# We also want to use Jinja2 for .html templates as well, because we just
# assume that all templates will be using Jinja.
config.add_jinja2_renderer(".html")
# Sometimes our files are .txt files and we still want to use Jinja2 to
# render them.
config.add_jinja2_renderer(".txt")
# Anytime we want to render a .xml template, we'll also use Jinja.
config.add_jinja2_renderer(".xml")
# We need to enable our Client Side Include extension
config.get_settings().setdefault(
"jinja2.extensions",
["warehouse.utils.html.ClientSideIncludeExtension"])
# We'll want to configure some filters for Jinja2 as well.
filters = config.get_settings().setdefault("jinja2.filters", {})
filters.setdefault("format_classifiers",
"warehouse.filters:format_classifiers")
filters.setdefault("classifier_id", "warehouse.filters:classifier_id")
filters.setdefault("format_tags", "warehouse.filters:format_tags")
filters.setdefault("json", "warehouse.filters:tojson")
filters.setdefault("camoify", "warehouse.filters:camoify")
filters.setdefault("shorten_number", "warehouse.filters:shorten_number")
filters.setdefault("urlparse", "warehouse.filters:urlparse")
filters.setdefault("contains_valid_uris",
"warehouse.filters:contains_valid_uris")
filters.setdefault("format_package_type",
"warehouse.filters:format_package_type")
filters.setdefault("parse_version", "warehouse.filters:parse_version")
filters.setdefault("localize_datetime",
"warehouse.filters:localize_datetime")
filters.setdefault("is_recent", "warehouse.filters:is_recent")
# We also want to register some global functions for Jinja
jglobals = config.get_settings().setdefault("jinja2.globals", {})
jglobals.setdefault("is_valid_uri", "warehouse.utils.http:is_valid_uri")
jglobals.setdefault("gravatar", "warehouse.utils.gravatar:gravatar")
jglobals.setdefault("gravatar_profile", "warehouse.utils.gravatar:profile")
jglobals.setdefault("now", "warehouse.utils:now")
# And some enums to reuse in the templates
jglobals.setdefault("AdminFlagValue",
"warehouse.admin.flags:AdminFlagValue")
jglobals.setdefault(
"OrganizationInvitationStatus",
"warehouse.organizations.models:OrganizationInvitationStatus",
)
jglobals.setdefault("OrganizationRoleType",
"warehouse.organizations.models:OrganizationRoleType")
jglobals.setdefault("OrganizationType",
"warehouse.organizations.models:OrganizationType")
jglobals.setdefault("RoleInvitationStatus",
"warehouse.packaging.models:RoleInvitationStatus")
# We'll store all of our templates in one location, warehouse/templates
# so we'll go ahead and add that to the Jinja2 search path.
config.add_jinja2_search_path("warehouse:templates", name=".html")
config.add_jinja2_search_path("warehouse:templates", name=".txt")
config.add_jinja2_search_path("warehouse:templates", name=".xml")
# We want to configure our JSON renderer to sort the keys, and also to use
# an ultra compact serialization format.
config.add_renderer("json",
renderers.JSON(sort_keys=True, separators=(",", ":")))
# Configure retry support.
config.add_settings({"retry.attempts": 3})
config.include("pyramid_retry")
# Configure our transaction handling so that each request gets its own
# transaction handler and the lifetime of the transaction is tied to the
# lifetime of the request.
config.add_settings({
"tm.manager_hook":
lambda request: transaction.TransactionManager(),
"tm.activate_hook":
activate_hook,
"tm.commit_veto":
commit_veto,
"tm.annotate_user":
False,
})
config.include("pyramid_tm")
# Register our XMLRPC service
config.include(".legacy.api.xmlrpc")
# Register our XMLRPC cache
config.include(".legacy.api.xmlrpc.cache")
# Register support for XMLRPC and override it's renderer to allow
# specifying custom dumps arguments.
config.include("pyramid_rpc.xmlrpc")
config.add_renderer("xmlrpc", XMLRPCRenderer(allow_none=True))
# Register support for our legacy action URLs
config.include(".legacy.action_routing")
# Register support for our custom predicates
config.include(".predicates")
# Register support for template views.
config.add_directive("add_template_view", template_view, action_wrap=False)
# Register support for internationalization and localization
config.include(".i18n")
# Register the configuration for the PostgreSQL database.
config.include(".db")
# Register the support for Celery Tasks
config.include(".tasks")
# Register support for our rate limiting mechanisms
config.include(".rate_limiting")
config.include(".static")
config.include(".policy")
config.include(".search")
# Register the support for AWS and Google Cloud
config.include(".aws")
config.include(".gcloud")
# Register our session support
config.include(".sessions")
# Register our support for http and origin caching
config.include(".cache.http")
config.include(".cache.origin")
# Register support for sending emails
config.include(".email")
# Register our authentication support.
config.include(".accounts")
# Register support for Macaroon based authentication
config.include(".macaroons")
# Register support for OIDC provider based authentication
config.include(".oidc")
# Register support for malware checks
config.include(".malware")
# Register logged-in views
config.include(".manage")
# Register our organization support.
config.include(".organizations")
# Allow the packaging app to register any services it has.
config.include(".packaging")
# Configure redirection support
config.include(".redirects")
# Register all our URL routes for Warehouse.
config.include(".routes")
# Allow the sponsors app to list sponsors
config.include(".sponsors")
# Allow the banners app to list banners
config.include(".banners")
# Include our admin application
config.include(".admin")
# Register forklift, at least until we split it out into it's own project.
config.include(".forklift")
# Block non HTTPS requests for the legacy ?:action= routes when they are
# sent via POST.
config.add_tween("warehouse.config.require_https_tween_factory")
# Enable compression of our HTTP responses
config.add_tween(
"warehouse.utils.compression.compression_tween_factory",
over=[
"warehouse.cache.http.conditional_http_tween_factory",
"pyramid_debugtoolbar.toolbar_tween_factory",
EXCVIEW,
],
)
# Enable Warehouse to serve our static files
prevent_http_cache = config.get_settings().get(
"pyramid.prevent_http_cache", False)
config.add_static_view(
"static",
"warehouse:static/dist/",
# Don't cache at all if prevent_http_cache is true, else we'll cache
# the files for 10 years.
cache_max_age=0 if prevent_http_cache else 10 * 365 * 24 * 60 * 60,
)
config.add_cache_buster(
"warehouse:static/dist/",
ManifestCacheBuster(
"warehouse:static/dist/manifest.json",
reload=config.registry.settings["pyramid.reload_assets"],
strict=not prevent_http_cache,
),
)
config.whitenoise_serve_static(
autorefresh=prevent_http_cache,
max_age=0 if prevent_http_cache else 10 * 365 * 24 * 60 * 60,
)
config.whitenoise_add_files("warehouse:static/dist/", prefix="/static/")
config.whitenoise_add_manifest("warehouse:static/dist/manifest.json",
prefix="/static/")
# Enable support of passing certain values like remote host, client
# address, and protocol support in from an outer proxy to the application.
config.add_wsgi_middleware(
ProxyFixer,
token=config.registry.settings["warehouse.token"],
num_proxies=config.registry.settings.get("warehouse.num_proxies", 1),
)
# Protect against cache poisoning via the X-Vhm-Root headers.
config.add_wsgi_middleware(VhmRootRemover)
# We want Sentry to be the last things we add here so that it's the outer
# most WSGI middleware.
config.include(".sentry")
# Register Content-Security-Policy service
config.include(".csp")
# Register Referrer-Policy service
config.include(".referrer_policy")
config.add_settings({"http": {"verify": "/etc/ssl/certs/"}})
config.include(".http")
# Scan everything for configuration
config.scan(
categories=(
"pyramid",
"warehouse",
),
ignore=[
"warehouse.migrations.env", "warehouse.celery", "warehouse.wsgi"
],
)
# Sanity check our request and responses.
# Note: It is very important that this go last. We need everything else that might
# have added a tween to be registered prior to this.
config.include(".sanity")
# Finally, commit all of our changes
config.commit()
return config
def configure(settings=None):
if settings is None:
settings = {}
# Add information about the current copy of the code.
settings.setdefault("warehouse.commit", __commit__)
# Set the environment from an environment variable, if one hasn't already
# been set.
maybe_set(
settings,
"warehouse.env",
"WAREHOUSE_ENV",
Environment,
default=Environment.production,
)
# Pull in default configuration from the environment.
maybe_set(settings, "warehouse.token", "WAREHOUSE_TOKEN")
maybe_set(settings, "warehouse.theme", "WAREHOUSE_THEME")
maybe_set(settings, "site.name", "SITE_NAME", default="Warehouse")
maybe_set(settings, "aws.key_id", "AWS_ACCESS_KEY_ID")
maybe_set(settings, "aws.secret_key", "AWS_SECRET_ACCESS_KEY")
maybe_set(settings, "aws.region", "AWS_REGION")
maybe_set(settings, "celery.broker_url", "AMQP_URL")
maybe_set(settings, "celery.result_url", "REDIS_URL")
maybe_set(settings, "csp.report_uri", "CSP_REPORT_URI")
maybe_set(settings, "database.url", "DATABASE_URL")
maybe_set(settings, "elasticsearch.url", "ELASTICSEARCH_URL")
maybe_set(settings, "sentry.dsn", "SENTRY_DSN")
maybe_set(settings, "sentry.transport", "SENTRY_TRANSPORT")
maybe_set(settings, "sessions.url", "REDIS_URL")
maybe_set(settings, "download_stats.url", "REDIS_URL")
maybe_set(settings, "sessions.secret", "SESSION_SECRET")
maybe_set(settings, "camo.url", "CAMO_URL")
maybe_set(settings, "camo.key", "CAMO_KEY")
maybe_set(settings, "docs.url", "DOCS_URL")
maybe_set_compound(settings, "files", "backend", "FILES_BACKEND")
maybe_set_compound(settings, "origin_cache", "backend", "ORIGIN_CACHE")
# Add the settings we use when the environment is set to development.
if settings["warehouse.env"] == Environment.development:
settings.setdefault("enforce_https", False)
settings.setdefault("pyramid.reload_assets", True)
settings.setdefault("pyramid.reload_templates", True)
settings.setdefault("pyramid.prevent_http_cache", True)
settings.setdefault("debugtoolbar.hosts", ["0.0.0.0/0"])
settings.setdefault(
"debugtoolbar.panels",
[
".".join(["pyramid_debugtoolbar.panels", panel]) for panel in [
"versions.VersionDebugPanel",
"settings.SettingsDebugPanel",
"headers.HeaderDebugPanel",
"request_vars.RequestVarsDebugPanel",
"renderings.RenderingsDebugPanel",
"logger.LoggingPanel",
"performance.PerformanceDebugPanel",
"routes.RoutesDebugPanel",
"sqla.SQLADebugPanel",
"tweens.TweensDebugPanel",
"introspection.IntrospectionDebugPanel",
]
],
)
# Actually setup our Pyramid Configurator with the values pulled in from
# the environment as well as the ones passed in to the configure function.
config = Configurator(settings=settings)
# Include anything needed by the development environment.
if config.registry.settings["warehouse.env"] == Environment.development:
config.include("pyramid_debugtoolbar")
# Register our logging support
config.include(".logging")
# We'll want to use Jinja2 as our template system.
config.include("pyramid_jinja2")
# We want to use newstyle gettext
config.add_settings({"jinja2.newstyle": True})
# We also want to use Jinja2 for .html templates as well, because we just
# assume that all templates will be using Jinja.
config.add_jinja2_renderer(".html")
# Sometimes our files are .txt files and we still want to use Jinja2 to
# render them.
config.add_jinja2_renderer(".txt")
# Anytime we want to render a .xml template, we'll also use Jinja.
config.add_jinja2_renderer(".xml")
# We'll want to configure some filters for Jinja2 as well.
filters = config.get_settings().setdefault("jinja2.filters", {})
filters.setdefault("format_tags", "warehouse.filters:format_tags")
filters.setdefault("json", "warehouse.filters:tojson")
filters.setdefault("readme", "warehouse.filters:readme")
filters.setdefault("shorten_number", "warehouse.filters:shorten_number")
filters.setdefault("urlparse", "warehouse.filters:urlparse")
# We also want to register some global functions for Jinja
jglobals = config.get_settings().setdefault("jinja2.globals", {})
jglobals.setdefault("gravatar", "warehouse.utils.gravatar:gravatar")
jglobals.setdefault("html_include", "warehouse.utils.html:html_include")
jglobals.setdefault("now", "warehouse.utils:now")
# We'll store all of our templates in one location, warehouse/templates
# so we'll go ahead and add that to the Jinja2 search path.
config.add_jinja2_search_path("warehouse:templates", name=".html")
config.add_jinja2_search_path("warehouse:templates", name=".txt")
config.add_jinja2_search_path("warehouse:templates", name=".xml")
# We want to configure our JSON renderer to sort the keys, and also to use
# an ultra compact serialization format.
config.add_renderer(
"json",
renderers.JSON(sort_keys=True, separators=(",", ":")),
)
# Configure our transaction handling so that each request gets its own
# transaction handler and the lifetime of the transaction is tied to the
# lifetime of the request.
config.add_settings({
"tm.attempts":
3,
"tm.manager_hook":
lambda request: transaction.TransactionManager(),
"tm.activate_hook":
activate_hook,
"tm.annotate_user":
False,
})
config.include("pyramid_tm")
# Register support for services
config.include("pyramid_services")
# Register our find_service_factory methods
config.add_request_method(find_service_factory)
config.add_directive("find_service_factory", find_service_factory)
# Register support for XMLRPC and override it's renderer to allow
# specifying custom dumps arguments.
config.include("pyramid_rpc.xmlrpc")
config.add_renderer("xmlrpc", XMLRPCRenderer(allow_none=True))
# Register support for our legacy action URLs
config.include(".legacy.action_routing")
# Register support for internationalization and localization
config.include(".i18n")
# Register the configuration for the PostgreSQL database.
config.include(".db")
config.include(".search")
# Register the support for AWS
config.include(".aws")
# Register the support for Celery
config.include(".celery")
# Register our session support
config.include(".sessions")
# Register our support for http and origin caching
config.include(".cache.http")
config.include(".cache.origin")
# Register our CSRF support
config.include(".csrf")
# Register our authentication support.
config.include(".accounts")
# Allow the packaging app to register any services it has.
config.include(".packaging")
# Configure redirection support
config.include(".redirects")
# Register all our URL routes for Warehouse.
config.include(".routes")
# Enable a Content Security Policy
config.add_settings({
"csp": {
"connect-src": ["'self'"],
"default-src": ["'none'"],
"font-src": ["'self'", "fonts.gstatic.com"],
"frame-ancestors": ["'none'"],
"img-src": [
"'self'",
config.registry.settings["camo.url"],
"https://secure.gravatar.com",
],
"referrer": ["origin-when-cross-origin"],
"reflected-xss": ["block"],
"report-uri": [config.registry.settings.get("csp.report_uri")],
"script-src": ["'self'"],
"style-src": ["'self'", "fonts.googleapis.com"],
},
})
config.add_tween("warehouse.config.content_security_policy_tween_factory")
# Block non HTTPS requests for the legacy ?:action= routes when they are
# sent via POST.
config.add_tween("warehouse.config.require_https_tween_factory")
# Enable compression of our HTTP responses
config.add_tween(
"warehouse.utils.compression.compression_tween_factory",
over=[
"warehouse.cache.http.conditional_http_tween_factory",
"pyramid_debugtoolbar.toolbar_tween_factory",
"warehouse.raven.raven_tween_factory",
EXCVIEW,
],
)
# Enable Warehouse to serve our static files
prevent_http_cache = \
config.get_settings().get("pyramid.prevent_http_cache", False)
config.add_static_view(
"static",
"warehouse:static/dist/",
# Don't cache at all if prevent_http_cache is true, else we'll cache
# the files for 10 years.
cache_max_age=0 if prevent_http_cache else 10 * 365 * 24 * 60 * 60,
)
config.add_cache_buster(
"warehouse:static/dist/",
ManifestCacheBuster(
"warehouse:static/dist/manifest.json",
reload=config.registry.settings["pyramid.reload_assets"],
strict=not prevent_http_cache,
),
)
# Enable Warehouse to serve our locale files
config.add_static_view("locales", "warehouse:locales/")
# Enable support of passing certain values like remote host, client
# address, and protocol support in from an outer proxy to the application.
config.add_wsgi_middleware(
ProxyFixer,
token=config.registry.settings["warehouse.token"],
)
# Protect against cache poisoning via the X-Vhm-Root headers.
config.add_wsgi_middleware(VhmRootRemover)
# We want Raven to be the last things we add here so that it's the outer
# most WSGI middleware.
config.include(".raven")
# Add our theme if one was configured
if config.get_settings().get("warehouse.theme"):
config.include(config.get_settings()["warehouse.theme"])
# Scan everything for configuration
config.scan(ignore=["warehouse.migrations.env", "warehouse.wsgi"])
return config
