示例#1
0
    def test_returns_when_invalid_and_not_strict(self):
        cb = ManifestCacheBuster("warehouse:static/dist/manifest.json",
                                 strict=False)
        cb._manifest = {}
        result = cb(None, "/the/path/style.css", {"keyword": "arg"})

        assert result == ("/the/path/style.css", {"keyword": "arg"})
示例#2
0
    def test_returns_when_invalid_and_not_strict(self):
        cb = ManifestCacheBuster(
            "warehouse:static/dist/manifest.json",
            strict=False,
        )
        cb._manifest = {}
        result = cb(None, "/the/path/style.css", {"keyword": "arg"})

        assert result == ("/the/path/style.css", {"keyword": "arg"})
示例#3
0
def includeme(config):
    from warehouse.accounts.views import login, logout

    sponsorlogos_storage_class = config.maybe_dotted(
        config.registry.settings["sponsorlogos.backend"])
    config.register_service_factory(sponsorlogos_storage_class.create_service,
                                    ISponsorLogoStorage)

    # Setup Jinja2 Rendering for the Admin application
    config.add_jinja2_search_path("templates", name=".html")

    # Setup our static assets
    prevent_http_cache = config.get_settings().get(
        "pyramid.prevent_http_cache", False)
    config.add_static_view(
        "admin/static",
        "warehouse.admin:static/dist",
        # Don't cache at all if prevent_http_cache is true, else we'll cache
        # the files for 10 years.
        cache_max_age=0 if prevent_http_cache else 10 * 365 * 24 * 60 * 60,
    )
    config.add_cache_buster(
        "warehouse.admin:static/dist/",
        ManifestCacheBuster(
            "warehouse.admin:static/dist/manifest.json",
            reload=config.registry.settings["pyramid.reload_assets"],
            strict=not prevent_http_cache,
        ),
    )
    config.whitenoise_add_files("warehouse.admin:static/dist/",
                                prefix="/admin/static/")
    config.whitenoise_add_manifest("warehouse.admin:static/dist/manifest.json",
                                   prefix="/admin/static/")

    # Add our routes
    config.include(".routes")

    # Add our flags
    config.include(".flags")

    config.add_view(
        login,
        route_name="admin.login",
        renderer="admin/login.html",
        uses_session=True,
        require_csrf=True,
        require_methods=False,
        has_translations=True,
    )
    config.add_view(
        logout,
        route_name="admin.logout",
        renderer="admin/logout.html",
        uses_session=True,
        require_csrf=True,
        require_methods=False,
        has_translations=True,
    )
示例#4
0
    def test_returns_when_valid(self, monkeypatch):
        monkeypatch.setattr(
            ManifestCacheBuster,
            "get_manifest",
            lambda x: {"/the/path/style.css": "/the/busted/path/style.css"},
        )
        cb = ManifestCacheBuster("warehouse:static/dist/manifest.json")
        result = cb(None, "/the/path/style.css", {"keyword": "arg"})

        assert result == ("/the/busted/path/style.css", {"keyword": "arg"})
示例#5
0
def configure(settings=None):
    if settings is None:
        settings = {}

    # Add information about the current copy of the code.
    settings.setdefault("warehouse.commit", __commit__)

    # Set the environment from an environment variable, if one hasn't already
    # been set.
    maybe_set(
        settings,
        "warehouse.env",
        "WAREHOUSE_ENV",
        Environment,
        default=Environment.production,
    )

    # Pull in default configuration from the environment.
    maybe_set(settings, "warehouse.token", "WAREHOUSE_TOKEN")
    maybe_set(settings, "warehouse.num_proxies", "WAREHOUSE_NUM_PROXIES", int)
    maybe_set(settings, "warehouse.theme", "WAREHOUSE_THEME")
    maybe_set(settings, "warehouse.domain", "WAREHOUSE_DOMAIN")
    maybe_set(settings, "forklift.domain", "FORKLIFT_DOMAIN")
    maybe_set(settings, "warehouse.legacy_domain", "WAREHOUSE_LEGACY_DOMAIN")
    maybe_set(settings, "site.name", "SITE_NAME", default="Warehouse")
    maybe_set(settings, "aws.key_id", "AWS_ACCESS_KEY_ID")
    maybe_set(settings, "aws.secret_key", "AWS_SECRET_ACCESS_KEY")
    maybe_set(settings, "aws.region", "AWS_REGION")
    maybe_set(settings, "gcloud.credentials", "GCLOUD_CREDENTIALS")
    maybe_set(settings, "gcloud.project", "GCLOUD_PROJECT")
    maybe_set(settings, "warehouse.trending_table", "WAREHOUSE_TRENDING_TABLE")
    maybe_set(settings, "celery.broker_url", "AMQP_URL")
    maybe_set(settings, "celery.result_url", "REDIS_URL")
    maybe_set(settings, "celery.scheduler_url", "REDIS_URL")
    maybe_set(settings, "database.url", "DATABASE_URL")
    maybe_set(settings, "elasticsearch.url", "ELASTICSEARCH_URL")
    maybe_set(settings, "sentry.dsn", "SENTRY_DSN")
    maybe_set(settings, "sentry.transport", "SENTRY_TRANSPORT")
    maybe_set(settings, "sessions.url", "REDIS_URL")
    maybe_set(settings, "download_stats.url", "REDIS_URL")
    maybe_set(settings, "ratelimit.url", "REDIS_URL")
    maybe_set(settings, "recaptcha.site_key", "RECAPTCHA_SITE_KEY")
    maybe_set(settings, "recaptcha.secret_key", "RECAPTCHA_SECRET_KEY")
    maybe_set(settings, "sessions.secret", "SESSION_SECRET")
    maybe_set(settings, "camo.url", "CAMO_URL")
    maybe_set(settings, "camo.key", "CAMO_KEY")
    maybe_set(settings, "docs.url", "DOCS_URL")
    maybe_set(settings, "mail.host", "MAIL_HOST")
    maybe_set(settings, "mail.port", "MAIL_PORT")
    maybe_set(settings, "mail.username", "MAIL_USERNAME")
    maybe_set(settings, "mail.password", "MAIL_PASSWORD")
    maybe_set(settings, "mail.sender", "MAIL_SENDER")
    maybe_set(settings, "mail.ssl", "MAIL_SSL", default=True)
    maybe_set(settings, "ga.tracking_id", "GA_TRACKING_ID")
    maybe_set(settings, "statuspage.url", "STATUSPAGE_URL")
    maybe_set(settings, "token.password.secret", "TOKEN_PASSWORD_SECRET")
    maybe_set(settings, "token.email.secret", "TOKEN_EMAIL_SECRET")
    maybe_set(
        settings,
        "token.password.max_age",
        "TOKEN_PASSWORD_MAX_AGE",
        coercer=int,
    )
    maybe_set(
        settings,
        "token.email.max_age",
        "TOKEN_EMAIL_MAX_AGE",
        coercer=int,
    )
    maybe_set(
        settings,
        "token.default.max_age",
        "TOKEN_DEFAULT_MAX_AGE",
        coercer=int,
        default=21600,  # 6 hours
    )
    maybe_set_compound(settings, "files", "backend", "FILES_BACKEND")
    maybe_set_compound(settings, "origin_cache", "backend", "ORIGIN_CACHE")

    # Add the settings we use when the environment is set to development.
    if settings["warehouse.env"] == Environment.development:
        settings.setdefault("enforce_https", False)
        settings.setdefault("pyramid.reload_assets", True)
        settings.setdefault("pyramid.reload_templates", True)
        settings.setdefault("pyramid.prevent_http_cache", True)
        settings.setdefault("debugtoolbar.hosts", ["0.0.0.0/0"])
        settings.setdefault(
            "debugtoolbar.panels",
            [
                ".".join(["pyramid_debugtoolbar.panels", panel]) for panel in [
                    "versions.VersionDebugPanel",
                    "settings.SettingsDebugPanel",
                    "headers.HeaderDebugPanel",
                    "request_vars.RequestVarsDebugPanel",
                    "renderings.RenderingsDebugPanel",
                    "logger.LoggingPanel",
                    "performance.PerformanceDebugPanel",
                    "routes.RoutesDebugPanel",
                    "sqla.SQLADebugPanel",
                    "tweens.TweensDebugPanel",
                    "introspection.IntrospectionDebugPanel",
                ]
            ],
        )

    # Actually setup our Pyramid Configurator with the values pulled in from
    # the environment as well as the ones passed in to the configure function.
    config = Configurator(settings=settings)
    config.set_root_factory(RootFactory)

    # Register our CSRF support. We do this here, immediately after we've
    # created the Configurator instance so that we ensure to get our defaults
    # set ASAP before anything else has a chance to set them and possibly call
    # Configurator().commit()
    config.include(".csrf")

    # Include anything needed by the development environment.
    if config.registry.settings["warehouse.env"] == Environment.development:
        config.include("pyramid_debugtoolbar")

    # Register our logging support
    config.include(".logging")

    # We'll want to use Jinja2 as our template system.
    config.include("pyramid_jinja2")

    # Including pyramid_mailer for sending emails through SMTP.
    config.include("pyramid_mailer")

    # We want to use newstyle gettext
    config.add_settings({"jinja2.newstyle": True})

    # We also want to use Jinja2 for .html templates as well, because we just
    # assume that all templates will be using Jinja.
    config.add_jinja2_renderer(".html")

    # Sometimes our files are .txt files and we still want to use Jinja2 to
    # render them.
    config.add_jinja2_renderer(".txt")

    # Anytime we want to render a .xml template, we'll also use Jinja.
    config.add_jinja2_renderer(".xml")

    # We need to enable our Client Side Include extension
    config.get_settings().setdefault(
        "jinja2.extensions",
        ["warehouse.utils.html.ClientSideIncludeExtension"],
    )

    # We'll want to configure some filters for Jinja2 as well.
    filters = config.get_settings().setdefault("jinja2.filters", {})
    filters.setdefault(
        "format_classifiers",
        "warehouse.filters:format_classifiers",
    )
    filters.setdefault("format_tags", "warehouse.filters:format_tags")
    filters.setdefault("json", "warehouse.filters:tojson")
    filters.setdefault("readme", "warehouse.filters:readme")
    filters.setdefault("shorten_number", "warehouse.filters:shorten_number")
    filters.setdefault("urlparse", "warehouse.filters:urlparse")
    filters.setdefault("contains_valid_uris",
                       "warehouse.filters:contains_valid_uris")
    filters.setdefault("format_package_type",
                       "warehouse.filters:format_package_type")
    filters.setdefault("parse_version", "warehouse.filters:parse_version")

    # We also want to register some global functions for Jinja
    jglobals = config.get_settings().setdefault("jinja2.globals", {})
    jglobals.setdefault("is_valid_uri", "warehouse.utils.http:is_valid_uri")
    jglobals.setdefault("gravatar", "warehouse.utils.gravatar:gravatar")
    jglobals.setdefault("gravatar_profile", "warehouse.utils.gravatar:profile")
    jglobals.setdefault("now", "warehouse.utils:now")

    # We'll store all of our templates in one location, warehouse/templates
    # so we'll go ahead and add that to the Jinja2 search path.
    config.add_jinja2_search_path("warehouse:templates", name=".html")
    config.add_jinja2_search_path("warehouse:templates", name=".txt")
    config.add_jinja2_search_path("warehouse:templates", name=".xml")

    # We want to configure our JSON renderer to sort the keys, and also to use
    # an ultra compact serialization format.
    config.add_renderer(
        "json",
        renderers.JSON(sort_keys=True, separators=(",", ":")),
    )

    # Configure retry support.
    config.add_settings({"retry.attempts": 3})
    config.include("pyramid_retry")

    # Configure our transaction handling so that each request gets its own
    # transaction handler and the lifetime of the transaction is tied to the
    # lifetime of the request.
    config.add_settings({
        "tm.manager_hook":
        lambda request: transaction.TransactionManager(),
        "tm.activate_hook":
        activate_hook,
        "tm.annotate_user":
        False,
    })
    config.include("pyramid_tm")

    # Register support for services
    config.include("pyramid_services")

    # Register support for XMLRPC and override it's renderer to allow
    # specifying custom dumps arguments.
    config.include("pyramid_rpc.xmlrpc")
    config.add_renderer("xmlrpc", XMLRPCRenderer(allow_none=True))

    # Register support for our legacy action URLs
    config.include(".legacy.action_routing")

    # Register support for our domain predicates
    config.include(".domain")

    # Register support for template views.
    config.add_directive("add_template_view", template_view, action_wrap=False)

    # Register support for internationalization and localization
    config.include(".i18n")

    # Register the configuration for the PostgreSQL database.
    config.include(".db")

    # Register support for our rate limiting mechanisms
    config.include(".rate_limiting")

    config.include(".static")

    config.include(".policy")

    config.include(".search")

    # Register the support for AWS and Google Cloud
    config.include(".aws")
    config.include(".gcloud")

    # Register the support for Celery Tasks
    config.include(".tasks")

    # Register our session support
    config.include(".sessions")

    # Register our support for http and origin caching
    config.include(".cache.http")
    config.include(".cache.origin")

    # Register our authentication support.
    config.include(".accounts")

    # Register logged-in views
    config.include(".manage")

    # Allow the packaging app to register any services it has.
    config.include(".packaging")

    # Configure redirection support
    config.include(".redirects")

    # Register all our URL routes for Warehouse.
    config.include(".routes")

    # Include our admin application
    config.include(".admin")

    # Register forklift, at least until we split it out into it's own project.
    config.include(".forklift")

    # Block non HTTPS requests for the legacy ?:action= routes when they are
    # sent via POST.
    config.add_tween("warehouse.config.require_https_tween_factory")

    # Enable compression of our HTTP responses
    config.add_tween(
        "warehouse.utils.compression.compression_tween_factory",
        over=[
            "warehouse.cache.http.conditional_http_tween_factory",
            "pyramid_debugtoolbar.toolbar_tween_factory",
            "warehouse.raven.raven_tween_factory",
            EXCVIEW,
        ],
    )

    # Enable Warehouse to serve our static files
    prevent_http_cache = \
        config.get_settings().get("pyramid.prevent_http_cache", False)
    config.add_static_view(
        "static",
        "warehouse:static/dist/",
        # Don't cache at all if prevent_http_cache is true, else we'll cache
        # the files for 10 years.
        cache_max_age=0 if prevent_http_cache else 10 * 365 * 24 * 60 * 60,
    )
    config.add_cache_buster(
        "warehouse:static/dist/",
        ManifestCacheBuster(
            "warehouse:static/dist/manifest.json",
            reload=config.registry.settings["pyramid.reload_assets"],
            strict=not prevent_http_cache,
        ),
    )
    config.whitenoise_serve_static(
        autorefresh=prevent_http_cache,
        max_age=0 if prevent_http_cache else 10 * 365 * 24 * 60 * 60,
        manifest="warehouse:static/dist/manifest.json",
    )
    config.whitenoise_add_files("warehouse:static/dist/", prefix="/static/")

    # Enable Warehouse to serve our locale files
    config.add_static_view("locales", "warehouse:locales/")

    # Enable support of passing certain values like remote host, client
    # address, and protocol support in from an outer proxy to the application.
    config.add_wsgi_middleware(
        ProxyFixer,
        token=config.registry.settings["warehouse.token"],
        num_proxies=config.registry.settings.get("warehouse.num_proxies", 1),
    )

    # Protect against cache poisoning via the X-Vhm-Root headers.
    config.add_wsgi_middleware(VhmRootRemover)

    # Fix our host header when getting sent upload.pypi.io as a HOST.
    # TODO: Remove this, this is at the wrong layer.
    config.add_wsgi_middleware(HostRewrite)

    # We want Raven to be the last things we add here so that it's the outer
    # most WSGI middleware.
    config.include(".raven")

    # Register Content-Security-Policy service
    config.include(".csp")

    # Register Referrer-Policy service
    config.include(".referrer_policy")

    # Register recaptcha service
    config.include(".recaptcha")

    config.add_settings({
        "http": {
            "verify": "/etc/ssl/certs/",
        },
    })
    config.include(".http")

    # Add our theme if one was configured
    if config.get_settings().get("warehouse.theme"):
        config.include(config.get_settings()["warehouse.theme"])

    # Scan everything for configuration
    config.scan(ignore=[
        "warehouse.migrations.env",
        "warehouse.celery",
        "warehouse.wsgi",
    ], )

    # Finally, commit all of our changes
    config.commit()

    return config
示例#6
0
    def test_raises_when_invalid(self):
        cb = ManifestCacheBuster("warehouse:static/dist/manifest.json")
        cb._manifest = {}

        with pytest.raises(ValueError):
            cb(None, "/the/path/style.css", {"keyword": "arg"})
示例#7
0
    def test_returns_when_valid(self):
        cb = ManifestCacheBuster("warehouse:static/dist/manifest.json")
        cb._manifest = {"/the/path/style.css": "/the/busted/path/style.css"}
        result = cb(None, "/the/path/style.css", {"keyword": "arg"})

        assert result == ("/the/busted/path/style.css", {"keyword": "arg"})
示例#8
0
    def test_returns_when_invalid_and_not_strict(self, monkeypatch):
        monkeypatch.setattr(ManifestCacheBuster, "get_manifest", lambda x: {})
        cb = ManifestCacheBuster("warehouse:static/dist/manifest.json", strict=False)
        result = cb(None, "/the/path/style.css", {"keyword": "arg"})

        assert result == ("/the/path/style.css", {"keyword": "arg"})
示例#9
0
    def test_raises_when_invalid(self, monkeypatch):
        monkeypatch.setattr(ManifestCacheBuster, "get_manifest", lambda x: {})
        cb = ManifestCacheBuster("warehouse:static/dist/manifest.json")

        with pytest.raises(ValueError):
            cb(None, "/the/path/style.css", {"keyword": "arg"})
示例#10
0
def configure(settings=None):
    if settings is None:
        settings = {}

    # Add information about the current copy of the code.
    maybe_set(settings, "warehouse.commit", "SOURCE_COMMIT", default="null")

    # Set the environment from an environment variable, if one hasn't already
    # been set.
    maybe_set(
        settings,
        "warehouse.env",
        "WAREHOUSE_ENV",
        Environment,
        default=Environment.production,
    )

    # Pull in default configuration from the environment.
    maybe_set(settings, "warehouse.token", "WAREHOUSE_TOKEN")
    maybe_set(settings, "warehouse.num_proxies", "WAREHOUSE_NUM_PROXIES", int)
    maybe_set(settings, "warehouse.domain", "WAREHOUSE_DOMAIN")
    maybe_set(settings, "forklift.domain", "FORKLIFT_DOMAIN")
    maybe_set(settings, "warehouse.legacy_domain", "WAREHOUSE_LEGACY_DOMAIN")
    maybe_set(settings, "site.name", "SITE_NAME", default="Warehouse")
    maybe_set(settings, "aws.key_id", "AWS_ACCESS_KEY_ID")
    maybe_set(settings, "aws.secret_key", "AWS_SECRET_ACCESS_KEY")
    maybe_set(settings, "aws.region", "AWS_REGION")
    maybe_set(settings, "gcloud.credentials", "GCLOUD_CREDENTIALS")
    maybe_set(settings, "gcloud.project", "GCLOUD_PROJECT")
    maybe_set(settings, "warehouse.release_files_table",
              "WAREHOUSE_RELEASE_FILES_TABLE")
    maybe_set(settings, "github.token", "GITHUB_TOKEN")
    maybe_set(
        settings,
        "github.token_scanning_meta_api.url",
        "GITHUB_TOKEN_SCANNING_META_API_URL",
        default="https://api.github.com/meta/public_keys/token_scanning",
    )
    maybe_set(settings, "warehouse.trending_table", "WAREHOUSE_TRENDING_TABLE")
    maybe_set(settings, "celery.broker_url", "BROKER_URL")
    maybe_set(settings, "celery.result_url", "REDIS_URL")
    maybe_set(settings, "celery.scheduler_url", "REDIS_URL")
    maybe_set(settings, "oidc.jwk_cache_url", "REDIS_URL")
    maybe_set(settings, "database.url", "DATABASE_URL")
    maybe_set(settings, "elasticsearch.url", "ELASTICSEARCH_URL")
    maybe_set(settings, "elasticsearch.url", "ELASTICSEARCH_SIX_URL")
    maybe_set(settings, "sentry.dsn", "SENTRY_DSN")
    maybe_set(settings, "sentry.transport", "SENTRY_TRANSPORT")
    maybe_set(settings, "sessions.url", "REDIS_URL")
    maybe_set(settings, "ratelimit.url", "REDIS_URL")
    maybe_set(settings, "sessions.secret", "SESSION_SECRET")
    maybe_set(settings, "camo.url", "CAMO_URL")
    maybe_set(settings, "camo.key", "CAMO_KEY")
    maybe_set(settings, "docs.url", "DOCS_URL")
    maybe_set(settings, "ga.tracking_id", "GA_TRACKING_ID")
    maybe_set(settings, "statuspage.url", "STATUSPAGE_URL")
    maybe_set(settings, "token.password.secret", "TOKEN_PASSWORD_SECRET")
    maybe_set(settings, "token.email.secret", "TOKEN_EMAIL_SECRET")
    maybe_set(settings, "token.two_factor.secret", "TOKEN_TWO_FACTOR_SECRET")
    maybe_set(
        settings,
        "warehouse.xmlrpc.search.enabled",
        "WAREHOUSE_XMLRPC_SEARCH",
        coercer=distutils.util.strtobool,
        default=True,
    )
    maybe_set(settings, "warehouse.xmlrpc.cache.url", "REDIS_URL")
    maybe_set(
        settings,
        "warehouse.xmlrpc.client.ratelimit_string",
        "XMLRPC_RATELIMIT_STRING",
        default="3600 per hour",
    )
    maybe_set(settings,
              "token.password.max_age",
              "TOKEN_PASSWORD_MAX_AGE",
              coercer=int)
    maybe_set(settings,
              "token.email.max_age",
              "TOKEN_EMAIL_MAX_AGE",
              coercer=int)
    maybe_set(
        settings,
        "token.two_factor.max_age",
        "TOKEN_TWO_FACTOR_MAX_AGE",
        coercer=int,
        default=300,
    )
    maybe_set(
        settings,
        "token.default.max_age",
        "TOKEN_DEFAULT_MAX_AGE",
        coercer=int,
        default=21600,  # 6 hours
    )
    maybe_set_compound(settings, "files", "backend", "FILES_BACKEND")
    maybe_set_compound(settings, "simple", "backend", "SIMPLE_BACKEND")
    maybe_set_compound(settings, "docs", "backend", "DOCS_BACKEND")
    maybe_set_compound(settings, "sponsorlogos", "backend",
                       "SPONSORLOGOS_BACKEND")
    maybe_set_compound(settings, "origin_cache", "backend", "ORIGIN_CACHE")
    maybe_set_compound(settings, "mail", "backend", "MAIL_BACKEND")
    maybe_set_compound(settings, "metrics", "backend", "METRICS_BACKEND")
    maybe_set_compound(settings, "breached_passwords", "backend",
                       "BREACHED_PASSWORDS")
    maybe_set_compound(settings, "malware_check", "backend",
                       "MALWARE_CHECK_BACKEND")

    # Pythondotorg integration settings
    maybe_set(settings,
              "pythondotorg.host",
              "PYTHONDOTORG_HOST",
              default="python.org")
    maybe_set(settings, "pythondotorg.api_token", "PYTHONDOTORG_API_TOKEN")

    # Configure our ratelimiters
    maybe_set(
        settings,
        "warehouse.account.user_login_ratelimit_string",
        "USER_LOGIN_RATELIMIT_STRING",
        default="10 per 5 minutes",
    )
    maybe_set(
        settings,
        "warehouse.account.ip_login_ratelimit_string",
        "IP_LOGIN_RATELIMIT_STRING",
        default="10 per 5 minutes",
    )
    maybe_set(
        settings,
        "warehouse.account.global_login_ratelimit_string",
        "GLOBAL_LOGIN_RATELIMIT_STRING",
        default="1000 per 5 minutes",
    )
    maybe_set(
        settings,
        "warehouse.account.email_add_ratelimit_string",
        "EMAIL_ADD_RATELIMIT_STRING",
        default="2 per day",
    )
    maybe_set(
        settings,
        "warehouse.account.password_reset_ratelimit_string",
        "PASSWORD_RESET_RATELIMIT_STRING",
        default="5 per day",
    )
    maybe_set(
        settings,
        "warehouse.manage.oidc.user_registration_ratelimit_string",
        "USER_OIDC_REGISTRATION_RATELIMIT_STRING",
        default="20 per day",
    )
    maybe_set(
        settings,
        "warehouse.manage.oidc.ip_registration_ratelimit_string",
        "IP_OIDC_REGISTRATION_RATELIMIT_STRING",
        default="20 per day",
    )

    # 2FA feature flags
    maybe_set(
        settings,
        "warehouse.two_factor_requirement.enabled",
        "TWOFACTORREQUIREMENT_ENABLED",
        coercer=distutils.util.strtobool,
        default=False,
    )
    maybe_set(
        settings,
        "warehouse.two_factor_mandate.available",
        "TWOFACTORMANDATE_AVAILABLE",
        coercer=distutils.util.strtobool,
        default=False,
    )
    maybe_set(
        settings,
        "warehouse.two_factor_mandate.enabled",
        "TWOFACTORMANDATE_ENABLED",
        coercer=distutils.util.strtobool,
        default=False,
    )

    # OIDC feature flags
    maybe_set(
        settings,
        "warehouse.oidc.enabled",
        "OIDC_ENABLED",
        coercer=distutils.util.strtobool,
        default=False,
    )

    # Add the settings we use when the environment is set to development.
    if settings["warehouse.env"] == Environment.development:
        settings.setdefault("enforce_https", False)
        settings.setdefault("pyramid.reload_assets", True)
        settings.setdefault("pyramid.reload_templates", True)
        settings.setdefault("pyramid.prevent_http_cache", True)
        settings.setdefault("debugtoolbar.hosts", ["0.0.0.0/0"])
        settings.setdefault(
            "debugtoolbar.panels",
            [
                ".".join(["pyramid_debugtoolbar.panels", panel]) for panel in [
                    "versions.VersionDebugPanel",
                    "settings.SettingsDebugPanel",
                    "headers.HeaderDebugPanel",
                    "request_vars.RequestVarsDebugPanel",
                    "renderings.RenderingsDebugPanel",
                    "logger.LoggingPanel",
                    "performance.PerformanceDebugPanel",
                    "routes.RoutesDebugPanel",
                    "sqla.SQLADebugPanel",
                    "tweens.TweensDebugPanel",
                    "introspection.IntrospectionDebugPanel",
                ]
            ],
        )

    # Actually setup our Pyramid Configurator with the values pulled in from
    # the environment as well as the ones passed in to the configure function.
    config = Configurator(settings=settings)
    config.set_root_factory(RootFactory)

    # Register support for services
    config.include("pyramid_services")

    # Register metrics
    config.include(".metrics")

    # Register our CSRF support. We do this here, immediately after we've
    # created the Configurator instance so that we ensure to get our defaults
    # set ASAP before anything else has a chance to set them and possibly call
    # Configurator().commit()
    config.include(".csrf")

    # Include anything needed by the development environment.
    if config.registry.settings["warehouse.env"] == Environment.development:
        config.include("pyramid_debugtoolbar")

    # Register our logging support
    config.include(".logging")

    # We'll want to use Jinja2 as our template system.
    config.include("pyramid_jinja2")

    # Include our filters
    config.include(".filters")

    # Including pyramid_mailer for sending emails through SMTP.
    config.include("pyramid_mailer")

    # We want to use newstyle gettext
    config.add_settings({"jinja2.newstyle": True})

    # Our translation strings are all in the "messages" domain
    config.add_settings({"jinja2.i18n.domain": "messages"})

    # We also want to use Jinja2 for .html templates as well, because we just
    # assume that all templates will be using Jinja.
    config.add_jinja2_renderer(".html")

    # Sometimes our files are .txt files and we still want to use Jinja2 to
    # render them.
    config.add_jinja2_renderer(".txt")

    # Anytime we want to render a .xml template, we'll also use Jinja.
    config.add_jinja2_renderer(".xml")

    # We need to enable our Client Side Include extension
    config.get_settings().setdefault(
        "jinja2.extensions",
        ["warehouse.utils.html.ClientSideIncludeExtension"])

    # We'll want to configure some filters for Jinja2 as well.
    filters = config.get_settings().setdefault("jinja2.filters", {})
    filters.setdefault("format_classifiers",
                       "warehouse.filters:format_classifiers")
    filters.setdefault("classifier_id", "warehouse.filters:classifier_id")
    filters.setdefault("format_tags", "warehouse.filters:format_tags")
    filters.setdefault("json", "warehouse.filters:tojson")
    filters.setdefault("camoify", "warehouse.filters:camoify")
    filters.setdefault("shorten_number", "warehouse.filters:shorten_number")
    filters.setdefault("urlparse", "warehouse.filters:urlparse")
    filters.setdefault("contains_valid_uris",
                       "warehouse.filters:contains_valid_uris")
    filters.setdefault("format_package_type",
                       "warehouse.filters:format_package_type")
    filters.setdefault("parse_version", "warehouse.filters:parse_version")
    filters.setdefault("localize_datetime",
                       "warehouse.filters:localize_datetime")
    filters.setdefault("is_recent", "warehouse.filters:is_recent")

    # We also want to register some global functions for Jinja
    jglobals = config.get_settings().setdefault("jinja2.globals", {})
    jglobals.setdefault("is_valid_uri", "warehouse.utils.http:is_valid_uri")
    jglobals.setdefault("gravatar", "warehouse.utils.gravatar:gravatar")
    jglobals.setdefault("gravatar_profile", "warehouse.utils.gravatar:profile")
    jglobals.setdefault("now", "warehouse.utils:now")

    # And some enums to reuse in the templates
    jglobals.setdefault("AdminFlagValue",
                        "warehouse.admin.flags:AdminFlagValue")
    jglobals.setdefault(
        "OrganizationInvitationStatus",
        "warehouse.organizations.models:OrganizationInvitationStatus",
    )
    jglobals.setdefault("OrganizationRoleType",
                        "warehouse.organizations.models:OrganizationRoleType")
    jglobals.setdefault("OrganizationType",
                        "warehouse.organizations.models:OrganizationType")
    jglobals.setdefault("RoleInvitationStatus",
                        "warehouse.packaging.models:RoleInvitationStatus")

    # We'll store all of our templates in one location, warehouse/templates
    # so we'll go ahead and add that to the Jinja2 search path.
    config.add_jinja2_search_path("warehouse:templates", name=".html")
    config.add_jinja2_search_path("warehouse:templates", name=".txt")
    config.add_jinja2_search_path("warehouse:templates", name=".xml")

    # We want to configure our JSON renderer to sort the keys, and also to use
    # an ultra compact serialization format.
    config.add_renderer("json",
                        renderers.JSON(sort_keys=True, separators=(",", ":")))

    # Configure retry support.
    config.add_settings({"retry.attempts": 3})
    config.include("pyramid_retry")

    # Configure our transaction handling so that each request gets its own
    # transaction handler and the lifetime of the transaction is tied to the
    # lifetime of the request.
    config.add_settings({
        "tm.manager_hook":
        lambda request: transaction.TransactionManager(),
        "tm.activate_hook":
        activate_hook,
        "tm.commit_veto":
        commit_veto,
        "tm.annotate_user":
        False,
    })
    config.include("pyramid_tm")

    # Register our XMLRPC service
    config.include(".legacy.api.xmlrpc")

    # Register our XMLRPC cache
    config.include(".legacy.api.xmlrpc.cache")

    # Register support for XMLRPC and override it's renderer to allow
    # specifying custom dumps arguments.
    config.include("pyramid_rpc.xmlrpc")
    config.add_renderer("xmlrpc", XMLRPCRenderer(allow_none=True))

    # Register support for our legacy action URLs
    config.include(".legacy.action_routing")

    # Register support for our custom predicates
    config.include(".predicates")

    # Register support for template views.
    config.add_directive("add_template_view", template_view, action_wrap=False)

    # Register support for internationalization and localization
    config.include(".i18n")

    # Register the configuration for the PostgreSQL database.
    config.include(".db")

    # Register the support for Celery Tasks
    config.include(".tasks")

    # Register support for our rate limiting mechanisms
    config.include(".rate_limiting")

    config.include(".static")

    config.include(".policy")

    config.include(".search")

    # Register the support for AWS and Google Cloud
    config.include(".aws")
    config.include(".gcloud")

    # Register our session support
    config.include(".sessions")

    # Register our support for http and origin caching
    config.include(".cache.http")
    config.include(".cache.origin")

    # Register support for sending emails
    config.include(".email")

    # Register our authentication support.
    config.include(".accounts")

    # Register support for Macaroon based authentication
    config.include(".macaroons")

    # Register support for OIDC provider based authentication
    config.include(".oidc")

    # Register support for malware checks
    config.include(".malware")

    # Register logged-in views
    config.include(".manage")

    # Register our organization support.
    config.include(".organizations")

    # Allow the packaging app to register any services it has.
    config.include(".packaging")

    # Configure redirection support
    config.include(".redirects")

    # Register all our URL routes for Warehouse.
    config.include(".routes")

    # Allow the sponsors app to list sponsors
    config.include(".sponsors")

    # Allow the banners app to list banners
    config.include(".banners")

    # Include our admin application
    config.include(".admin")

    # Register forklift, at least until we split it out into it's own project.
    config.include(".forklift")

    # Block non HTTPS requests for the legacy ?:action= routes when they are
    # sent via POST.
    config.add_tween("warehouse.config.require_https_tween_factory")

    # Enable compression of our HTTP responses
    config.add_tween(
        "warehouse.utils.compression.compression_tween_factory",
        over=[
            "warehouse.cache.http.conditional_http_tween_factory",
            "pyramid_debugtoolbar.toolbar_tween_factory",
            EXCVIEW,
        ],
    )

    # Enable Warehouse to serve our static files
    prevent_http_cache = config.get_settings().get(
        "pyramid.prevent_http_cache", False)
    config.add_static_view(
        "static",
        "warehouse:static/dist/",
        # Don't cache at all if prevent_http_cache is true, else we'll cache
        # the files for 10 years.
        cache_max_age=0 if prevent_http_cache else 10 * 365 * 24 * 60 * 60,
    )
    config.add_cache_buster(
        "warehouse:static/dist/",
        ManifestCacheBuster(
            "warehouse:static/dist/manifest.json",
            reload=config.registry.settings["pyramid.reload_assets"],
            strict=not prevent_http_cache,
        ),
    )
    config.whitenoise_serve_static(
        autorefresh=prevent_http_cache,
        max_age=0 if prevent_http_cache else 10 * 365 * 24 * 60 * 60,
    )
    config.whitenoise_add_files("warehouse:static/dist/", prefix="/static/")
    config.whitenoise_add_manifest("warehouse:static/dist/manifest.json",
                                   prefix="/static/")

    # Enable support of passing certain values like remote host, client
    # address, and protocol support in from an outer proxy to the application.
    config.add_wsgi_middleware(
        ProxyFixer,
        token=config.registry.settings["warehouse.token"],
        num_proxies=config.registry.settings.get("warehouse.num_proxies", 1),
    )

    # Protect against cache poisoning via the X-Vhm-Root headers.
    config.add_wsgi_middleware(VhmRootRemover)

    # We want Sentry to be the last things we add here so that it's the outer
    # most WSGI middleware.
    config.include(".sentry")

    # Register Content-Security-Policy service
    config.include(".csp")

    # Register Referrer-Policy service
    config.include(".referrer_policy")

    config.add_settings({"http": {"verify": "/etc/ssl/certs/"}})
    config.include(".http")

    # Scan everything for configuration
    config.scan(
        categories=(
            "pyramid",
            "warehouse",
        ),
        ignore=[
            "warehouse.migrations.env", "warehouse.celery", "warehouse.wsgi"
        ],
    )

    # Sanity check our request and responses.
    # Note: It is very important that this go last. We need everything else that might
    #       have added a tween to be registered prior to this.
    config.include(".sanity")

    # Finally, commit all of our changes
    config.commit()

    return config
示例#11
0
def configure(settings=None):
    if settings is None:
        settings = {}

    # Add information about the current copy of the code.
    settings.setdefault("warehouse.commit", __commit__)

    # Set the environment from an environment variable, if one hasn't already
    # been set.
    maybe_set(
        settings,
        "warehouse.env",
        "WAREHOUSE_ENV",
        Environment,
        default=Environment.production,
    )

    # Pull in default configuration from the environment.
    maybe_set(settings, "warehouse.token", "WAREHOUSE_TOKEN")
    maybe_set(settings, "warehouse.theme", "WAREHOUSE_THEME")
    maybe_set(settings, "site.name", "SITE_NAME", default="Warehouse")
    maybe_set(settings, "aws.key_id", "AWS_ACCESS_KEY_ID")
    maybe_set(settings, "aws.secret_key", "AWS_SECRET_ACCESS_KEY")
    maybe_set(settings, "aws.region", "AWS_REGION")
    maybe_set(settings, "celery.broker_url", "AMQP_URL")
    maybe_set(settings, "celery.result_url", "REDIS_URL")
    maybe_set(settings, "csp.report_uri", "CSP_REPORT_URI")
    maybe_set(settings, "database.url", "DATABASE_URL")
    maybe_set(settings, "elasticsearch.url", "ELASTICSEARCH_URL")
    maybe_set(settings, "sentry.dsn", "SENTRY_DSN")
    maybe_set(settings, "sentry.transport", "SENTRY_TRANSPORT")
    maybe_set(settings, "sessions.url", "REDIS_URL")
    maybe_set(settings, "download_stats.url", "REDIS_URL")
    maybe_set(settings, "sessions.secret", "SESSION_SECRET")
    maybe_set(settings, "camo.url", "CAMO_URL")
    maybe_set(settings, "camo.key", "CAMO_KEY")
    maybe_set(settings, "docs.url", "DOCS_URL")
    maybe_set_compound(settings, "files", "backend", "FILES_BACKEND")
    maybe_set_compound(settings, "origin_cache", "backend", "ORIGIN_CACHE")

    # Add the settings we use when the environment is set to development.
    if settings["warehouse.env"] == Environment.development:
        settings.setdefault("enforce_https", False)
        settings.setdefault("pyramid.reload_assets", True)
        settings.setdefault("pyramid.reload_templates", True)
        settings.setdefault("pyramid.prevent_http_cache", True)
        settings.setdefault("debugtoolbar.hosts", ["0.0.0.0/0"])
        settings.setdefault(
            "debugtoolbar.panels",
            [
                ".".join(["pyramid_debugtoolbar.panels", panel]) for panel in [
                    "versions.VersionDebugPanel",
                    "settings.SettingsDebugPanel",
                    "headers.HeaderDebugPanel",
                    "request_vars.RequestVarsDebugPanel",
                    "renderings.RenderingsDebugPanel",
                    "logger.LoggingPanel",
                    "performance.PerformanceDebugPanel",
                    "routes.RoutesDebugPanel",
                    "sqla.SQLADebugPanel",
                    "tweens.TweensDebugPanel",
                    "introspection.IntrospectionDebugPanel",
                ]
            ],
        )

    # Actually setup our Pyramid Configurator with the values pulled in from
    # the environment as well as the ones passed in to the configure function.
    config = Configurator(settings=settings)

    # Include anything needed by the development environment.
    if config.registry.settings["warehouse.env"] == Environment.development:
        config.include("pyramid_debugtoolbar")

    # Register our logging support
    config.include(".logging")

    # We'll want to use Jinja2 as our template system.
    config.include("pyramid_jinja2")

    # We want to use newstyle gettext
    config.add_settings({"jinja2.newstyle": True})

    # We also want to use Jinja2 for .html templates as well, because we just
    # assume that all templates will be using Jinja.
    config.add_jinja2_renderer(".html")

    # Sometimes our files are .txt files and we still want to use Jinja2 to
    # render them.
    config.add_jinja2_renderer(".txt")

    # Anytime we want to render a .xml template, we'll also use Jinja.
    config.add_jinja2_renderer(".xml")

    # We'll want to configure some filters for Jinja2 as well.
    filters = config.get_settings().setdefault("jinja2.filters", {})
    filters.setdefault("format_tags", "warehouse.filters:format_tags")
    filters.setdefault("json", "warehouse.filters:tojson")
    filters.setdefault("readme", "warehouse.filters:readme")
    filters.setdefault("shorten_number", "warehouse.filters:shorten_number")
    filters.setdefault("urlparse", "warehouse.filters:urlparse")

    # We also want to register some global functions for Jinja
    jglobals = config.get_settings().setdefault("jinja2.globals", {})
    jglobals.setdefault("gravatar", "warehouse.utils.gravatar:gravatar")
    jglobals.setdefault("html_include", "warehouse.utils.html:html_include")
    jglobals.setdefault("now", "warehouse.utils:now")

    # We'll store all of our templates in one location, warehouse/templates
    # so we'll go ahead and add that to the Jinja2 search path.
    config.add_jinja2_search_path("warehouse:templates", name=".html")
    config.add_jinja2_search_path("warehouse:templates", name=".txt")
    config.add_jinja2_search_path("warehouse:templates", name=".xml")

    # We want to configure our JSON renderer to sort the keys, and also to use
    # an ultra compact serialization format.
    config.add_renderer(
        "json",
        renderers.JSON(sort_keys=True, separators=(",", ":")),
    )

    # Configure our transaction handling so that each request gets its own
    # transaction handler and the lifetime of the transaction is tied to the
    # lifetime of the request.
    config.add_settings({
        "tm.attempts":
        3,
        "tm.manager_hook":
        lambda request: transaction.TransactionManager(),
        "tm.activate_hook":
        activate_hook,
        "tm.annotate_user":
        False,
    })
    config.include("pyramid_tm")

    # Register support for services
    config.include("pyramid_services")

    # Register our find_service_factory methods
    config.add_request_method(find_service_factory)
    config.add_directive("find_service_factory", find_service_factory)

    # Register support for XMLRPC and override it's renderer to allow
    # specifying custom dumps arguments.
    config.include("pyramid_rpc.xmlrpc")
    config.add_renderer("xmlrpc", XMLRPCRenderer(allow_none=True))

    # Register support for our legacy action URLs
    config.include(".legacy.action_routing")

    # Register support for internationalization and localization
    config.include(".i18n")

    # Register the configuration for the PostgreSQL database.
    config.include(".db")

    config.include(".search")

    # Register the support for AWS
    config.include(".aws")

    # Register the support for Celery
    config.include(".celery")

    # Register our session support
    config.include(".sessions")

    # Register our support for http and origin caching
    config.include(".cache.http")
    config.include(".cache.origin")

    # Register our CSRF support
    config.include(".csrf")

    # Register our authentication support.
    config.include(".accounts")

    # Allow the packaging app to register any services it has.
    config.include(".packaging")

    # Configure redirection support
    config.include(".redirects")

    # Register all our URL routes for Warehouse.
    config.include(".routes")

    # Enable a Content Security Policy
    config.add_settings({
        "csp": {
            "connect-src": ["'self'"],
            "default-src": ["'none'"],
            "font-src": ["'self'", "fonts.gstatic.com"],
            "frame-ancestors": ["'none'"],
            "img-src": [
                "'self'",
                config.registry.settings["camo.url"],
                "https://secure.gravatar.com",
            ],
            "referrer": ["origin-when-cross-origin"],
            "reflected-xss": ["block"],
            "report-uri": [config.registry.settings.get("csp.report_uri")],
            "script-src": ["'self'"],
            "style-src": ["'self'", "fonts.googleapis.com"],
        },
    })
    config.add_tween("warehouse.config.content_security_policy_tween_factory")

    # Block non HTTPS requests for the legacy ?:action= routes when they are
    # sent via POST.
    config.add_tween("warehouse.config.require_https_tween_factory")

    # Enable compression of our HTTP responses
    config.add_tween(
        "warehouse.utils.compression.compression_tween_factory",
        over=[
            "warehouse.cache.http.conditional_http_tween_factory",
            "pyramid_debugtoolbar.toolbar_tween_factory",
            "warehouse.raven.raven_tween_factory",
            EXCVIEW,
        ],
    )

    # Enable Warehouse to serve our static files
    prevent_http_cache = \
        config.get_settings().get("pyramid.prevent_http_cache", False)
    config.add_static_view(
        "static",
        "warehouse:static/dist/",
        # Don't cache at all if prevent_http_cache is true, else we'll cache
        # the files for 10 years.
        cache_max_age=0 if prevent_http_cache else 10 * 365 * 24 * 60 * 60,
    )
    config.add_cache_buster(
        "warehouse:static/dist/",
        ManifestCacheBuster(
            "warehouse:static/dist/manifest.json",
            reload=config.registry.settings["pyramid.reload_assets"],
            strict=not prevent_http_cache,
        ),
    )

    # Enable Warehouse to serve our locale files
    config.add_static_view("locales", "warehouse:locales/")

    # Enable support of passing certain values like remote host, client
    # address, and protocol support in from an outer proxy to the application.
    config.add_wsgi_middleware(
        ProxyFixer,
        token=config.registry.settings["warehouse.token"],
    )

    # Protect against cache poisoning via the X-Vhm-Root headers.
    config.add_wsgi_middleware(VhmRootRemover)

    # We want Raven to be the last things we add here so that it's the outer
    # most WSGI middleware.
    config.include(".raven")

    # Add our theme if one was configured
    if config.get_settings().get("warehouse.theme"):
        config.include(config.get_settings()["warehouse.theme"])

    # Scan everything for configuration
    config.scan(ignore=["warehouse.migrations.env", "warehouse.wsgi"])

    return config