def __init__(self, db, config): self.db = db self.config = config self.es = Elasticsearch( hosts=self.config.hosts, **self.config.get("client_options", {}) ) self.types = AttributeDict() self._index = config.index
def __init__(self, config, engine=None, redis_class=redis.StrictRedis): self.config = AttributeDict(config) self.metadata = db.metadata # configure logging logging.config.dictConfig(self.config.logging) # Connect to the database if engine is None and self.config.get("database", {}).get("url"): engine = sqlalchemy.create_engine(self.config.database.url) self.engine = engine # Create our redis connections self.redises = { key: redis_class.from_url(url) for key, url in self.config.redis.items() } # Create our Store instance and associate our store modules with it self.db = AttributeDict() for name, klass in self.db_classes.items(): self.db[name] = klass( self, self.metadata, self.engine, self.redises["downloads"], ) # Create our Search Index instance and associate our mappings with it self.search = Index(self.db, self.config.search) self.search.register(ProjectMapping) # Set up our URL routing self.urls = urls.urls # Initialize our Translations engine self.translations = babel.support.NullTranslations() # Setup our Jinja2 Environment self.templates = jinja2.Environment( autoescape=True, auto_reload=self.config.debug, extensions=[ "jinja2.ext.i18n", ], loader=jinja2.PackageLoader("warehouse"), ) # Install Babel self.templates.filters.update({ "package_type_display": packaging_helpers.package_type_display, "format_number": babel.numbers.format_number, "format_decimal": babel.numbers.format_decimal, "format_percent": babel.numbers.format_percent, "format_date": babel.dates.format_date, "format_datetime": babel.dates.format_datetime, "format_time": babel.dates.format_time, }) # Install our translations self.templates.install_gettext_translations( self.translations, newstyle=True, ) # Setup our password hasher self.passlib = passlib.context.CryptContext( schemes=[ "bcrypt_sha256", "bcrypt", "django_bcrypt", "unix_disabled", ], default="bcrypt_sha256", deprecated=["auto"], ) # Setup our session storage self.session_store = RedisSessionStore( self.redises["sessions"], session_class=Session, ) # Add our Content Security Policy Middleware img_src = ["'self'"] if self.config.camo: camo_parsed = urllib.parse.urlparse(self.config.camo.url) img_src += [ "{}://{}".format(camo_parsed.scheme, camo_parsed.netloc), "https://secure.gravatar.com", ] else: img_src += ["*"] self.wsgi_app = guard.ContentSecurityPolicy( self.wsgi_app, { "default-src": ["'self'"], "font-src": ["'self'", "data:"], "img-src": img_src, "style-src": ["'self'", "cloud.typography.com"], }, ) if "sentry" in self.config: self.wsgi_app = Sentry(self.wsgi_app, Client(**self.config.sentry)) # Serve the static files that are packaged as part of Warehouse self.wsgi_app = WhiteNoise( self.wsgi_app, root=self.static_dir, prefix=self.static_path, max_age=31557600, ) # Add our Powered By Middleware self.wsgi_app = HeaderRewriterFix( self.wsgi_app, add_headers=[ ( "X-Powered-By", "Warehouse {__version__} ({__build__})".format( __version__=warehouse.__version__, __build__=warehouse.__build__, ), ), ], ) # This is last because we want it processed first in the stack of # middlewares. This will ensure that we strip X-Forwarded-* headers # if the request doesn't come from Fastly self.wsgi_app = XForwardedTokenMiddleware( self.wsgi_app, self.config.site.access_token, )
class Warehouse(object): db_classes = { "accounts": warehouse.accounts.db.Database, "packaging": warehouse.packaging.db.Database, } static_dir = os.path.abspath( os.path.join(os.path.dirname(warehouse.__file__), "static", "compiled") ) static_path = "/static/" def __init__(self, config, engine=None, redis_class=redis.StrictRedis): self.config = AttributeDict(config) self.metadata = db.metadata # configure logging logging.config.dictConfig(self.config.logging) # Connect to the database if engine is None and self.config.get("database", {}).get("url"): engine = sqlalchemy.create_engine(self.config.database.url) self.engine = engine # Create our redis connections self.redises = { key: redis_class.from_url(url) for key, url in self.config.redis.items() } # Create our Store instance and associate our store modules with it self.db = AttributeDict() for name, klass in self.db_classes.items(): self.db[name] = klass( self, self.metadata, self.engine, self.redises["downloads"], ) # Create our Search Index instance and associate our mappings with it self.search = Index(self.db, self.config.search) self.search.register(ProjectMapping) # Set up our URL routing self.urls = urls.urls # Initialize our Translations engine self.translations = babel.support.NullTranslations() # Setup our Jinja2 Environment self.templates = jinja2.Environment( autoescape=True, auto_reload=self.config.debug, extensions=[ "jinja2.ext.i18n", ], loader=jinja2.PackageLoader("warehouse"), ) # Install Babel self.templates.filters.update({ "package_type_display": packaging_helpers.package_type_display, "format_number": babel.numbers.format_number, "format_decimal": babel.numbers.format_decimal, "format_percent": babel.numbers.format_percent, "format_date": babel.dates.format_date, "format_datetime": babel.dates.format_datetime, "format_time": babel.dates.format_time, }) # Install our translations self.templates.install_gettext_translations( self.translations, newstyle=True, ) # Setup our password hasher self.passlib = passlib.context.CryptContext( schemes=[ "bcrypt_sha256", "bcrypt", "django_bcrypt", "unix_disabled", ], default="bcrypt_sha256", deprecated=["auto"], ) # Setup our session storage self.session_store = RedisSessionStore( self.redises["sessions"], session_class=Session, ) # Add our Content Security Policy Middleware img_src = ["'self'"] if self.config.camo: camo_parsed = urllib.parse.urlparse(self.config.camo.url) img_src += [ "{}://{}".format(camo_parsed.scheme, camo_parsed.netloc), "https://secure.gravatar.com", ] else: img_src += ["*"] self.wsgi_app = guard.ContentSecurityPolicy( self.wsgi_app, { "default-src": ["'self'"], "font-src": ["'self'", "data:"], "img-src": img_src, "style-src": ["'self'", "cloud.typography.com"], }, ) if "sentry" in self.config: self.wsgi_app = Sentry(self.wsgi_app, Client(**self.config.sentry)) # Serve the static files that are packaged as part of Warehouse self.wsgi_app = WhiteNoise( self.wsgi_app, root=self.static_dir, prefix=self.static_path, max_age=31557600, ) # Add our Powered By Middleware self.wsgi_app = HeaderRewriterFix( self.wsgi_app, add_headers=[ ( "X-Powered-By", "Warehouse {__version__} ({__build__})".format( __version__=warehouse.__version__, __build__=warehouse.__build__, ), ), ], ) # This is last because we want it processed first in the stack of # middlewares. This will ensure that we strip X-Forwarded-* headers # if the request doesn't come from Fastly self.wsgi_app = XForwardedTokenMiddleware( self.wsgi_app, self.config.site.access_token, ) def __call__(self, environ, start_response): """ Shortcut for :attr:`wsgi_app`. """ return self.wsgi_app(environ, start_response) @classmethod def from_yaml(cls, *paths, **kwargs): # Pull out other keyword arguments override = kwargs.pop("override", None) default = os.path.abspath(os.path.join( os.path.dirname(warehouse.__file__), "config.yml", )) paths = [default] + list(paths) config = {} for path in paths: with open(path) as configfile: # Use no cover to work around a coverage bug config = merge_dict( # pragma: no cover config, yaml.safe_load(configfile) ) if override: config = merge_dict(config, override) return cls(config=config, **kwargs) @classmethod def from_cli(cls, argv): def _generate_parser(parser, commands): # Generate our commands subparsers = parser.add_subparsers() for name, command in commands.items(): cmd_parser = subparsers.add_parser(name) if hasattr(command, "create_parser"): command.create_parser(cmd_parser) if isinstance(command, collections.Mapping): _generate_parser(cmd_parser, command) else: cmd_parser.set_defaults(_cmd=command) parser = argparse.ArgumentParser(prog="warehouse") parser.add_argument("-c", "--config", action="append", dest="_configs") _generate_parser(parser, warehouse.cli.__commands__) args = parser.parse_args(argv) configs = args._configs if args._configs is not None else [] app = cls.from_yaml(*configs) return args._cmd( app, *args._get_args(), **{k: v for k, v in args._get_kwargs() if not k.startswith("_")} ) # The order of these decorators matter. We need @handle_session to come # before anything that depends on it, like @handle_csrf @handle_session @handle_csrf def dispatch_view(self, view, *args, **kwargs): return view(*args, **kwargs) @responder def wsgi_app(self, environ, start_response): """ The actual WSGI application. This is not implemented in `__call__` so that middlewares can be applied without losing a reference to the class. So instead of doing this:: app = MyMiddleware(app) It's a better idea to do this instead:: app.wsgi_app = MyMiddleware(app.wsgi_app) Then you still have the original application object around and can continue to call methods on it. :param environ: a WSGI environment :param start_response: a callable accepting a status code, a list of headers and an optional exception context to start the response """ try: # Figure out what endpoint to call urls = self.urls.bind_to_environ(environ) endpoint, kwargs = urls.match() # Load our view function modname, viewname = endpoint.rsplit(".", 1) module = importlib.import_module(modname) view = getattr(module, viewname) # Create our request object request = Request(environ) request.url_adapter = urls # Attach our trusted hosts to this request request.trusted_hosts = self.config.site.hosts # Access request.host to trigger a check against our trusted_hosts request.host # Dispatch to the loaded view function return self.dispatch_view(view, self, request, **kwargs) except HTTPException as exc: return exc
class Index(object): def __init__(self, db, config): self.db = db self.config = config self.es = Elasticsearch( hosts=self.config.hosts, **self.config.get("client_options", {}) ) self.types = AttributeDict() self._index = config.index def register(self, type_): obj = type_(self) self.types[obj._type] = obj def reindex(self, alias=True, keep_old=False): # Generate an Index Name for Warehouse index = "".join([ self._index, binascii.hexlify(os.urandom(4)).decode("ascii"), ]) # Create this index self.es.indices.create(index, { "mappings": { doc_type._type: doc_type.get_mapping() for doc_type in self.types.values() }, }) # Index everything into the new index for doc_type in self.types.values(): doc_type.index_all(index=index) # Update the alias unless we've been told not to if alias: self.update_alias(self._index, index, keep_old=keep_old) def update_alias(self, alias, index, keep_old=False): # Get the old index from ElasticSearch try: old_index = list(self.es.indices.get_alias(self._index))[0] except TransportError as exc: if not exc.status_code == 404: raise old_index = None # Remove the alias to the old index if it exists if old_index is not None: actions = [{"remove": {"index": old_index, "alias": alias}}] else: actions = [] # Add the alias to the new index actions += [{"add": {"index": index, "alias": alias}}] # Update To the New Index self.es.indices.update_aliases({"actions": actions}) # Delete the old index if it exists and unless we're keeping it if not keep_old and old_index is not None: self.es.indices.delete(old_index)
def test_convert_to_attribute_dict(): adict = AttributeDict({"a": {"b": 1, "c": 2}}) assert adict.a == {"b": 1, "c": 2} assert adict.a.b == 1 assert adict.a.c == 2
def test_attribute_dict_unknown_access(): adict = AttributeDict() with pytest.raises(AttributeError): adict.unknown
def test_basic_attribute_dict_access(): adict = AttributeDict({"foo": None, "bar": "Success!"}) assert adict.foo is adict["foo"] assert adict.bar is adict["bar"]
def __init__(self, config, engine=None, redis_class=redis.StrictRedis): self.config = AttributeDict(config) self.metadata = db.metadata # configure logging logging.config.dictConfig(self.config.logging) # Connect to the database if engine is None and self.config.get("database", {}).get("url"): engine = sqlalchemy.create_engine(self.config.database.url) self.engine = engine # Create our redis connections self.redises = { key: redis_class.from_url(url) for key, url in self.config.redis.items() } # Create our Store instance and associate our store modules with it self.db = AttributeDict() for name, klass in self.db_classes.items(): self.db[name] = klass( self, self.metadata, self.engine, self.redises["downloads"], ) # Create our Search Index instance and associate our mappings with it self.search = Index(self.db, self.config.search) self.search.register(ProjectMapping) # Set up our URL routing self.urls = urls.urls # Initialize our Translations engine self.translations = babel.support.NullTranslations() # Setup our Jinja2 Environment self.templates = jinja2.Environment( autoescape=True, auto_reload=self.config.debug, extensions=[ "jinja2.ext.i18n", ], loader=jinja2.PackageLoader("warehouse"), ) # Install Babel self.templates.filters.update({ "package_type_display": packaging_helpers.package_type_display, "format_number": babel.numbers.format_number, "format_decimal": babel.numbers.format_decimal, "format_percent": babel.numbers.format_percent, "format_date": babel.dates.format_date, "format_datetime": babel.dates.format_datetime, "format_time": babel.dates.format_time, }) # Install our translations self.templates.install_gettext_translations( self.translations, newstyle=True, ) # Setup our password hasher self.passlib = passlib.context.CryptContext( schemes=[ "bcrypt_sha256", "bcrypt", "django_bcrypt", "unix_disabled", ], default="bcrypt_sha256", deprecated=["auto"], ) # Setup our session storage self.session_store = RedisSessionStore( self.redises["sessions"], session_class=Session, ) # Add our Content Security Policy Middleware img_src = ["'self'"] if self.config.camo: camo_parsed = urllib.parse.urlparse(self.config.camo.url) img_src += [ "{}://{}".format(camo_parsed.scheme, camo_parsed.netloc), "https://secure.gravatar.com", ] else: img_src += ["*"] self.wsgi_app = guard.ContentSecurityPolicy( self.wsgi_app, { "default-src": ["'self'"], "font-src": ["'self'", "data:"], "img-src": img_src, "style-src": ["'self'", "cloud.typography.com"], }, ) if "sentry" in self.config: self.wsgi_app = Sentry(self.wsgi_app, Client(**self.config.sentry)) # Serve the static files that are packaged as part of Warehouse self.wsgi_app = WhiteNoise( self.wsgi_app, root=self.static_dir, prefix=self.static_path, max_age=31557600, ) # Add our Powered By Middleware self.wsgi_app = HeaderRewriterFix( self.wsgi_app, add_headers=[ ( "X-Powered-By", "Warehouse {__version__} ({__build__})".format( __version__=warehouse.__version__, __build__=warehouse.__build__, ), ), ], ) # Previously PyPI used a hand written disaptch method which depended # on things like the request's content type or url parameters. In order # to sanely support that in Warehouse we use this middleware to rewrite # those to "internal" URLs which we can then dispatch based on. self.wsgi_app = LegacyRewriteMiddleware(self.wsgi_app) # This is last because we want it processed first in the stack of # middlewares. This will ensure that we strip X-Forwarded-* headers # if the request doesn't come from Fastly self.wsgi_app = XForwardedTokenMiddleware( self.wsgi_app, self.config.site.access_token, )
class Warehouse(object): db_classes = { "accounts": warehouse.accounts.db.Database, "packaging": warehouse.packaging.db.Database, } static_dir = os.path.abspath( os.path.join(os.path.dirname(warehouse.__file__), "static", "compiled")) static_path = "/static/" def __init__(self, config, engine=None, redis_class=redis.StrictRedis): self.config = AttributeDict(config) self.metadata = db.metadata # configure logging logging.config.dictConfig(self.config.logging) # Connect to the database if engine is None and self.config.get("database", {}).get("url"): engine = sqlalchemy.create_engine(self.config.database.url) self.engine = engine # Create our redis connections self.redises = { key: redis_class.from_url(url) for key, url in self.config.redis.items() } # Create our Store instance and associate our store modules with it self.db = AttributeDict() for name, klass in self.db_classes.items(): self.db[name] = klass( self, self.metadata, self.engine, self.redises["downloads"], ) # Create our Search Index instance and associate our mappings with it self.search = Index(self.db, self.config.search) self.search.register(ProjectMapping) # Set up our URL routing self.urls = urls.urls # Initialize our Translations engine self.translations = babel.support.NullTranslations() # Setup our Jinja2 Environment self.templates = jinja2.Environment( autoescape=True, auto_reload=self.config.debug, extensions=[ "jinja2.ext.i18n", ], loader=jinja2.PackageLoader("warehouse"), ) # Install Babel self.templates.filters.update({ "package_type_display": packaging_helpers.package_type_display, "format_number": babel.numbers.format_number, "format_decimal": babel.numbers.format_decimal, "format_percent": babel.numbers.format_percent, "format_date": babel.dates.format_date, "format_datetime": babel.dates.format_datetime, "format_time": babel.dates.format_time, }) # Install our translations self.templates.install_gettext_translations( self.translations, newstyle=True, ) # Setup our password hasher self.passlib = passlib.context.CryptContext( schemes=[ "bcrypt_sha256", "bcrypt", "django_bcrypt", "unix_disabled", ], default="bcrypt_sha256", deprecated=["auto"], ) # Setup our session storage self.session_store = RedisSessionStore( self.redises["sessions"], session_class=Session, ) # Add our Content Security Policy Middleware img_src = ["'self'"] if self.config.camo: camo_parsed = urllib.parse.urlparse(self.config.camo.url) img_src += [ "{}://{}".format(camo_parsed.scheme, camo_parsed.netloc), "https://secure.gravatar.com", ] else: img_src += ["*"] self.wsgi_app = guard.ContentSecurityPolicy( self.wsgi_app, { "default-src": ["'self'"], "font-src": ["'self'", "data:"], "img-src": img_src, "style-src": ["'self'", "cloud.typography.com"], }, ) if "sentry" in self.config: self.wsgi_app = Sentry(self.wsgi_app, Client(**self.config.sentry)) # Serve the static files that are packaged as part of Warehouse self.wsgi_app = WhiteNoise( self.wsgi_app, root=self.static_dir, prefix=self.static_path, max_age=31557600, ) # Add our Powered By Middleware self.wsgi_app = HeaderRewriterFix( self.wsgi_app, add_headers=[ ( "X-Powered-By", "Warehouse {__version__} ({__build__})".format( __version__=warehouse.__version__, __build__=warehouse.__build__, ), ), ], ) # Previously PyPI used a hand written disaptch method which depended # on things like the request's content type or url parameters. In order # to sanely support that in Warehouse we use this middleware to rewrite # those to "internal" URLs which we can then dispatch based on. self.wsgi_app = LegacyRewriteMiddleware(self.wsgi_app) # This is last because we want it processed first in the stack of # middlewares. This will ensure that we strip X-Forwarded-* headers # if the request doesn't come from Fastly self.wsgi_app = XForwardedTokenMiddleware( self.wsgi_app, self.config.site.access_token, ) def __call__(self, environ, start_response): """ Shortcut for :attr:`wsgi_app`. """ return self.wsgi_app(environ, start_response) @classmethod def from_yaml(cls, *paths, **kwargs): # Pull out other keyword arguments override = kwargs.pop("override", None) default = os.path.abspath( os.path.join( os.path.dirname(warehouse.__file__), "config.yml", )) paths = [default] + list(paths) config = {} for path in paths: with open(path) as configfile: # Use no cover to work around a coverage bug config = merge_dict( # pragma: no cover config, yaml.safe_load(configfile)) if override: config = merge_dict(config, override) return cls(config=config, **kwargs) @classmethod def from_cli(cls, argv): def _generate_parser(parser, commands): # Generate our commands subparsers = parser.add_subparsers() for name, command in commands.items(): cmd_parser = subparsers.add_parser(name) if hasattr(command, "create_parser"): command.create_parser(cmd_parser) if isinstance(command, collections.Mapping): _generate_parser(cmd_parser, command) else: cmd_parser.set_defaults(_cmd=command) parser = argparse.ArgumentParser(prog="warehouse") parser.add_argument("-c", "--config", action="append", dest="_configs") _generate_parser(parser, warehouse.cli.__commands__) args = parser.parse_args(argv) configs = args._configs if args._configs is not None else [] app = cls.from_yaml(*configs) return args._cmd( app, *args._get_args(), **{k: v for k, v in args._get_kwargs() if not k.startswith("_")}) # The order of these decorators matter. We need @handle_session to come # before anything that depends on it, like @handle_csrf @handle_session @handle_csrf def dispatch_view(self, view, *args, **kwargs): return view(*args, **kwargs) @responder def wsgi_app(self, environ, start_response): """ The actual WSGI application. This is not implemented in `__call__` so that middlewares can be applied without losing a reference to the class. So instead of doing this:: app = MyMiddleware(app) It's a better idea to do this instead:: app.wsgi_app = MyMiddleware(app.wsgi_app) Then you still have the original application object around and can continue to call methods on it. :param environ: a WSGI environment :param start_response: a callable accepting a status code, a list of headers and an optional exception context to start the response """ try: # Figure out what endpoint to call urls = self.urls.bind_to_environ(environ) endpoint, kwargs = urls.match() # Load our view function modname, viewname = endpoint.rsplit(".", 1) module = importlib.import_module(modname) view = getattr(module, viewname) # Create our request object request = Request(environ) request.url_adapter = urls # Attach our trusted hosts to this request request.trusted_hosts = self.config.site.hosts # Access request.host to trigger a check against our trusted_hosts request.host # Dispatch to the loaded view function return self.dispatch_view(view, self, request, **kwargs) except HTTPException as exc: return exc