def test_initial_master_connection(self, plugin, makexom, monkeypatch): from devpi_server.replica import ReplicaThread import time now = time.time() xom = makexom(["--master=http://localhost"]) request = self._xomrequest(xom) xom.replica_thread = ReplicaThread(xom) assert xom.is_replica() assert xom.replica_thread.started_at is None # fake replica start xom.replica_thread.started_at = now # no report in the first minute result = plugin(request) assert result == [] # warning after one minute monkeypatch.setattr(devpi_server.views, "time", lambda: now + 70) result = plugin(request) assert result == [ dict(status='warn', msg='No contact to master for more than 1 minute') ] # fatal after five minutes monkeypatch.setattr(devpi_server.views, "time", lambda: now + 310) result = plugin(request) assert result == [ dict(status='fatal', msg='No contact to master for more than 5 minutes') ]
def test_replica_lagging(self, plugin, makexom, monkeypatch): from devpi_server.replica import ReplicaThread import time now = time.time() xom = makexom(["--master=http://localhost"]) request = self._xomrequest(xom) xom.replica_thread = ReplicaThread(xom) assert xom.is_replica() # fake first serial processed xom.replica_thread.update_master_serial(0) xom.replica_thread.replica_in_sync_at = now # no report in the first minute result = plugin(request) assert result == [] # warning after one minute monkeypatch.setattr(devpi_server.views, "time", lambda: now + 70) result = plugin(request) assert result == [ dict(status='warn', msg='Replica is behind master for more than 1 minute') ] # fatal after five minutes monkeypatch.setattr(devpi_server.views, "time", lambda: now + 310) result = plugin(request) assert result == [ dict(status='fatal', msg='Replica is behind master for more than 5 minutes') ]
def makexom(opts=(), httpget=httpget, plugins=()): from devpi_server import auth_basic from devpi_server import auth_devpi plugins = [ plugin[0] if isinstance(plugin, tuple) else plugin for plugin in plugins ] for plugin in [auth_basic, auth_devpi, storage_info["_test_plugin"]]: if plugin not in plugins: plugins.append(plugin) pm = get_pluginmanager(load_entrypoints=False) for plugin in plugins: pm.register(plugin) serverdir = gentmp() fullopts = ["devpi-server", "--serverdir", serverdir] + list(opts) if request.node.get_marker("with_replica_thread"): fullopts.append("--master=http://localhost") if not request.node.get_marker("no_storage_option"): if storage_info["name"] != "sqlite": fullopts.append("--storage=%s" % storage_info["name"]) fullopts = [str(x) for x in fullopts] config = parseoptions(pm, fullopts) config.init_nodeinfo() for marker in ("storage_with_filesystem", ): if request.node.get_marker(marker): info = config._storage_info() markers = info.get("_test_markers", []) if marker not in markers: pytest.skip("The storage doesn't have marker '%s'." % marker) if not request.node.get_marker("no_storage_option"): assert storage_info["storage"] is config.storage if request.node.get_marker("nomocking"): xom = XOM(config) else: xom = XOM(config, httpget=httpget) if not request.node.get_marker("nomockprojectsremote"): monkeypatch.setattr(extpypi.PyPIStage, "_get_remote_projects", lambda self: set()) add_pypistage_mocks(monkeypatch, httpget) # initialize default indexes from devpi_server.main import set_default_indexes if not xom.config.args.master_url: with xom.keyfs.transaction(write=True): set_default_indexes(xom.model) if request.node.get_marker("with_replica_thread"): from devpi_server.replica import ReplicaThread rt = ReplicaThread(xom) xom.replica_thread = rt xom.thread_pool.register(rt) xom.thread_pool.start_one(rt) if request.node.get_marker("start_threads"): xom.thread_pool.start() elif request.node.get_marker("with_notifier"): xom.thread_pool.start_one(xom.keyfs.notifier) request.addfinalizer(xom.thread_pool.shutdown) return xom
def replica_mapp(makemapp, master_host_port): from devpi_server.replica import ReplicaThread app = makemapp(options=['--master', 'http://%s:%s' % master_host_port]) rt = ReplicaThread(app.xom) app.xom.replica_thread = rt app.xom.thread_pool.register(rt) try: yield app finally: app.xom.thread_pool.shutdown()
def __init__(self, config, httpget=None): self.config = config self.thread_pool = mythread.ThreadPool() if httpget is not None: self.httpget = httpget self.log = threadlog self.polling_replicas = {} self._stagecache = {} if self.is_replica(): from devpi_server.replica import ReplicaThread from devpi_server.replica import register_key_subscribers search_path = self.config.replica_file_search_path if search_path and not os.path.exists(search_path): fatal("search path for existing replica files doesn't " "exist: %s" % search_path) register_key_subscribers(self) # the replica thread replays keyfs changes # and project-specific changes are discovered # and replayed through the PypiProjectChange event if not self.config.requests_only: self.replica_thread = ReplicaThread(self) self.thread_pool.register(self.replica_thread)
def rt(self, makexom): xom = makexom(["--master=http://localhost"]) rt = ReplicaThread(xom) xom.thread_pool.register(rt) return rt
def create_app(self): from devpi_server.view_auth import DevpiAuthenticationPolicy from devpi_server.views import ContentTypePredicate from devpi_server.views import OutsideURLMiddleware from devpi_server.views import route_url, INSTALLER_USER_AGENT from pkg_resources import get_distribution from pyramid.authorization import ACLAuthorizationPolicy from pyramid.config import Configurator from pyramid.viewderivers import INGRESS log = self.log log.debug("creating application in process %s", os.getpid()) pyramid_config = Configurator(root_factory='devpi_server.view_auth.RootFactory') pyramid_config.set_authentication_policy(DevpiAuthenticationPolicy(self)) pyramid_config.set_authorization_policy(ACLAuthorizationPolicy()) version_info = [ ("devpi-server", get_distribution("devpi_server").version)] for plug, distinfo in self.config.pluginmanager.list_plugin_distinfo(): threadlog.info("Found plugin %s-%s (%s)." % ( distinfo.project_name, distinfo.version, distinfo.location)) key = (distinfo.project_name, distinfo.version) if key not in version_info: version_info.append(key) version_info.sort() pyramid_config.registry['devpi_version_info'] = version_info pyramid_config.registry['xom'] = self index_classes = {} customizer_classes = sum( self.config.hook.devpiserver_get_stage_customizer_classes(), []) for ixtype, ixclass in customizer_classes: index_classes.setdefault(ixtype, []).append(ixclass) for ixtype, ixclasses in index_classes.items(): if len(ixclasses) > 1: fatal( "multiple implementation classes for index type '%s':\n%s" % ( ixtype, "\n".join( "%s.%s" % (x.__module__, x.__name__) for x in ixclasses))) self.config.hook.devpiserver_pyramid_configure( config=self.config, pyramid_config=pyramid_config) pyramid_config.add_view_deriver(self.view_deriver, under=INGRESS) pyramid_config.add_view_predicate('content_type', ContentTypePredicate) pyramid_config.add_route("/+changelog/{serial}", r"/+changelog/{serial:\d+}") pyramid_config.add_route("/+changelog/{serial}-", r"/+changelog/{serial:\d+}-") pyramid_config.add_route("/+status", "/+status") pyramid_config.add_route("/+api", "/+api", accept="application/json") pyramid_config.add_route("{path:.*}/+api", "{path:.*}/+api", accept="application/json") pyramid_config.add_route("/+login", "/+login", accept="application/json") pyramid_config.add_route("/{user}/{index}/+e/{relpath:.*}", "/{user}/{index}/+e/{relpath:.*}") pyramid_config.add_route("/{user}/{index}/+f/{relpath:.*}", "/{user}/{index}/+f/{relpath:.*}") pyramid_config.add_route("/{user}/{index}/+simple", "/{user}/{index}/+simple") pyramid_config.add_route("/{user}/{index}/+simple/", "/{user}/{index}/+simple/") pyramid_config.add_route("/{user}/{index}/+simple/{project}", "/{user}/{index}/+simple/{project}") pyramid_config.add_route("/{user}/{index}/+simple/{project}/", "/{user}/{index}/+simple/{project}/") pyramid_config.add_route("/{user}/{index}/+simple/{project}/refresh", "/{user}/{index}/+simple/{project}/refresh") pyramid_config.add_route("/{user}/{index}/{project}/{version}", "/{user}/{index}/{project}/{version:[^/]+/?}") pyramid_config.add_route( "simple_redirect", "/{user}/{index}/{project:[^/]+/?}", header="User-Agent:" + INSTALLER_USER_AGENT, accept="text/html", ) pyramid_config.add_route("/{user}/{index}/{project}", "/{user}/{index}/{project:[^/]+/?}") pyramid_config.add_route("/{user}/{index}/", "/{user}/{index}/") pyramid_config.add_route("/{user}/{index}", "/{user}/{index}") pyramid_config.add_route("/{user}", "/{user}") pyramid_config.add_route("/", "/") # register tweens for logging, transaction and replication pyramid_config.add_tween("devpi_server.views.tween_request_logging") pyramid_config.add_tween( "devpi_server.views.tween_keyfs_transaction", under="devpi_server.views.tween_request_logging") if self.config.args.profile_requests: pyramid_config.add_tween("devpi_server.main.tween_request_profiling") pyramid_config.add_request_method(get_remote_ip) pyramid_config.add_request_method(stage_url) pyramid_config.add_request_method(simpleindex_url) pyramid_config.add_request_method(apifatal) # overwrite route_url method with our own pyramid_config.add_request_method(route_url) # XXX end hack pyramid_config.scan() app = pyramid_config.make_wsgi_app() if self.is_replica(): from devpi_server.replica import ReplicaThread, register_key_subscribers search_path = self.config.args.replica_file_search_path if search_path and not os.path.exists(search_path): fatal( "search path for existing replica files doesn't " "exist: %s" % search_path) register_key_subscribers(self) self.replica_thread = ReplicaThread(self) # the replica thread replays keyfs changes # and project-specific changes are discovered # and replayed through the PypiProjectChange event if not self.config.requests_only: self.thread_pool.register(self.replica_thread) return OutsideURLMiddleware(app, self)
def create_app(self): from devpi_server.view_auth import DevpiAuthenticationPolicy from devpi_server.views import ContentTypePredicate from devpi_server.views import OutsideURLMiddleware from devpi_server.views import route_url, INSTALLER_USER_AGENT from pkg_resources import get_distribution from pyramid.authorization import ACLAuthorizationPolicy from pyramid.config import Configurator log = self.log log.debug("creating application in process %s", os.getpid()) pyramid_config = Configurator( root_factory='devpi_server.view_auth.RootFactory') pyramid_config.set_authentication_policy( DevpiAuthenticationPolicy(self)) pyramid_config.set_authorization_policy(ACLAuthorizationPolicy()) version_info = [("devpi-server", get_distribution("devpi_server").version)] for plug, distinfo in self.config.pluginmanager.list_plugin_distinfo(): threadlog.info( "Found plugin %s-%s (%s)." % (distinfo.project_name, distinfo.version, distinfo.location)) key = (distinfo.project_name, distinfo.version) if key not in version_info: version_info.append(key) version_info.sort() pyramid_config.registry['devpi_version_info'] = version_info self.config.hook.devpiserver_pyramid_configure( config=self.config, pyramid_config=pyramid_config) pyramid_config.add_view_predicate('content_type', ContentTypePredicate) pyramid_config.add_route("/+changelog/{serial}", "/+changelog/{serial}") pyramid_config.add_route("/+status", "/+status") pyramid_config.add_route("/+api", "/+api", accept="application/json") pyramid_config.add_route("{path:.*}/+api", "{path:.*}/+api", accept="application/json") pyramid_config.add_route("/+login", "/+login", accept="application/json") pyramid_config.add_route("/{user}/{index}/+e/{relpath:.*}", "/{user}/{index}/+e/{relpath:.*}") pyramid_config.add_route("/{user}/{index}/+f/{relpath:.*}", "/{user}/{index}/+f/{relpath:.*}") pyramid_config.add_route("/{user}/{index}/+simple", "/{user}/{index}/+simple") pyramid_config.add_route("/{user}/{index}/+simple/", "/{user}/{index}/+simple/") pyramid_config.add_route("/{user}/{index}/+simple/{project}", "/{user}/{index}/+simple/{project}") pyramid_config.add_route("/{user}/{index}/+simple/{project}/", "/{user}/{index}/+simple/{project}/") pyramid_config.add_route("/{user}/{index}/+simple/{project}/refresh", "/{user}/{index}/+simple/{project}/refresh") pyramid_config.add_route( "/{user}/{index}/{project}/{version}", "/{user}/{index}/{project}/{version:[^/]+/?}") pyramid_config.add_route( "simple_redirect", "/{user}/{index}/{project:[^/]+/?}", header="User-Agent:" + INSTALLER_USER_AGENT, accept="text/html", ) pyramid_config.add_route("/{user}/{index}/{project}", "/{user}/{index}/{project:[^/]+/?}") pyramid_config.add_route("/{user}/{index}/", "/{user}/{index}/") pyramid_config.add_route("/{user}/{index}", "/{user}/{index}") pyramid_config.add_route("/{user}", "/{user}") pyramid_config.add_route("/", "/") # register tweens for logging, transaction and replication pyramid_config.add_tween("devpi_server.views.tween_request_logging") if self.is_replica(): pyramid_config.add_tween( "devpi_server.replica.tween_replica_proxy", over="devpi_server.views.tween_keyfs_transaction", under="devpi_server.views.tween_request_logging", ) pyramid_config.add_tween( "devpi_server.views.tween_keyfs_transaction", under="devpi_server.views.tween_request_logging") if self.config.args.profile_requests: pyramid_config.add_tween( "devpi_server.main.tween_request_profiling") pyramid_config.add_request_method(get_remote_ip) pyramid_config.add_request_method(stage_url) pyramid_config.add_request_method(simpleindex_url) # overwrite route_url method with our own pyramid_config.add_request_method(route_url) # XXX end hack pyramid_config.scan() pyramid_config.registry['xom'] = self app = pyramid_config.make_wsgi_app() if self.is_replica(): from devpi_server.replica import ReplicaThread, register_key_subscribers register_key_subscribers(self) self.replica_thread = ReplicaThread(self) # the replica thread replays keyfs changes # and project-specific changes are discovered # and replayed through the PypiProjectChange event if not self.config.args.requests_only: self.thread_pool.register(self.replica_thread) return OutsideURLMiddleware(app, self)
def create_app(self): from devpi_server.view_auth import DevpiAuthenticationPolicy from devpi_server.views import route_url from pyramid.authorization import ACLAuthorizationPolicy from pyramid.config import Configurator log = self.log log.debug("creating application in process %s", os.getpid()) pyramid_config = Configurator( root_factory='devpi_server.view_auth.RootFactory') pyramid_config.set_authentication_policy( DevpiAuthenticationPolicy(self)) pyramid_config.set_authorization_policy(ACLAuthorizationPolicy()) self.config.hook.devpiserver_pyramid_configure( config=self.config, pyramid_config=pyramid_config) pyramid_config.add_route("/+changelog/{serial}", "/+changelog/{serial}") pyramid_config.add_route("/root/pypi/+name2serials", "/root/pypi/+name2serials") pyramid_config.add_route("/+api", "/+api", accept="application/json") pyramid_config.add_route("{path:.*}/+api", "{path:.*}/+api", accept="application/json") pyramid_config.add_route("/+login", "/+login", accept="application/json") pyramid_config.add_route("/{user}/{index}/+e/{relpath:.*}", "/{user}/{index}/+e/{relpath:.*}") pyramid_config.add_route("/{user}/{index}/+f/{relpath:.*}", "/{user}/{index}/+f/{relpath:.*}") pyramid_config.add_route("/{user}/{index}/+simple/", "/{user}/{index}/+simple/") pyramid_config.add_route("/{user}/{index}/+simple/{name}", "/{user}/{index}/+simple/{name:[^/]+/?}") pyramid_config.add_route("/{user}/{index}/+simple/{name}/refresh", "/{user}/{index}/+simple/{name}/refresh") pyramid_config.add_route("/{user}/{index}/{name}/{version}", "/{user}/{index}/{name}/{version:[^/]+/?}") pyramid_config.add_route( "simple_redirect", "/{user}/{index}/{name:[^/]+/?}", header="User-Agent:(distribute|setuptools|pip)/.*", accept="text/html", ) pyramid_config.add_route("/{user}/{index}/{name}", "/{user}/{index}/{name:[^/]+/?}") pyramid_config.add_route("/{user}/{index}/", "/{user}/{index}/") pyramid_config.add_route("/{user}/{index}", "/{user}/{index}") pyramid_config.add_route("/{user}", "/{user}") pyramid_config.add_route("/", "/") # register tweens for logging, transaction and replication pyramid_config.add_tween("devpi_server.views.tween_request_logging") if self.is_replica(): pyramid_config.add_tween( "devpi_server.replica.tween_replica_proxy", over="devpi_server.views.tween_keyfs_transaction", under="devpi_server.views.tween_request_logging", ) pyramid_config.add_tween( "devpi_server.views.tween_keyfs_transaction", under="devpi_server.views.tween_request_logging") # overwrite route_url method with our own pyramid_config.add_request_method(route_url) # XXX end hack pyramid_config.scan() pyramid_config.registry['xom'] = self app = pyramid_config.make_wsgi_app() if self.is_replica(): from devpi_server.replica import ReplicaThread replica_thread = ReplicaThread(self) # the replica thread replays keyfs changes # and pypimirror.name2serials changes are discovered # and replayed through the PypiProjectChange event self.thread_pool.register(replica_thread) else: # the master thread directly syncs using the # pypi changelog protocol self.thread_pool.register(self.pypimirror, dict(proxy=self.proxy)) return app