def test_initial_master_connection(self, plugin, makexom, monkeypatch):
from devpi_server.replica import ReplicaThread
import time
now = time.time()
xom = makexom(["--master=http://localhost"])
request = self._xomrequest(xom)
xom.replica_thread = ReplicaThread(xom)
assert xom.is_replica()
assert xom.replica_thread.started_at is None
# fake replica start
xom.replica_thread.started_at = now
# no report in the first minute
result = plugin(request)
assert result == []
# warning after one minute
monkeypatch.setattr(devpi_server.views, "time", lambda: now + 70)
result = plugin(request)
assert result == [
dict(status='warn',
msg='No contact to master for more than 1 minute')
]
# fatal after five minutes
monkeypatch.setattr(devpi_server.views, "time", lambda: now + 310)
result = plugin(request)
assert result == [
dict(status='fatal',
msg='No contact to master for more than 5 minutes')
]
def test_replica_lagging(self, plugin, makexom, monkeypatch):
from devpi_server.replica import ReplicaThread
import time
now = time.time()
xom = makexom(["--master=http://localhost"])
request = self._xomrequest(xom)
xom.replica_thread = ReplicaThread(xom)
assert xom.is_replica()
# fake first serial processed
xom.replica_thread.update_master_serial(0)
xom.replica_thread.replica_in_sync_at = now
# no report in the first minute
result = plugin(request)
assert result == []
# warning after one minute
monkeypatch.setattr(devpi_server.views, "time", lambda: now + 70)
result = plugin(request)
assert result == [
dict(status='warn',
msg='Replica is behind master for more than 1 minute')
]
# fatal after five minutes
monkeypatch.setattr(devpi_server.views, "time", lambda: now + 310)
result = plugin(request)
assert result == [
dict(status='fatal',
msg='Replica is behind master for more than 5 minutes')
]
def makexom(opts=(), httpget=httpget, plugins=()):
from devpi_server import auth_basic
from devpi_server import auth_devpi
plugins = [
plugin[0] if isinstance(plugin, tuple) else plugin
for plugin in plugins
]
for plugin in [auth_basic, auth_devpi, storage_info["_test_plugin"]]:
if plugin not in plugins:
plugins.append(plugin)
pm = get_pluginmanager(load_entrypoints=False)
for plugin in plugins:
pm.register(plugin)
serverdir = gentmp()
fullopts = ["devpi-server", "--serverdir", serverdir] + list(opts)
if request.node.get_marker("with_replica_thread"):
fullopts.append("--master=http://localhost")
if not request.node.get_marker("no_storage_option"):
if storage_info["name"] != "sqlite":
fullopts.append("--storage=%s" % storage_info["name"])
fullopts = [str(x) for x in fullopts]
config = parseoptions(pm, fullopts)
config.init_nodeinfo()
for marker in ("storage_with_filesystem", ):
if request.node.get_marker(marker):
info = config._storage_info()
markers = info.get("_test_markers", [])
if marker not in markers:
pytest.skip("The storage doesn't have marker '%s'." %
marker)
if not request.node.get_marker("no_storage_option"):
assert storage_info["storage"] is config.storage
if request.node.get_marker("nomocking"):
xom = XOM(config)
else:
xom = XOM(config, httpget=httpget)
if not request.node.get_marker("nomockprojectsremote"):
monkeypatch.setattr(extpypi.PyPIStage, "_get_remote_projects",
lambda self: set())
add_pypistage_mocks(monkeypatch, httpget)
# initialize default indexes
from devpi_server.main import set_default_indexes
if not xom.config.args.master_url:
with xom.keyfs.transaction(write=True):
set_default_indexes(xom.model)
if request.node.get_marker("with_replica_thread"):
from devpi_server.replica import ReplicaThread
rt = ReplicaThread(xom)
xom.replica_thread = rt
xom.thread_pool.register(rt)
xom.thread_pool.start_one(rt)
if request.node.get_marker("start_threads"):
xom.thread_pool.start()
elif request.node.get_marker("with_notifier"):
xom.thread_pool.start_one(xom.keyfs.notifier)
request.addfinalizer(xom.thread_pool.shutdown)
return xom
def replica_mapp(makemapp, master_host_port):
from devpi_server.replica import ReplicaThread
app = makemapp(options=['--master', 'http://%s:%s' % master_host_port])
rt = ReplicaThread(app.xom)
app.xom.replica_thread = rt
app.xom.thread_pool.register(rt)
try:
yield app
finally:
app.xom.thread_pool.shutdown()
def __init__(self, config, httpget=None):
self.config = config
self.thread_pool = mythread.ThreadPool()
if httpget is not None:
self.httpget = httpget
self.log = threadlog
self.polling_replicas = {}
self._stagecache = {}
if self.is_replica():
from devpi_server.replica import ReplicaThread
from devpi_server.replica import register_key_subscribers
search_path = self.config.replica_file_search_path
if search_path and not os.path.exists(search_path):
fatal("search path for existing replica files doesn't "
"exist: %s" % search_path)
register_key_subscribers(self)
# the replica thread replays keyfs changes
# and project-specific changes are discovered
# and replayed through the PypiProjectChange event
if not self.config.requests_only:
self.replica_thread = ReplicaThread(self)
self.thread_pool.register(self.replica_thread)
def rt(self, makexom):
xom = makexom(["--master=http://localhost"])
rt = ReplicaThread(xom)
xom.thread_pool.register(rt)
return rt
def create_app(self):
from devpi_server.view_auth import DevpiAuthenticationPolicy
from devpi_server.views import ContentTypePredicate
from devpi_server.views import OutsideURLMiddleware
from devpi_server.views import route_url, INSTALLER_USER_AGENT
from pkg_resources import get_distribution
from pyramid.authorization import ACLAuthorizationPolicy
from pyramid.config import Configurator
from pyramid.viewderivers import INGRESS
log = self.log
log.debug("creating application in process %s", os.getpid())
pyramid_config = Configurator(root_factory='devpi_server.view_auth.RootFactory')
pyramid_config.set_authentication_policy(DevpiAuthenticationPolicy(self))
pyramid_config.set_authorization_policy(ACLAuthorizationPolicy())
version_info = [
("devpi-server", get_distribution("devpi_server").version)]
for plug, distinfo in self.config.pluginmanager.list_plugin_distinfo():
threadlog.info("Found plugin %s-%s (%s)." % (
distinfo.project_name, distinfo.version, distinfo.location))
key = (distinfo.project_name, distinfo.version)
if key not in version_info:
version_info.append(key)
version_info.sort()
pyramid_config.registry['devpi_version_info'] = version_info
pyramid_config.registry['xom'] = self
index_classes = {}
customizer_classes = sum(
self.config.hook.devpiserver_get_stage_customizer_classes(),
[])
for ixtype, ixclass in customizer_classes:
index_classes.setdefault(ixtype, []).append(ixclass)
for ixtype, ixclasses in index_classes.items():
if len(ixclasses) > 1:
fatal(
"multiple implementation classes for index type '%s':\n%s"
% (
ixtype,
"\n".join(
"%s.%s" % (x.__module__, x.__name__)
for x in ixclasses)))
self.config.hook.devpiserver_pyramid_configure(
config=self.config,
pyramid_config=pyramid_config)
pyramid_config.add_view_deriver(self.view_deriver, under=INGRESS)
pyramid_config.add_view_predicate('content_type', ContentTypePredicate)
pyramid_config.add_route("/+changelog/{serial}",
r"/+changelog/{serial:\d+}")
pyramid_config.add_route("/+changelog/{serial}-",
r"/+changelog/{serial:\d+}-")
pyramid_config.add_route("/+status", "/+status")
pyramid_config.add_route("/+api", "/+api", accept="application/json")
pyramid_config.add_route("{path:.*}/+api", "{path:.*}/+api", accept="application/json")
pyramid_config.add_route("/+login", "/+login", accept="application/json")
pyramid_config.add_route("/{user}/{index}/+e/{relpath:.*}", "/{user}/{index}/+e/{relpath:.*}")
pyramid_config.add_route("/{user}/{index}/+f/{relpath:.*}", "/{user}/{index}/+f/{relpath:.*}")
pyramid_config.add_route("/{user}/{index}/+simple", "/{user}/{index}/+simple")
pyramid_config.add_route("/{user}/{index}/+simple/", "/{user}/{index}/+simple/")
pyramid_config.add_route("/{user}/{index}/+simple/{project}",
"/{user}/{index}/+simple/{project}")
pyramid_config.add_route("/{user}/{index}/+simple/{project}/",
"/{user}/{index}/+simple/{project}/")
pyramid_config.add_route("/{user}/{index}/+simple/{project}/refresh",
"/{user}/{index}/+simple/{project}/refresh")
pyramid_config.add_route("/{user}/{index}/{project}/{version}",
"/{user}/{index}/{project}/{version:[^/]+/?}")
pyramid_config.add_route(
"simple_redirect", "/{user}/{index}/{project:[^/]+/?}",
header="User-Agent:" + INSTALLER_USER_AGENT,
accept="text/html",
)
pyramid_config.add_route("/{user}/{index}/{project}",
"/{user}/{index}/{project:[^/]+/?}")
pyramid_config.add_route("/{user}/{index}/", "/{user}/{index}/")
pyramid_config.add_route("/{user}/{index}", "/{user}/{index}")
pyramid_config.add_route("/{user}", "/{user}")
pyramid_config.add_route("/", "/")
# register tweens for logging, transaction and replication
pyramid_config.add_tween("devpi_server.views.tween_request_logging")
pyramid_config.add_tween(
"devpi_server.views.tween_keyfs_transaction",
under="devpi_server.views.tween_request_logging")
if self.config.args.profile_requests:
pyramid_config.add_tween("devpi_server.main.tween_request_profiling")
pyramid_config.add_request_method(get_remote_ip)
pyramid_config.add_request_method(stage_url)
pyramid_config.add_request_method(simpleindex_url)
pyramid_config.add_request_method(apifatal)
# overwrite route_url method with our own
pyramid_config.add_request_method(route_url)
# XXX end hack
pyramid_config.scan()
app = pyramid_config.make_wsgi_app()
if self.is_replica():
from devpi_server.replica import ReplicaThread, register_key_subscribers
search_path = self.config.args.replica_file_search_path
if search_path and not os.path.exists(search_path):
fatal(
"search path for existing replica files doesn't "
"exist: %s" % search_path)
register_key_subscribers(self)
self.replica_thread = ReplicaThread(self)
# the replica thread replays keyfs changes
# and project-specific changes are discovered
# and replayed through the PypiProjectChange event
if not self.config.requests_only:
self.thread_pool.register(self.replica_thread)
return OutsideURLMiddleware(app, self)
def create_app(self):
from devpi_server.view_auth import DevpiAuthenticationPolicy
from devpi_server.views import ContentTypePredicate
from devpi_server.views import OutsideURLMiddleware
from devpi_server.views import route_url, INSTALLER_USER_AGENT
from pkg_resources import get_distribution
from pyramid.authorization import ACLAuthorizationPolicy
from pyramid.config import Configurator
log = self.log
log.debug("creating application in process %s", os.getpid())
pyramid_config = Configurator(
root_factory='devpi_server.view_auth.RootFactory')
pyramid_config.set_authentication_policy(
DevpiAuthenticationPolicy(self))
pyramid_config.set_authorization_policy(ACLAuthorizationPolicy())
version_info = [("devpi-server",
get_distribution("devpi_server").version)]
for plug, distinfo in self.config.pluginmanager.list_plugin_distinfo():
threadlog.info(
"Found plugin %s-%s (%s)." %
(distinfo.project_name, distinfo.version, distinfo.location))
key = (distinfo.project_name, distinfo.version)
if key not in version_info:
version_info.append(key)
version_info.sort()
pyramid_config.registry['devpi_version_info'] = version_info
self.config.hook.devpiserver_pyramid_configure(
config=self.config, pyramid_config=pyramid_config)
pyramid_config.add_view_predicate('content_type', ContentTypePredicate)
pyramid_config.add_route("/+changelog/{serial}",
"/+changelog/{serial}")
pyramid_config.add_route("/+status", "/+status")
pyramid_config.add_route("/+api", "/+api", accept="application/json")
pyramid_config.add_route("{path:.*}/+api",
"{path:.*}/+api",
accept="application/json")
pyramid_config.add_route("/+login",
"/+login",
accept="application/json")
pyramid_config.add_route("/{user}/{index}/+e/{relpath:.*}",
"/{user}/{index}/+e/{relpath:.*}")
pyramid_config.add_route("/{user}/{index}/+f/{relpath:.*}",
"/{user}/{index}/+f/{relpath:.*}")
pyramid_config.add_route("/{user}/{index}/+simple",
"/{user}/{index}/+simple")
pyramid_config.add_route("/{user}/{index}/+simple/",
"/{user}/{index}/+simple/")
pyramid_config.add_route("/{user}/{index}/+simple/{project}",
"/{user}/{index}/+simple/{project}")
pyramid_config.add_route("/{user}/{index}/+simple/{project}/",
"/{user}/{index}/+simple/{project}/")
pyramid_config.add_route("/{user}/{index}/+simple/{project}/refresh",
"/{user}/{index}/+simple/{project}/refresh")
pyramid_config.add_route(
"/{user}/{index}/{project}/{version}",
"/{user}/{index}/{project}/{version:[^/]+/?}")
pyramid_config.add_route(
"simple_redirect",
"/{user}/{index}/{project:[^/]+/?}",
header="User-Agent:" + INSTALLER_USER_AGENT,
accept="text/html",
)
pyramid_config.add_route("/{user}/{index}/{project}",
"/{user}/{index}/{project:[^/]+/?}")
pyramid_config.add_route("/{user}/{index}/", "/{user}/{index}/")
pyramid_config.add_route("/{user}/{index}", "/{user}/{index}")
pyramid_config.add_route("/{user}", "/{user}")
pyramid_config.add_route("/", "/")
# register tweens for logging, transaction and replication
pyramid_config.add_tween("devpi_server.views.tween_request_logging")
if self.is_replica():
pyramid_config.add_tween(
"devpi_server.replica.tween_replica_proxy",
over="devpi_server.views.tween_keyfs_transaction",
under="devpi_server.views.tween_request_logging",
)
pyramid_config.add_tween(
"devpi_server.views.tween_keyfs_transaction",
under="devpi_server.views.tween_request_logging")
if self.config.args.profile_requests:
pyramid_config.add_tween(
"devpi_server.main.tween_request_profiling")
pyramid_config.add_request_method(get_remote_ip)
pyramid_config.add_request_method(stage_url)
pyramid_config.add_request_method(simpleindex_url)
# overwrite route_url method with our own
pyramid_config.add_request_method(route_url)
# XXX end hack
pyramid_config.scan()
pyramid_config.registry['xom'] = self
app = pyramid_config.make_wsgi_app()
if self.is_replica():
from devpi_server.replica import ReplicaThread, register_key_subscribers
register_key_subscribers(self)
self.replica_thread = ReplicaThread(self)
# the replica thread replays keyfs changes
# and project-specific changes are discovered
# and replayed through the PypiProjectChange event
if not self.config.args.requests_only:
self.thread_pool.register(self.replica_thread)
return OutsideURLMiddleware(app, self)
def create_app(self):
from devpi_server.view_auth import DevpiAuthenticationPolicy
from devpi_server.views import route_url
from pyramid.authorization import ACLAuthorizationPolicy
from pyramid.config import Configurator
log = self.log
log.debug("creating application in process %s", os.getpid())
pyramid_config = Configurator(
root_factory='devpi_server.view_auth.RootFactory')
pyramid_config.set_authentication_policy(
DevpiAuthenticationPolicy(self))
pyramid_config.set_authorization_policy(ACLAuthorizationPolicy())
self.config.hook.devpiserver_pyramid_configure(
config=self.config, pyramid_config=pyramid_config)
pyramid_config.add_route("/+changelog/{serial}",
"/+changelog/{serial}")
pyramid_config.add_route("/root/pypi/+name2serials",
"/root/pypi/+name2serials")
pyramid_config.add_route("/+api", "/+api", accept="application/json")
pyramid_config.add_route("{path:.*}/+api",
"{path:.*}/+api",
accept="application/json")
pyramid_config.add_route("/+login",
"/+login",
accept="application/json")
pyramid_config.add_route("/{user}/{index}/+e/{relpath:.*}",
"/{user}/{index}/+e/{relpath:.*}")
pyramid_config.add_route("/{user}/{index}/+f/{relpath:.*}",
"/{user}/{index}/+f/{relpath:.*}")
pyramid_config.add_route("/{user}/{index}/+simple/",
"/{user}/{index}/+simple/")
pyramid_config.add_route("/{user}/{index}/+simple/{name}",
"/{user}/{index}/+simple/{name:[^/]+/?}")
pyramid_config.add_route("/{user}/{index}/+simple/{name}/refresh",
"/{user}/{index}/+simple/{name}/refresh")
pyramid_config.add_route("/{user}/{index}/{name}/{version}",
"/{user}/{index}/{name}/{version:[^/]+/?}")
pyramid_config.add_route(
"simple_redirect",
"/{user}/{index}/{name:[^/]+/?}",
header="User-Agent:(distribute|setuptools|pip)/.*",
accept="text/html",
)
pyramid_config.add_route("/{user}/{index}/{name}",
"/{user}/{index}/{name:[^/]+/?}")
pyramid_config.add_route("/{user}/{index}/", "/{user}/{index}/")
pyramid_config.add_route("/{user}/{index}", "/{user}/{index}")
pyramid_config.add_route("/{user}", "/{user}")
pyramid_config.add_route("/", "/")
# register tweens for logging, transaction and replication
pyramid_config.add_tween("devpi_server.views.tween_request_logging")
if self.is_replica():
pyramid_config.add_tween(
"devpi_server.replica.tween_replica_proxy",
over="devpi_server.views.tween_keyfs_transaction",
under="devpi_server.views.tween_request_logging",
)
pyramid_config.add_tween(
"devpi_server.views.tween_keyfs_transaction",
under="devpi_server.views.tween_request_logging")
# overwrite route_url method with our own
pyramid_config.add_request_method(route_url)
# XXX end hack
pyramid_config.scan()
pyramid_config.registry['xom'] = self
app = pyramid_config.make_wsgi_app()
if self.is_replica():
from devpi_server.replica import ReplicaThread
replica_thread = ReplicaThread(self)
# the replica thread replays keyfs changes
# and pypimirror.name2serials changes are discovered
# and replayed through the PypiProjectChange event
self.thread_pool.register(replica_thread)
else:
# the master thread directly syncs using the
# pypi changelog protocol
self.thread_pool.register(self.pypimirror, dict(proxy=self.proxy))
return app
