def __init__(self, urls, controllers, config=None,
auth_class=Authentication):
super(StorageServerApp, self).__init__(urls, controllers, config,
auth_class)
self.config = config
# Collecting the host-specific config and building connectors.
self.storages = {'default': get_storage(config)}
hostnames = set()
host_token = 'host:'
for cfgkey in config:
if cfgkey.startswith(host_token):
# Get the hostname from the config key. This assumes
# that host-specific keys have two trailing components
# that specify the setting to override.
# E.g: "host:localhost.storage.sqluri" => "localhost"
hostname = cfgkey[len(host_token):].rsplit(".", 2)[0]
hostnames.add(hostname)
for hostname in hostnames:
host_cfg = self._host_specific(hostname, config)
self.storages[hostname] = get_storage(host_cfg)
self.check_blacklist = \
self.config.get('storage.check_blacklisted_nodes', False)
if self.check_blacklist and Client is not None:
servers = self.config.get('storage.cache_servers',
'127.0.0.1:11211')
self.cache = Client(servers.split(','))
else:
if self.check_blacklist:
raise ValueError('The "check_blacklisted_node" option '
'needs a memcached server')
self.cache = None
def __init__(self, urls, controllers, config=None,
auth_class=WhoAuthentication):
super(StorageServerApp, self).__init__(urls, controllers, config,
auth_class)
self.config = config
# Collecting the host-specific config and building connectors.
self.storages = {'default': get_storage(config)}
hostnames = set()
host_token = 'host:'
for cfgkey in config:
if cfgkey.startswith(host_token):
# Get the hostname from the config key. This assumes
# that host-specific keys have two trailing components
# that specify the setting to override.
# E.g: "host:localhost.storage.sqluri" => "localhost"
hostname = cfgkey[len(host_token):].rsplit(".", 2)[0]
hostnames.add(hostname)
for hostname in hostnames:
host_cfg = self._host_specific(hostname, config)
self.storages[hostname] = get_storage(host_cfg)
# If we need to check node status, then we need to
# obtain a memcache client object.
self.cache = None
self.check_node_status = \
self.config.get('storage.check_node_status', False)
if self.check_node_status:
if Client is None:
raise ValueError('The "check_node_status" option '
'needs a memcached server')
servers = self.config.get('storage.cache_servers',
'127.0.0.1:11211')
self.cache = Client(servers.split(','))
# Config knobs for managing staged rollout of EOL headers.
self.eol_header_value = json.dumps({
'code': self.config.get('storage.eol_header_code', 'soft-eol'),
'message': self.config.get('storage.eol_header_message', ''),
'url': self.config.get('storage.eol_header_url', ''),
})
self.eol_general_release = \
self.config.get('storage.eol_general_release', False)
self.eol_rollout_percent = \
int(self.config.get('storage.eol_rollout_percent', 0))
self.eol_rollout_percent_hwm = \
int(self.config.get('storage.eol_rollout_percent_hwm', 0))
if not 0 <= self.eol_rollout_percent <= 100:
raise ValueError('storage.eol_rollout_percent should be ' \
'between 0 and 100')
if not 0 <= self.eol_rollout_percent_hwm <= 100:
raise ValueError('storage.eol_rollout_percent_hwm should be ' \
'between 0 and 100')
if self.eol_rollout_percent_hwm > 0:
if not self.eol_rollout_percent_hwm >= self.eol_rollout_percent:
raise ValueError('storage.eol_rollout_percent_hwm should be ' \
'greater than storage.eol_rollout_percent')
def test_host_specific_config(self):
req = self.make_request(environ={"HTTP_HOST": "localhost"})
sqluri = get_storage(req).sqluri
self.assertEquals(sqluri, "sqlite:///:memory:")
req = self.make_request(environ={"HTTP_HOST": "some-test-host"})
sqluri = get_storage(req).sqluri
self.assertTrue(sqluri.startswith("sqlite:////tmp/some-test-host-"))
req = self.make_request(environ={"HTTP_HOST": "some-test-host:8000"})
sqluri = get_storage(req).sqluri
self.assertTrue(sqluri.startswith("sqlite:////tmp/some-test-host-"))
req = self.make_request(environ={"HTTP_HOST": "another-test-host"})
sqluri = get_storage(req).sqluri
self.assertTrue(sqluri.startswith("sqlite:////tmp/another-test-host-"))
def test_host_specific_config(self):
req = self.make_request(environ={"HTTP_HOST": "localhost"})
sqluri = get_storage(req).sqluri
self.assertEquals(sqluri, "sqlite:///:memory:")
req = self.make_request(environ={"HTTP_HOST": "some-test-host"})
sqluri = get_storage(req).sqluri
self.assertTrue(sqluri.startswith("sqlite:////tmp/some-test-host-"))
req = self.make_request(environ={"HTTP_HOST": "some-test-host:8000"})
sqluri = get_storage(req).sqluri
self.assertTrue(sqluri.startswith("sqlite:////tmp/some-test-host-"))
req = self.make_request(environ={"HTTP_HOST": "another-test-host"})
sqluri = get_storage(req).sqluri
self.assertTrue(sqluri.startswith("sqlite:////tmp/another-test-host-"))
def test_503s_for_migrating_users(self):
user = {
"uid": 42,
"fxa_uid": "foobar",
"fxa_kid": "001122",
"hashed_fxa_uid": "aabbcc",
}
app = self._make_test_app(user)
req = self._make_signed_req(user["uid"], user)
# There's no public API for marking a user as migrated.
storage = get_storage(req)
with storage.dbconnector.connect() as connection:
connection.execute("""
INSERT INTO migration (fxa_uid, started_at, state)
VALUES (:fxa_uid, :started_at, :state)
/* queryName=migrate */
""", params={
"fxa_uid": "foobar",
"started_at": 12345,
"state": MigrationState.IN_PROGRESS,
})
res = app.get("/1.5/42/info/collections", status=503)
self.assertTrue("Retry-After" in res.headers)
def make_request(self, *args, **kwds):
req = super(TestMetlog, self).make_request(*args, **kwds)
req.user = {"uid": "aa"}
req.matchdict = {}
cornice.wrap_request(NewRequest(req))
req.validated["storage"] = get_storage(req)
req.validated["userid"] = req.user["uid"]
return req
def make_request(self, *args, **kwds):
req = super(TestMetlog, self).make_request(*args, **kwds)
req.user = {'uid': 'aa'}
req.matchdict = {}
cornice.wrap_request(NewRequest(req))
req.validated["storage"] = get_storage(req)
req.validated["userid"] = req.user["uid"]
return req
def extract_target_resource(request):
"""Validator to extract the target resource of a request.
This validator will extract the userid, collection name and item id if
they appear in the matched URL of the request. It assumes they have
already been checked for validity by the authentication and url-matching
logic of the application.
It also looks up the appropriate storage backend based on the hostname
in the request.
It places these items under the keys "storage", "userid", "collection"
and "item".
"""
request.validated["storage"] = get_storage(request)
request.validated["userid"] = int(request.matchdict["userid"])
if "collection" in request.matchdict:
request.validated["collection"] = request.matchdict["collection"]
if "item" in request.matchdict:
request.validated["item"] = request.matchdict["item"]
def extract_target_resource(request):
"""Validator to extract the target resource of a request.
This validator will extract the userid, collection name and item id if
they appear in the matched URL of the request. It assumes they have
already been checked for validity by the authentication and url-matching
logic of the application.
It also looks up the appropriate storage backend based on the hostname
in the request.
It places these items under the keys "storage", "userid", "collection"
and "item".
"""
request.validated["storage"] = get_storage(request)
request.validated["userid"] = int(request.matchdict["userid"])
if "collection" in request.matchdict:
request.validated["collection"] = request.matchdict["collection"]
if "item" in request.matchdict:
request.validated["item"] = request.matchdict["item"]
