class TestDirectoryAPI(BaseTestCase):
def setUp(self):
super(TestDirectoryAPI, self).setUp()
self.api = DirectoryAPI(self.ns, self.uri)
def _create(self, name, metadata=None):
return self.api.create(self.account, name, metadata=metadata)
def _delete(self, name):
self.api.delete(self.account, name)
def _clean(self, name, clear=False):
if clear:
# must clean properties before
self.api.del_properties(self.account, name, [])
self._delete(name)
def _get_properties(self, name, properties=None):
return self.api.get_properties(
self.account, name, properties=properties)
def _set_properties(self, name, properties=None):
return self.api.set_properties(
self.account, name, properties=properties)
def test_get(self):
# get on unknown reference
name = random_str(32)
self.assertRaises(exc.NotFound, self.api.get, self.account, name)
self._create(name)
# get on existing reference
res = self.api.get(self.account, name)
self.assertIsNot(res['dir'], None)
self.assertIsNot(res['srv'], None)
self._delete(name)
# get on deleted reference
self.assertRaises(exc.NotFound, self.api.get, self.account, name)
def test_create(self):
name = random_str(32)
res = self._create(name)
self.assertEqual(res, True)
# second create
res = self._create(name)
self.assertEqual(res, False)
# clean
self._delete(name)
def test_create_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
res = self._create(name, metadata)
self.assertEqual(res, True)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
def test_delete(self):
name = random_str(32)
# delete on unknown reference
self.assertRaises(exc.NotFound, self.api.delete, self.account, name)
res = self._create(name)
self.assertEqual(res, True)
# delete on existing reference
self._delete(name)
# verify deleted
self.assertRaises(exc.NotFound, self.api.get, self.account, name)
# second delete
self.assertRaises(exc.NotFound, self.api.delete, self.account, name)
# verify deleted
self.assertRaises(exc.NotFound, self.api.get, self.account, name)
def test_get_properties(self):
name = random_str(32)
# get_properties on unknown reference
self.assertRaises(
exc.NotFound, self.api.get_properties, self.account, name)
res = self._create(name)
self.assertEqual(res, True)
# get_properties on existing reference
data = self.api.get_properties(self.account, name)
self.assertEqual(data['properties'], {})
# get_properties
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
self._set_properties(name, metadata)
data = self.api.get_properties(self.account, name)
self.assertEqual(data['properties'], metadata)
# get_properties specify key
key = metadata.keys().pop(0)
data = self.api.get_properties(self.account, name, [key])
self.assertEqual(data['properties'], {key: metadata[key]})
# clean
self._clean(name, True)
# get_properties on deleted reference
self.assertRaises(
exc.NotFound, self.api.get_properties, self.account, name)
def test_set_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
# set_properties on unknown reference
self.assertRaises(
exc.NotFound, self.api.set_properties, self.account, name,
metadata)
res = self._create(name)
self.assertEqual(res, True)
# set_properties on existing reference
self.api.set_properties(self.account, name, metadata)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# set_properties
key = random_str(32)
value = random_str(32)
metadata2 = {key: value}
self._set_properties(name, metadata2)
metadata.update(metadata2)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# set_properties overwrite key
key = metadata.keys().pop(0)
value = random_str(32)
metadata3 = {key: value}
metadata.update(metadata3)
self.api.set_properties(self.account, name, metadata3)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# set_properties on deleted reference
self.assertRaises(
exc.NotFound, self.api.set_properties, self.account, name,
metadata)
def test_del_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
# del_properties on unknown reference
self.assertRaises(
exc.NotFound, self.api.del_properties, self.account, name, [])
res = self._create(name, metadata)
self.assertEqual(res, True)
key = metadata.keys().pop()
del metadata[key]
# del_properties on existing reference
self.api.del_properties(self.account, name, [key])
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# del_properties on unknown key
key = random_str(32)
# We do not check if a property exists before deleting it
# self.assertRaises(
# exc.NotFound, self.api.del_properties, self.account, name,
# [key])
self.api.del_properties(self.account, name, [key])
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# del_properties on deleted reference
self.assertRaises(
exc.NotFound, self.api.set_properties, self.account, name,
metadata)
def test_list_services(self):
# list_services on unknown reference
name = random_str(32)
echo = 'echo'
self.assertRaises(
exc.NotFound, self.api.list_services, self.account, name, echo)
self._create(name)
# list_services on existing reference
res = self.api.list_services(self.account, name, echo)
self.assertIsNot(res['dir'], None)
self.assertIsNot(res['srv'], None)
self._delete(name)
# get on deleted reference
self.assertRaises(exc.NotFound, self.api.get, self.account, name)
def test_rdir_linking(self):
"""
Tests that rdir services linked to rawx services
are not on the same locations
"""
cs = ConscienceClient({'namespace': self.ns})
rawx_list = cs.all_services('rawx')
rdir_dict = {x['addr']: x for x in cs.all_services('rdir')}
# Link the services
for rawx in rawx_list:
self.api.link('_RDIR_TEST', rawx['addr'], 'rdir',
autocreate=True)
# Do the checks
for rawx in rawx_list:
linked_rdir = self.api.get(
'_RDIR_TEST', rawx['addr'], service_type='rdir')['srv']
rdir = rdir_dict[linked_rdir[0]['host']]
rawx_loc = rawx['tags'].get('tag.loc')
rdir_loc = rdir['tags'].get('tag.loc')
self.assertNotEqual(rawx_loc, rdir_loc)
# Unlink the services
for rawx in rawx_list:
self.api.unlink('_RDIR_TEST', rawx['addr'], 'rdir')
self.api.delete('_RDIR_TEST', rawx['addr'])
def test_rdir_repartition(self):
client = RdirClient({'namespace': self.ns})
all_rawx = client.assign_all_rawx()
by_rdir = dict()
total = 0
for rawx in all_rawx:
count = by_rdir.get(rawx['rdir']['addr'], 0)
total += 1
by_rdir[rawx['rdir']['addr']] = count + 1
avg = total / float(len(by_rdir))
print "Ideal number of bases per rdir: ", avg
print "Current repartition: ", by_rdir
for count in by_rdir.itervalues():
self.assertLessEqual(count, avg + 1)
class RdirClient(Client):
def __init__(self, conf, **kwargs):
super(RdirClient, self).__init__(conf, **kwargs)
self.directory = DirectoryAPI(self.ns, self.endpoint, **kwargs)
self._addr_cache = dict()
def assign_all_rawx(self):
"""
Find a rdir service for all rawx that don't have one already.
"""
cs = ConscienceClient(self.conf)
all_rawx = cs.all_services("rawx")
all_rdir = cs.all_services("rdir", True)
by_id = {_make_id(self.ns, "rdir", x["addr"]): x for x in all_rdir}
for rawx in all_rawx:
try:
# Verify that there is no rdir linked
resp = self.directory.get(RDIR_ACCT, rawx["addr"], service_type="rdir")
rawx["rdir"] = by_id[_make_id(self.ns, "rdir", self._lookup_rdir_host(resp))]
except (NotFound, ClientException):
rdir = self._smart_link_rdir(rawx["addr"], cs, all_rdir)
n_bases = by_id[rdir]["tags"].get("stat.opened_db_count", 0)
by_id[rdir]["tags"]["stat.opened_db_count"] = n_bases + 1
rawx["rdir"] = by_id[rdir]
return all_rawx
def _smart_link_rdir(self, volume_id, cs=None, all_rdir=None):
"""
Force the load balancer to avoid services that already host more
bases than the average while selecting rdir services.
"""
if not cs:
cs = ConscienceClient(self.conf)
if not all_rdir:
all_rdir = cs.all_services("rdir", True)
avail_base_count = [x["tags"]["stat.opened_db_count"] for x in all_rdir if x["score"] > 0]
mean = sum(avail_base_count) / float(len(avail_base_count))
avoids = [
_make_id(self.ns, "rdir", x["addr"])
for x in all_rdir
if x["score"] > 0 and x["tags"]["stat.opened_db_count"] > mean
]
known = [_make_id(self.ns, "rawx", volume_id)]
try:
polled = cs.poll("rdir", avoid=avoids, known=known)[0]
except ClientException as exc:
if exc.status != 481:
raise
# Retry without `avoids`, hoping the next iteration will rebalance
polled = cs.poll("rdir", known=known)[0]
forced = {"host": polled["addr"], "type": "rdir", "seq": 1, "args": "", "id": polled["id"]}
self.directory.force(RDIR_ACCT, volume_id, "rdir", forced, autocreate=True)
return polled["id"]
def _link_rdir(self, volume_id, smart=True):
if not smart:
self.directory.link(RDIR_ACCT, volume_id, "rdir", autocreate=True)
else:
self._smart_link_rdir(volume_id)
return self.directory.get(RDIR_ACCT, volume_id, service_type="rdir")
def _lookup_rdir_host(self, resp):
host = None
for srv in resp.get("srv", {}):
if srv["type"] == "rdir":
host = srv["host"]
if not host:
raise ClientException("No rdir service found")
return host
def _get_rdir_addr(self, volume_id, create=False, nocache=False):
if not nocache and volume_id in self._addr_cache:
return self._addr_cache[volume_id]
resp = {}
try:
resp = self.directory.get(RDIR_ACCT, volume_id, service_type="rdir")
except NotFound:
if not create:
raise VolumeException("No such volume %s" % volume_id)
try:
host = self._lookup_rdir_host(resp)
except ClientException:
# Reference exists but no rdir linked
if not create:
raise
resp = self._link_rdir(volume_id)
host = self._lookup_rdir_host(resp)
self._addr_cache[volume_id] = host
return host
def _make_uri(self, action, volume_id, create=False, nocache=False):
rdir_host = self._get_rdir_addr(volume_id, create=create, nocache=nocache)
uri = "http://%s/v1/%s" % (rdir_host, action)
return uri
def _rdir_request(self, volume, method, action, create=False, **kwargs):
params = {"vol": volume}
if create:
params["create"] = "1"
uri = self._make_uri(action, volume, create=create)
try:
resp, body = self._direct_request(method, uri, params=params, **kwargs)
except NotFound:
uri = self._make_uri(action, volume, create=create, nocache=True)
resp, body = self._direct_request(method, uri, params=params, **kwargs)
return resp, body
def chunk_push(self, volume_id, container_id, content_id, chunk_id, **data):
"""Reference a chunk in the reverse directory"""
body = {"container_id": container_id, "content_id": content_id, "chunk_id": chunk_id}
for key, value in data.iteritems():
body[key] = value
headers = {}
self._rdir_request(volume_id, "POST", "rdir/push", create=True, json=body, headers=headers)
def chunk_delete(self, volume_id, container_id, content_id, chunk_id):
"""Unreference a chunk from the reverse directory"""
body = {"container_id": container_id, "content_id": content_id, "chunk_id": chunk_id}
self._rdir_request(volume_id, "DELETE", "rdir/delete", json=body)
def chunk_fetch(self, volume, limit=100, rebuild=False):
"""Fetch the list of chunks belonging to the specified volume"""
req_body = {"limit": limit}
if rebuild:
req_body["rebuild"] = True
while True:
resp, resp_body = self._rdir_request(volume, "POST", "rdir/fetch", json=req_body)
resp.raise_for_status()
if len(resp_body) == 0:
break
for (key, value) in resp_body:
container, content, chunk = key.split("|")
yield container, content, chunk, value
req_body["start_after"] = key
def admin_incident_set(self, volume, date):
body = {"date": int(float(date))}
self._rdir_request(volume, "POST", "rdir/admin/incident", json=body)
def admin_incident_get(self, volume):
resp, resp_body = self._rdir_request(volume, "GET", "rdir/admin/incident")
return resp_body.get("date")
def admin_lock(self, volume, who):
body = {"who": who}
self._rdir_request(volume, "POST", "rdir/admin/lock", json=body)
def admin_unlock(self, volume):
self._rdir_request(volume, "POST", "rdir/admin/unlock")
def admin_show(self, volume):
resp, resp_body = self._rdir_request(volume, "GET", "rdir/admin/show")
return resp_body
def admin_clear(self, volume, clear_all=False):
body = {"all": clear_all}
resp, resp_body = self._rdir_request(volume, "POST", "rdir/admin/clear", json=body)
return resp_body
def status(self, volume):
resp, resp_body = self._rdir_request(volume, "GET", "rdir/status")
return resp_body