class TestDirectoryAPI(BaseTestCase): def setUp(self): super(TestDirectoryAPI, self).setUp() self.api = DirectoryAPI(self.ns, self.uri) def _create(self, name, metadata=None): return self.api.create(self.account, name, metadata=metadata) def _delete(self, name): self.api.delete(self.account, name) def _clean(self, name, clear=False): if clear: # must clean properties before self.api.del_properties(self.account, name, []) self._delete(name) def _get_properties(self, name, properties=None): return self.api.get_properties( self.account, name, properties=properties) def _set_properties(self, name, properties=None): return self.api.set_properties( self.account, name, properties=properties) def test_get(self): # get on unknown reference name = random_str(32) self.assertRaises(exc.NotFound, self.api.get, self.account, name) self._create(name) # get on existing reference res = self.api.get(self.account, name) self.assertIsNot(res['dir'], None) self.assertIsNot(res['srv'], None) self._delete(name) # get on deleted reference self.assertRaises(exc.NotFound, self.api.get, self.account, name) def test_create(self): name = random_str(32) res = self._create(name) self.assertEqual(res, True) # second create res = self._create(name) self.assertEqual(res, False) # clean self._delete(name) def test_create_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } res = self._create(name, metadata) self.assertEqual(res, True) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) def test_delete(self): name = random_str(32) # delete on unknown reference self.assertRaises(exc.NotFound, self.api.delete, self.account, name) res = self._create(name) self.assertEqual(res, True) # delete on existing reference self._delete(name) # verify deleted self.assertRaises(exc.NotFound, self.api.get, self.account, name) # second delete self.assertRaises(exc.NotFound, self.api.delete, self.account, name) # verify deleted self.assertRaises(exc.NotFound, self.api.get, self.account, name) def test_get_properties(self): name = random_str(32) # get_properties on unknown reference self.assertRaises( exc.NotFound, self.api.get_properties, self.account, name) res = self._create(name) self.assertEqual(res, True) # get_properties on existing reference data = self.api.get_properties(self.account, name) self.assertEqual(data['properties'], {}) # get_properties metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } self._set_properties(name, metadata) data = self.api.get_properties(self.account, name) self.assertEqual(data['properties'], metadata) # get_properties specify key key = metadata.keys().pop(0) data = self.api.get_properties(self.account, name, [key]) self.assertEqual(data['properties'], {key: metadata[key]}) # clean self._clean(name, True) # get_properties on deleted reference self.assertRaises( exc.NotFound, self.api.get_properties, self.account, name) def test_set_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } # set_properties on unknown reference self.assertRaises( exc.NotFound, self.api.set_properties, self.account, name, metadata) res = self._create(name) self.assertEqual(res, True) # set_properties on existing reference self.api.set_properties(self.account, name, metadata) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # set_properties key = random_str(32) value = random_str(32) metadata2 = {key: value} self._set_properties(name, metadata2) metadata.update(metadata2) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # set_properties overwrite key key = metadata.keys().pop(0) value = random_str(32) metadata3 = {key: value} metadata.update(metadata3) self.api.set_properties(self.account, name, metadata3) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # set_properties on deleted reference self.assertRaises( exc.NotFound, self.api.set_properties, self.account, name, metadata) def test_del_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } # del_properties on unknown reference self.assertRaises( exc.NotFound, self.api.del_properties, self.account, name, []) res = self._create(name, metadata) self.assertEqual(res, True) key = metadata.keys().pop() del metadata[key] # del_properties on existing reference self.api.del_properties(self.account, name, [key]) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # del_properties on unknown key key = random_str(32) # We do not check if a property exists before deleting it # self.assertRaises( # exc.NotFound, self.api.del_properties, self.account, name, # [key]) self.api.del_properties(self.account, name, [key]) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # del_properties on deleted reference self.assertRaises( exc.NotFound, self.api.set_properties, self.account, name, metadata) def test_list_services(self): # list_services on unknown reference name = random_str(32) echo = 'echo' self.assertRaises( exc.NotFound, self.api.list_services, self.account, name, echo) self._create(name) # list_services on existing reference res = self.api.list_services(self.account, name, echo) self.assertIsNot(res['dir'], None) self.assertIsNot(res['srv'], None) self._delete(name) # get on deleted reference self.assertRaises(exc.NotFound, self.api.get, self.account, name) def test_rdir_linking(self): """ Tests that rdir services linked to rawx services are not on the same locations """ cs = ConscienceClient({'namespace': self.ns}) rawx_list = cs.all_services('rawx') rdir_dict = {x['addr']: x for x in cs.all_services('rdir')} # Link the services for rawx in rawx_list: self.api.link('_RDIR_TEST', rawx['addr'], 'rdir', autocreate=True) # Do the checks for rawx in rawx_list: linked_rdir = self.api.get( '_RDIR_TEST', rawx['addr'], service_type='rdir')['srv'] rdir = rdir_dict[linked_rdir[0]['host']] rawx_loc = rawx['tags'].get('tag.loc') rdir_loc = rdir['tags'].get('tag.loc') self.assertNotEqual(rawx_loc, rdir_loc) # Unlink the services for rawx in rawx_list: self.api.unlink('_RDIR_TEST', rawx['addr'], 'rdir') self.api.delete('_RDIR_TEST', rawx['addr']) def test_rdir_repartition(self): client = RdirClient({'namespace': self.ns}) all_rawx = client.assign_all_rawx() by_rdir = dict() total = 0 for rawx in all_rawx: count = by_rdir.get(rawx['rdir']['addr'], 0) total += 1 by_rdir[rawx['rdir']['addr']] = count + 1 avg = total / float(len(by_rdir)) print "Ideal number of bases per rdir: ", avg print "Current repartition: ", by_rdir for count in by_rdir.itervalues(): self.assertLessEqual(count, avg + 1)
class RdirClient(Client): def __init__(self, conf, **kwargs): super(RdirClient, self).__init__(conf, **kwargs) self.directory = DirectoryAPI(self.ns, self.endpoint, **kwargs) self._addr_cache = dict() def assign_all_rawx(self): """ Find a rdir service for all rawx that don't have one already. """ cs = ConscienceClient(self.conf) all_rawx = cs.all_services("rawx") all_rdir = cs.all_services("rdir", True) by_id = {_make_id(self.ns, "rdir", x["addr"]): x for x in all_rdir} for rawx in all_rawx: try: # Verify that there is no rdir linked resp = self.directory.get(RDIR_ACCT, rawx["addr"], service_type="rdir") rawx["rdir"] = by_id[_make_id(self.ns, "rdir", self._lookup_rdir_host(resp))] except (NotFound, ClientException): rdir = self._smart_link_rdir(rawx["addr"], cs, all_rdir) n_bases = by_id[rdir]["tags"].get("stat.opened_db_count", 0) by_id[rdir]["tags"]["stat.opened_db_count"] = n_bases + 1 rawx["rdir"] = by_id[rdir] return all_rawx def _smart_link_rdir(self, volume_id, cs=None, all_rdir=None): """ Force the load balancer to avoid services that already host more bases than the average while selecting rdir services. """ if not cs: cs = ConscienceClient(self.conf) if not all_rdir: all_rdir = cs.all_services("rdir", True) avail_base_count = [x["tags"]["stat.opened_db_count"] for x in all_rdir if x["score"] > 0] mean = sum(avail_base_count) / float(len(avail_base_count)) avoids = [ _make_id(self.ns, "rdir", x["addr"]) for x in all_rdir if x["score"] > 0 and x["tags"]["stat.opened_db_count"] > mean ] known = [_make_id(self.ns, "rawx", volume_id)] try: polled = cs.poll("rdir", avoid=avoids, known=known)[0] except ClientException as exc: if exc.status != 481: raise # Retry without `avoids`, hoping the next iteration will rebalance polled = cs.poll("rdir", known=known)[0] forced = {"host": polled["addr"], "type": "rdir", "seq": 1, "args": "", "id": polled["id"]} self.directory.force(RDIR_ACCT, volume_id, "rdir", forced, autocreate=True) return polled["id"] def _link_rdir(self, volume_id, smart=True): if not smart: self.directory.link(RDIR_ACCT, volume_id, "rdir", autocreate=True) else: self._smart_link_rdir(volume_id) return self.directory.get(RDIR_ACCT, volume_id, service_type="rdir") def _lookup_rdir_host(self, resp): host = None for srv in resp.get("srv", {}): if srv["type"] == "rdir": host = srv["host"] if not host: raise ClientException("No rdir service found") return host def _get_rdir_addr(self, volume_id, create=False, nocache=False): if not nocache and volume_id in self._addr_cache: return self._addr_cache[volume_id] resp = {} try: resp = self.directory.get(RDIR_ACCT, volume_id, service_type="rdir") except NotFound: if not create: raise VolumeException("No such volume %s" % volume_id) try: host = self._lookup_rdir_host(resp) except ClientException: # Reference exists but no rdir linked if not create: raise resp = self._link_rdir(volume_id) host = self._lookup_rdir_host(resp) self._addr_cache[volume_id] = host return host def _make_uri(self, action, volume_id, create=False, nocache=False): rdir_host = self._get_rdir_addr(volume_id, create=create, nocache=nocache) uri = "http://%s/v1/%s" % (rdir_host, action) return uri def _rdir_request(self, volume, method, action, create=False, **kwargs): params = {"vol": volume} if create: params["create"] = "1" uri = self._make_uri(action, volume, create=create) try: resp, body = self._direct_request(method, uri, params=params, **kwargs) except NotFound: uri = self._make_uri(action, volume, create=create, nocache=True) resp, body = self._direct_request(method, uri, params=params, **kwargs) return resp, body def chunk_push(self, volume_id, container_id, content_id, chunk_id, **data): """Reference a chunk in the reverse directory""" body = {"container_id": container_id, "content_id": content_id, "chunk_id": chunk_id} for key, value in data.iteritems(): body[key] = value headers = {} self._rdir_request(volume_id, "POST", "rdir/push", create=True, json=body, headers=headers) def chunk_delete(self, volume_id, container_id, content_id, chunk_id): """Unreference a chunk from the reverse directory""" body = {"container_id": container_id, "content_id": content_id, "chunk_id": chunk_id} self._rdir_request(volume_id, "DELETE", "rdir/delete", json=body) def chunk_fetch(self, volume, limit=100, rebuild=False): """Fetch the list of chunks belonging to the specified volume""" req_body = {"limit": limit} if rebuild: req_body["rebuild"] = True while True: resp, resp_body = self._rdir_request(volume, "POST", "rdir/fetch", json=req_body) resp.raise_for_status() if len(resp_body) == 0: break for (key, value) in resp_body: container, content, chunk = key.split("|") yield container, content, chunk, value req_body["start_after"] = key def admin_incident_set(self, volume, date): body = {"date": int(float(date))} self._rdir_request(volume, "POST", "rdir/admin/incident", json=body) def admin_incident_get(self, volume): resp, resp_body = self._rdir_request(volume, "GET", "rdir/admin/incident") return resp_body.get("date") def admin_lock(self, volume, who): body = {"who": who} self._rdir_request(volume, "POST", "rdir/admin/lock", json=body) def admin_unlock(self, volume): self._rdir_request(volume, "POST", "rdir/admin/unlock") def admin_show(self, volume): resp, resp_body = self._rdir_request(volume, "GET", "rdir/admin/show") return resp_body def admin_clear(self, volume, clear_all=False): body = {"all": clear_all} resp, resp_body = self._rdir_request(volume, "POST", "rdir/admin/clear", json=body) return resp_body def status(self, volume): resp, resp_body = self._rdir_request(volume, "GET", "rdir/status") return resp_body