class RdirClient(Client): def __init__(self, conf, **kwargs): super(RdirClient, self).__init__(conf, **kwargs) self.directory_client = DirectoryClient(conf, **kwargs) def _lookup_rdir_host(self, resp): host = None for srv in resp.get('srv', {}): if srv['type'] == 'rdir': host = srv['host'] if not host: raise ClientException("No rdir service found") return host def _link_rdir(self, volume_id): self.directory_client.link(acct=RDIR_ACCT, ref=volume_id, srv_type='rdir', autocreate=True) return self.directory_client.show( acct=RDIR_ACCT, ref=volume_id, srv_type='rdir') # TODO keep rdir addr in local cache to avoid lookup requests def _get_rdir_addr(self, volume_id, create=False): resp = {} try: resp = self.directory_client.show(acct=RDIR_ACCT, ref=volume_id, srv_type='rdir') except NotFound: if not create: raise try: host = self._lookup_rdir_host(resp) except ClientException: # Reference exists but no rdir linked if not create: raise resp = self._link_rdir(volume_id) host = self._lookup_rdir_host(resp) return host def _make_uri(self, action, volume_id, create=False): rdir_host = self._get_rdir_addr(volume_id, create=create) uri = 'http://%s/v1/%s/%s' % (rdir_host, self.ns, action) return uri def _rdir_request(self, volume, method, action, create=False, **kwargs): uri = self._make_uri(action, volume, create=create) params = {'vol': volume} if create: params['create'] = '1' resp, body = self._direct_request(method, uri, params=params, **kwargs) return resp, body def chunk_push(self, volume_id, container_id, content_id, chunk_id, **data): body = {'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id} for key, value in data.iteritems(): body[key] = value headers = {} self._rdir_request(volume_id, 'POST', 'rdir/push', create=True, json=body, headers=headers) def chunk_delete(self, volume_id, container_id, content_id, chunk_id): body = {'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id} self._rdir_request(volume_id, 'DELETE', 'rdir/delete', json=body) def chunk_fetch(self, volume, limit=100, rebuild=False): req_body = {'limit': limit} if rebuild: req_body['rebuild'] = True while True: resp, resp_body = self._rdir_request(volume, 'POST', 'rdir/fetch', json=req_body) resp.raise_for_status() if len(resp_body) == 0: break for (key, value) in resp_body: container, content, chunk = key.split('|') yield container, content, chunk, value req_body['start_after'] = key def admin_incident_set(self, volume, date): body = {'date': date} self._rdir_request(volume, 'POST', 'rdir/admin/incident', json=body) def admin_incident_get(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/incident') return resp_body.get('date') def admin_lock(self, volume, who): body = {'who': who} self._rdir_request(volume, 'POST', 'rdir/admin/lock', json=body) def admin_unlock(self, volume): self._rdir_request(volume, 'POST', 'rdir/admin/unlock') def admin_show(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/show') return resp_body def status(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/status') return resp_body
class RdirClient(Client): def __init__(self, conf, **kwargs): super(RdirClient, self).__init__(conf, **kwargs) self.autocreate = true_value(conf.get('autocreate', True)) self.directory_client = DirectoryClient(conf) # TODO keep rdir addr in local cache to avoid lookup requests def _get_rdir_addr(self, volume_id): try: resp = self.directory_client.show(acct='_RDIR', ref=volume_id, srv_type='rdir') except NotFound as e: if self.autocreate: self.directory_client.link('_RDIR', volume_id, 'rdir', autocreate=True) resp = self.directory_client.show(acct='_RDIR', ref=volume_id, srv_type='rdir') else: raise e for srv in resp['srv']: if srv['type'] == 'rdir': return srv['host'] raise ClientException("No rdir service found") def _make_uri(self, action, volume_id): rdir_host = self._get_rdir_addr(volume_id) uri = 'http://%s/v1/%s/%s?vol=%s' % (rdir_host, self.ns, action, volume_id) return uri def _rdir_request(self, volume, method, action, **kwargs): uri = self._make_uri(action, volume) resp, body = self._direct_request(method, uri, **kwargs) return resp, body def chunk_push(self, volume_id, container_id, content_id, chunk_id, **data): body = { 'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id } for key, value in data.iteritems(): body[key] = value headers = {} self._rdir_request(volume_id, 'POST', 'rdir/push', json=body, headers=headers) def chunk_delete(self, volume_id, container_id, content_id, chunk_id): body = { 'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id } self._rdir_request(volume_id, 'DELETE', 'rdir/delete', json=body) def chunk_fetch(self, volume, limit=100, rebuild=False): req_body = {'limit': limit} if rebuild: req_body['rebuild'] = True while True: resp, resp_body = self._rdir_request(volume, 'POST', 'rdir/fetch', json=req_body) resp.raise_for_status() if len(resp_body) == 0: break for (key, value) in resp_body: container, content, chunk = key.split('|') yield container, content, chunk, value req_body['start_after'] = key def admin_incident_set(self, volume, date): body = {'date': date} self._rdir_request(volume, 'POST', 'rdir/admin/incident', json=body) def admin_incident_get(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/incident') return resp_body.get('date') def admin_lock(self, volume, who): body = {'who': who} self._rdir_request(volume, 'POST', 'rdir/admin/lock', json=body) def admin_unlock(self, volume): self._rdir_request(volume, 'POST', 'rdir/admin/unlock') def admin_show(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/show') return resp_body
class RdirClient(Client): def __init__(self, conf, **kwargs): super(RdirClient, self).__init__(conf, **kwargs) self.autocreate = true_value(conf.get('autocreate', True)) self.directory_client = DirectoryClient(conf) # TODO keep rdir addr in local cache to avoid lookup requests def _get_rdir_addr(self, volume_id): try: resp = self.directory_client.show(acct='_RDIR', ref=volume_id, srv_type='rdir') except NotFound as e: if self.autocreate: self.directory_client.link('_RDIR', volume_id, 'rdir', autocreate=True) resp = self.directory_client.show(acct='_RDIR', ref=volume_id, srv_type='rdir') else: raise e for srv in resp['srv']: if srv['type'] == 'rdir': return srv['host'] raise ClientException("No rdir service found") def _make_uri(self, action, volume_id): rdir_host = self._get_rdir_addr(volume_id) uri = 'http://%s/v1/%s/%s?vol=%s' % ( rdir_host, self.ns, action, volume_id) return uri def _rdir_request(self, volume, method, action, **kwargs): uri = self._make_uri(action, volume) resp, body = self._direct_request(method, uri, **kwargs) return resp, body def chunk_push(self, volume_id, container_id, content_id, chunk_id, **data): body = {'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id} for key, value in data.iteritems(): body[key] = value headers = {} self._rdir_request(volume_id, 'POST', 'rdir/push', json=body, headers=headers) def chunk_delete(self, volume_id, container_id, content_id, chunk_id): body = {'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id} self._rdir_request(volume_id, 'DELETE', 'rdir/delete', json=body) def chunk_fetch(self, volume, limit=100, rebuild=False): req_body = {'limit': limit} if rebuild: req_body['rebuild'] = True while True: resp, resp_body = self._rdir_request(volume, 'POST', 'rdir/fetch', json=req_body) resp.raise_for_status() if len(resp_body) == 0: break for (key, value) in resp_body: container, content, chunk = key.split('|') yield container, content, chunk, value req_body['start_after'] = key def admin_incident_set(self, volume, date): body = {'date': date} self._rdir_request(volume, 'POST', 'rdir/admin/incident', json=body) def admin_incident_get(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/incident') return resp_body.get('date') def admin_lock(self, volume, who): body = {'who': who} self._rdir_request(volume, 'POST', 'rdir/admin/lock', json=body) def admin_unlock(self, volume): self._rdir_request(volume, 'POST', 'rdir/admin/unlock') def admin_show(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/show') return resp_body
class TestDirectoryAPI(BaseTestCase): def setUp(self): super(TestDirectoryAPI, self).setUp() self.api = DirectoryClient({'namespace': self.ns}, endpoint=self.uri) def _create(self, name, metadata=None): return self.api.create(self.account, name, properties=metadata) def _delete(self, name): self.api.delete(self.account, name) def _clean(self, name, clear=False): if clear: # must clean properties before self.api.del_properties(self.account, name, []) self._delete(name) def _get_properties(self, name, properties=None): return self.api.get_properties( self.account, name, properties=properties) def _set_properties(self, name, properties=None): return self.api.set_properties( self.account, name, properties=properties) def test_list(self): # get on unknown reference name = random_str(32) self.assertRaises(exc.NotFound, self.api.list, self.account, name) self._create(name) # get on existing reference res = self.api.list(self.account, name) self.assertIsNot(res['dir'], None) self.assertIsNot(res['srv'], None) self._delete(name) # get on deleted reference self.assertRaises(exc.NotFound, self.api.list, self.account, name) def test_create(self): name = random_str(32) res = self._create(name) self.assertEqual(res, True) # second create res = self._create(name) self.assertEqual(res, False) # clean self._delete(name) def test_create_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } res = self._create(name, metadata) self.assertEqual(res, True) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) def test_create_without_account(self): account = random_str(32) name = random_str(32) account_client = AccountClient(self.conf) self.assertRaises(exc.NotFound, account_client.account_show, account) self.api.create(account, name) time.sleep(0.5) # ensure account event have been processed self.assertEqual(account_client.account_show(account)['id'], account) # clean self.api.delete(account, name) account_client.account_delete(account) def test_delete(self): name = random_str(32) # delete on unknown reference self.assertRaises(exc.NotFound, self.api.delete, self.account, name) res = self._create(name) self.assertEqual(res, True) # delete on existing reference self._delete(name) # verify deleted self.assertRaises(exc.NotFound, self.api.list, self.account, name) # second delete self.assertRaises(exc.NotFound, self.api.delete, self.account, name) # verify deleted self.assertRaises(exc.NotFound, self.api.list, self.account, name) def test_get_properties(self): name = random_str(32) # get_properties on unknown reference self.assertRaises( exc.NotFound, self.api.get_properties, self.account, name) res = self._create(name) self.assertEqual(res, True) # get_properties on existing reference data = self.api.get_properties(self.account, name) self.assertEqual(data['properties'], {}) # get_properties metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } self._set_properties(name, metadata) data = self.api.get_properties(self.account, name) self.assertEqual(data['properties'], metadata) # get_properties specify key key = metadata.keys().pop(0) data = self.api.get_properties(self.account, name, [key]) self.assertEqual(data['properties'], {key: metadata[key]}) # clean self._clean(name, True) # get_properties on deleted reference self.assertRaises( exc.NotFound, self.api.get_properties, self.account, name) def test_set_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } # set_properties on unknown reference self.assertRaises( exc.NotFound, self.api.set_properties, self.account, name, metadata) res = self._create(name) self.assertEqual(res, True) # set_properties on existing reference self.api.set_properties(self.account, name, metadata) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # set_properties key = random_str(32) value = random_str(32) metadata2 = {key: value} self._set_properties(name, metadata2) metadata.update(metadata2) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # set_properties overwrite key key = metadata.keys().pop(0) value = random_str(32) metadata3 = {key: value} metadata.update(metadata3) self.api.set_properties(self.account, name, metadata3) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # set_properties on deleted reference self.assertRaises( exc.NotFound, self.api.set_properties, self.account, name, metadata) def test_del_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } # del_properties on unknown reference self.assertRaises( exc.NotFound, self.api.del_properties, self.account, name, []) res = self._create(name, metadata) self.assertEqual(res, True) key = metadata.keys().pop() del metadata[key] # del_properties on existing reference self.api.del_properties(self.account, name, [key]) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # del_properties on unknown key key = random_str(32) # We do not check if a property exists before deleting it # self.assertRaises( # exc.NotFound, self.api.del_properties, self.account, name, # [key]) self.api.del_properties(self.account, name, [key]) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # del_properties on deleted reference self.assertRaises( exc.NotFound, self.api.set_properties, self.account, name, metadata) def test_list_services(self): # list_services on unknown reference name = random_str(32) echo = 'echo' self.assertRaises( exc.NotFound, self.api.list, self.account, name, service_type=echo) self._create(name) # list_services on existing reference res = self.api.list(self.account, name, service_type=echo) self.assertIsNot(res['dir'], None) self.assertIsNot(res['srv'], None) self._delete(name) # get on deleted reference self.assertRaises(exc.NotFound, self.api.list, self.account, name) def test_rdir_linking_old(self): """ Tests that rdir services linked to rawx services are not on the same locations """ self.skipTest('Deprecated way of linking rdir services') self._reload_proxy() cs = ConscienceClient({'namespace': self.ns}) rawx_list = cs.all_services('rawx') rdir_dict = {x['addr']: x for x in cs.all_services('rdir')} # Link the services for rawx in rawx_list: self.api.link('_RDIR_TEST', rawx['addr'], 'rdir', autocreate=True) # Do the checks for rawx in rawx_list: linked_rdir = self.api.list( '_RDIR_TEST', rawx['addr'], service_type='rdir')['srv'] rdir = rdir_dict[linked_rdir[0]['host']] rawx_loc = rawx['tags'].get('tag.loc') rdir_loc = rdir['tags'].get('tag.loc') self.assertNotEqual(rawx_loc, rdir_loc) # Unlink the services for rawx in rawx_list: self.api.unlink('_RDIR_TEST', rawx['addr'], 'rdir') self.api.delete('_RDIR_TEST', rawx['addr']) def test_link_rdir_to_zero_scored_rawx(self): client = RdirClient({'namespace': self.ns}) disp = RdirDispatcher({'namespace': self.ns}) # Register a service, with score locked to zero new_rawx = self._srv('rawx', {'tag.loc': 'whatever'}) new_rawx['score'] = 0 self._register_srv(new_rawx) self._reload_proxy() all_rawx = disp.assign_all_rawx() all_rawx_keys = [x['addr'] for x in all_rawx] self.assertIn(new_rawx['addr'], all_rawx_keys) rdir_addr = client._get_rdir_addr(new_rawx['addr']) self.assertIsNotNone(rdir_addr) try: self.api.unlink('_RDIR', new_rawx['addr'], 'rdir') self.api.delete('_RDIR', new_rawx['addr']) # self._flush_cs('rawx') except Exception: pass def test_rdir_repartition(self): client = RdirDispatcher({'namespace': self.ns}) self._reload_proxy() all_rawx = client.assign_all_rawx() self.assertGreater(len(all_rawx), 0) by_rdir = dict() total = 0 for rawx in all_rawx: count = by_rdir.get(rawx['rdir']['addr'], 0) total += 1 by_rdir[rawx['rdir']['addr']] = count + 1 avg = total / float(len(by_rdir)) print("Ideal number of bases per rdir: ", avg) print("Current repartition: ", by_rdir) for count in by_rdir.itervalues(): self.assertLessEqual(count, avg + 1)
class TestDirectoryAPI(BaseTestCase): def setUp(self): super(TestDirectoryAPI, self).setUp() self.api = DirectoryClient({'namespace': self.ns}, endpoint=self.uri) def _create(self, name, metadata=None): return self.api.create(self.account, name, properties=metadata) def _delete(self, name): self.api.delete(self.account, name) def _clean(self, name, clear=False): if clear: # must clean properties before self.api.del_properties(self.account, name, []) self._delete(name) def _get_properties(self, name, properties=None): return self.api.get_properties(self.account, name, properties=properties) def _set_properties(self, name, properties=None): return self.api.set_properties(self.account, name, properties=properties) def test_list(self): # get on unknown reference name = random_str(32) self.assertRaises(exc.NotFound, self.api.list, self.account, name) self._create(name) # get on existing reference res = self.api.list(self.account, name) self.assertIsNot(res['dir'], None) self.assertIsNot(res['srv'], None) self._delete(name) # get on deleted reference self.assertRaises(exc.NotFound, self.api.list, self.account, name) def test_create(self): name = random_str(32) res = self._create(name) self.assertEqual(res, True) # second create res = self._create(name) self.assertEqual(res, False) # clean self._delete(name) def test_create_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } res = self._create(name, metadata) self.assertEqual(res, True) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) def test_delete(self): name = random_str(32) # delete on unknown reference self.assertRaises(exc.NotFound, self.api.delete, self.account, name) res = self._create(name) self.assertEqual(res, True) # delete on existing reference self._delete(name) # verify deleted self.assertRaises(exc.NotFound, self.api.list, self.account, name) # second delete self.assertRaises(exc.NotFound, self.api.delete, self.account, name) # verify deleted self.assertRaises(exc.NotFound, self.api.list, self.account, name) def test_get_properties(self): name = random_str(32) # get_properties on unknown reference self.assertRaises(exc.NotFound, self.api.get_properties, self.account, name) res = self._create(name) self.assertEqual(res, True) # get_properties on existing reference data = self.api.get_properties(self.account, name) self.assertEqual(data['properties'], {}) # get_properties metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } self._set_properties(name, metadata) data = self.api.get_properties(self.account, name) self.assertEqual(data['properties'], metadata) # get_properties specify key key = metadata.keys().pop(0) data = self.api.get_properties(self.account, name, [key]) self.assertEqual(data['properties'], {key: metadata[key]}) # clean self._clean(name, True) # get_properties on deleted reference self.assertRaises(exc.NotFound, self.api.get_properties, self.account, name) def test_set_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } # set_properties on unknown reference self.assertRaises(exc.NotFound, self.api.set_properties, self.account, name, metadata) res = self._create(name) self.assertEqual(res, True) # set_properties on existing reference self.api.set_properties(self.account, name, metadata) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # set_properties key = random_str(32) value = random_str(32) metadata2 = {key: value} self._set_properties(name, metadata2) metadata.update(metadata2) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # set_properties overwrite key key = metadata.keys().pop(0) value = random_str(32) metadata3 = {key: value} metadata.update(metadata3) self.api.set_properties(self.account, name, metadata3) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # set_properties on deleted reference self.assertRaises(exc.NotFound, self.api.set_properties, self.account, name, metadata) def test_del_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } # del_properties on unknown reference self.assertRaises(exc.NotFound, self.api.del_properties, self.account, name, []) res = self._create(name, metadata) self.assertEqual(res, True) key = metadata.keys().pop() del metadata[key] # del_properties on existing reference self.api.del_properties(self.account, name, [key]) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # del_properties on unknown key key = random_str(32) # We do not check if a property exists before deleting it # self.assertRaises( # exc.NotFound, self.api.del_properties, self.account, name, # [key]) self.api.del_properties(self.account, name, [key]) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # del_properties on deleted reference self.assertRaises(exc.NotFound, self.api.set_properties, self.account, name, metadata) def test_list_services(self): # list_services on unknown reference name = random_str(32) echo = 'echo' self.assertRaises(exc.NotFound, self.api.list, self.account, name, service_type=echo) self._create(name) # list_services on existing reference res = self.api.list(self.account, name, service_type=echo) self.assertIsNot(res['dir'], None) self.assertIsNot(res['srv'], None) self._delete(name) # get on deleted reference self.assertRaises(exc.NotFound, self.api.list, self.account, name) def test_rdir_linking(self): """ Tests that rdir services linked to rawx services are not on the same locations """ self._reload() cs = ConscienceClient({'namespace': self.ns}) rawx_list = cs.all_services('rawx') rdir_dict = {x['addr']: x for x in cs.all_services('rdir')} # Link the services for rawx in rawx_list: self.api.link('_RDIR_TEST', rawx['addr'], 'rdir', autocreate=True) # Do the checks for rawx in rawx_list: linked_rdir = self.api.list('_RDIR_TEST', rawx['addr'], service_type='rdir')['srv'] rdir = rdir_dict[linked_rdir[0]['host']] rawx_loc = rawx['tags'].get('tag.loc') rdir_loc = rdir['tags'].get('tag.loc') self.assertNotEqual(rawx_loc, rdir_loc) # Unlink the services for rawx in rawx_list: self.api.unlink('_RDIR_TEST', rawx['addr'], 'rdir') self.api.delete('_RDIR_TEST', rawx['addr']) def test_rdir_repartition(self): client = RdirDispatcher({'namespace': self.ns}) all_rawx = client.assign_all_rawx() by_rdir = dict() total = 0 for rawx in all_rawx: count = by_rdir.get(rawx['rdir']['addr'], 0) total += 1 by_rdir[rawx['rdir']['addr']] = count + 1 avg = total / float(len(by_rdir)) print "Ideal number of bases per rdir: ", avg print "Current repartition: ", by_rdir for count in by_rdir.itervalues(): self.assertLessEqual(count, avg + 1)
class TestDirectoryAPI(BaseTestCase): def setUp(self): super(TestDirectoryAPI, self).setUp() self.api = DirectoryClient({'namespace': self.ns}, endpoint=self.uri) def _create(self, name, metadata=None): return self.api.create(self.account, name, properties=metadata) def _delete(self, name): self.api.delete(self.account, name) def _clean(self, name, clear=False): if clear: # must clean properties before self.api.del_properties(self.account, name, []) self._delete(name) def _get_properties(self, name, properties=None): return self.api.get_properties(self.account, name, properties=properties) def _set_properties(self, name, properties=None): return self.api.set_properties(self.account, name, properties=properties) def test_list(self): # get on unknown reference name = random_str(32) self.assertRaises(exc.NotFound, self.api.list, self.account, name) self._create(name) # get on existing reference res = self.api.list(self.account, name) self.assertIsNot(res['dir'], None) self.assertIsNot(res['srv'], None) self.assertEqual(res['name'], name) self.assertEqual(res['account'], self.account) self._delete(name) # get on deleted reference self.assertRaises(exc.NotFound, self.api.list, self.account, name) def test_show_by_cid(self): name = random_str(32) self._create(name) res = self.api.list(cid=cid_from_name(self.account, name)) self.assertIsNotNone(res['dir']) self.assertIsNotNone(res['srv']) self.assertEqual(res['name'], name) self.assertEqual(res['account'], self.account) self._delete(name) def test_create(self): name = random_str(32) res = self._create(name) self.assertEqual(res, True) # second create res = self._create(name) self.assertEqual(res, False) # clean self._delete(name) def test_create_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } res = self._create(name, metadata) self.assertEqual(res, True) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) def test_create_without_account(self): account = random_str(32) name = random_str(32) account_client = AccountClient(self.conf) self.assertRaises(exc.NotFound, account_client.account_show, account) self.api.create(account, name) time.sleep(0.5) # ensure account event have been processed self.assertEqual(account_client.account_show(account)['id'], account) # clean self.api.delete(account, name) account_client.account_delete(account) def test_delete(self): name = random_str(32) # delete on unknown reference self.assertRaises(exc.NotFound, self.api.delete, self.account, name) res = self._create(name) self.assertEqual(res, True) # delete on existing reference self._delete(name) # verify deleted self.assertRaises(exc.NotFound, self.api.list, self.account, name) # second delete self.assertRaises(exc.NotFound, self.api.delete, self.account, name) # verify deleted self.assertRaises(exc.NotFound, self.api.list, self.account, name) def test_get_properties(self): name = random_str(32) # get_properties on unknown reference self.assertRaises(exc.NotFound, self.api.get_properties, self.account, name) res = self._create(name) self.assertEqual(res, True) # get_properties on existing reference data = self.api.get_properties(self.account, name) self.assertEqual(data['properties'], {}) # get_properties metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } self._set_properties(name, metadata) data = self.api.get_properties(self.account, name) self.assertEqual(data['properties'], metadata) # get_properties specify key key = metadata.keys().pop(0) data = self.api.get_properties(self.account, name, [key]) self.assertEqual(data['properties'], {key: metadata[key]}) # clean self._clean(name, True) # get_properties on deleted reference self.assertRaises(exc.NotFound, self.api.get_properties, self.account, name) def test_set_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } # set_properties on unknown reference self.assertRaises(exc.NotFound, self.api.set_properties, self.account, name, metadata) res = self._create(name) self.assertEqual(res, True) # set_properties on existing reference self.api.set_properties(self.account, name, metadata) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # set_properties key = random_str(32) value = random_str(32) metadata2 = {key: value} self._set_properties(name, metadata2) metadata.update(metadata2) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # set_properties overwrite key key = metadata.keys().pop(0) value = random_str(32) metadata3 = {key: value} metadata.update(metadata3) self.api.set_properties(self.account, name, metadata3) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # set_properties on deleted reference self.assertRaises(exc.NotFound, self.api.set_properties, self.account, name, metadata) def test_del_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } # del_properties on unknown reference self.assertRaises(exc.NotFound, self.api.del_properties, self.account, name, []) res = self._create(name, metadata) self.assertEqual(res, True) key = metadata.keys().pop() del metadata[key] # del_properties on existing reference self.api.del_properties(self.account, name, [key]) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # del_properties on unknown key key = random_str(32) # We do not check if a property exists before deleting it # self.assertRaises( # exc.NotFound, self.api.del_properties, self.account, name, # [key]) self.api.del_properties(self.account, name, [key]) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # del_properties on deleted reference self.assertRaises(exc.NotFound, self.api.set_properties, self.account, name, metadata) def test_list_services(self): # list_services on unknown reference name = random_str(32) echo = 'echo' self.assertRaises(exc.NotFound, self.api.list, self.account, name, service_type=echo) self._create(name) # list_services on existing reference res = self.api.list(self.account, name, service_type=echo) self.assertIsNot(res['dir'], None) self.assertIsNot(res['srv'], None) self.assertEqual(res['name'], name) self.assertEqual(res['account'], self.account) self._delete(name) # get on deleted reference self.assertRaises(exc.NotFound, self.api.list, self.account, name) def test_rdir_linking_old(self): """ Tests that rdir services linked to rawx services are not on the same locations """ def _id(r): return r['tags'].get('tag.service_id', r['addr']) self.skipTest('Deprecated way of linking rdir services') self._reload_proxy() cs = ConscienceClient({'namespace': self.ns}, endpoint=self.uri) rawx_list = cs.all_services('rawx') rdir_dict = {x['addr']: x for x in cs.all_services('rdir')} # Link the services for rawx in rawx_list: self.api.link('_RDIR_TEST', _id(rawx), 'rdir', autocreate=True) # Do the checks for rawx in rawx_list: linked_rdir = self.api.list('_RDIR_TEST', _id(rawx), service_type='rdir')['srv'] rdir = rdir_dict[linked_rdir[0]['host']] rawx_loc = rawx['tags'].get('tag.loc') rdir_loc = rdir['tags'].get('tag.loc') self.assertNotEqual(rawx_loc, rdir_loc) # Unlink the services for rawx in rawx_list: self.api.unlink('_RDIR_TEST', _id(rawx), 'rdir') self.api.delete('_RDIR_TEST', _id(rawx)) def test_link_rdir_to_zero_scored_rawx(self): client = RdirClient({'namespace': self.ns}) disp = RdirDispatcher({'namespace': self.ns}) # Register a service, with score locked to zero new_rawx = self._srv('rawx', {'tag.loc': 'whatever'}) new_rawx['score'] = 0 self._register_srv(new_rawx) self._reload_proxy() all_rawx = disp.assign_all_rawx() all_rawx_keys = [x['addr'] for x in all_rawx] self.assertIn(new_rawx['addr'], all_rawx_keys) rdir_addr = client._get_rdir_addr(new_rawx['addr']) self.assertIsNotNone(rdir_addr) try: self.api.unlink(RDIR_ACCT, new_rawx['addr'], 'rdir') self.api.delete(RDIR_ACCT, new_rawx['addr']) # self._flush_cs('rawx') except Exception: pass def _generate_services(self, types, score=50): all_srvs = dict() for type_, count in types.items(): srvs = [ self._srv(type_, {'tag.loc': 'whatever%d' % i}) for i in range(count) ] for srv in srvs: srv['score'] = score srv['id'] = _make_id(self.ns, type_, srv['addr']) all_srvs[type_] = srvs return all_srvs def _test_link_rdir_fail_to_force(self, side_effects, expected_exc): disp = RdirDispatcher({'namespace': self.ns}) # Mock rdir and rawx services so we do not pollute following tests all_srvs = self._generate_services({'rdir': 3, 'rawx': 3}) def _all_services(type_, *args, **kwargs): """Return all mocked services of specified type""" return all_srvs[type_] def _poll(*args, **kwargs): """Pick one mocked random service""" return [random.choice(all_srvs['rdir'])] disp.cs.all_services = Mock(side_effect=_all_services) disp.cs.poll = Mock(side_effect=_poll) # Mock the check method to avoid calling the proxy disp.directory.list = Mock(side_effect=exc.NotFound) # Mock the assignation methods so we can check the calls disp._smart_link_rdir = \ Mock(wraps=disp._smart_link_rdir) disp.directory.force = \ Mock(wraps=disp.directory.force, side_effect=side_effects) # Expect an exception since some assignations will fail self.assertRaises(expected_exc, disp.assign_all_rawx, max_attempts=1) # But ensure all calls have been made link_calls = [ call(rawx['addr'], ANY, max_per_rdir=ANY, max_attempts=1, service_type='rawx') for rawx in all_srvs['rawx'] ] disp._smart_link_rdir.assert_has_calls(link_calls) force_calls = \ [call(RDIR_ACCT, rawx['addr'], 'rdir', ANY, autocreate=True) for rawx in all_srvs['rawx']] disp.directory.force.assert_has_calls(force_calls) def test_link_rdir_fail_to_force_one(self): """ Verify that the failure of one 'force' operation does not break the whole operation. """ self._test_link_rdir_fail_to_force( [exc.ServiceBusy('Failed :('), None, None], exc.ServiceBusy) def test_link_rdir_fail_to_force_several(self): """ Verify that the failure of two 'force' operations does not break the whole operation. """ self._test_link_rdir_fail_to_force( [exc.ServiceBusy('Failed :('), exc.OioTimeout('Timeout :('), None], exc.OioException) def test_rdir_repartition(self): # FIXME(FVE): this test will fail if run after self._flush_cs('rawx') client = RdirDispatcher({'namespace': self.ns}) self._reload_proxy() all_rawx = client.assign_all_rawx() self.assertGreater(len(all_rawx), 0) by_rdir = dict() total = 0 for rawx in all_rawx: count = by_rdir.get(rawx['rdir']['addr'], 0) total += 1 by_rdir[rawx['rdir']['addr']] = count + 1 avg = total / float(len(by_rdir)) print "Ideal number of bases per rdir: ", avg print "Current repartition: ", by_rdir for count in by_rdir.itervalues(): self.assertLessEqual(count, avg + 1)