class MetaRebuilderWorker(RebuilderWorker):
def __init__(self, rebuilder, type_, max_attempts=5, **kwargs):
super(MetaRebuilderWorker, self).__init__(rebuilder, **kwargs)
self.type = type_
self.max_attempts = max_attempts
self.admin_client = AdminClient(self.conf, logger=self.logger)
def _rebuild_one(self, cid, **kwargs):
missing_base = False
for attempts in range(self.max_attempts):
try:
if not missing_base:
# Check if the bases exist
info = self.admin_client.has_base(self.type, cid=cid)
for meta in info.values():
if meta['status']['status'] != 200:
missing_base = True
break
if missing_base:
self.admin_client.election_leave(self.type, cid=cid)
# TODO(adu): use self.admin_client.election_sync
# Setting a property will trigger a database replication
properties = {'sys.last_rebuild': str(int(time.time()))}
self.admin_client.set_properties(self.type,
cid=cid,
properties=properties)
break
except Exception as err:
if attempts < self.max_attempts - 1:
if isinstance(err, NotFound):
self.logger.warn('%s: %s', cid, err)
continue
if isinstance(err, OioTimeout) \
or isinstance(err, ServiceBusy):
self.logger.warn('%s: %s', cid, err)
time.sleep(attempts * 0.5)
continue
self.logger.error('ERROR while rebuilding %s: %s', cid, err)
sys.stdout.write(cid + '\n')
break
class TestAdmin(BaseTestCase):
def setUp(self):
super(TestAdmin, self).setUp()
self.admin = AdminClient(self.conf)
self.api = ObjectStorageApi(self.ns)
self.account = "test_admin"
self.container = "admin-" + random_str(4)
self.api.container_create(self.account, self.container)
def tearDown(self):
super(TestAdmin, self).tearDown()
try:
self.api.container_delete(self.account, self.container)
except Exception:
pass
def test_election_leave_service_id(self):
status = self.admin.election_status("meta2",
account=self.account,
reference=self.container)
peers = status["peers"]
service_id = peers.keys()[random.randrange(len(peers))]
election = self.admin.election_leave("meta2",
account=self.account,
reference=self.container,
service_id=service_id)
self.assertEqual(1, len(election))
self.assertEqual(200, election[service_id]["status"]["status"])
def test_election_leave_serveral_service_ids(self):
status = self.admin.election_status("meta2",
account=self.account,
reference=self.container)
peers = status["peers"]
if len(peers) < 2:
self.skipTest('Can only run in a replicated environment')
service_ids = peers.keys()[:2]
election = self.admin.election_leave("meta2",
account=self.account,
reference=self.container,
service_id=service_ids)
self.assertEquals(2, len(election))
self.assertEquals(200, election[service_ids[0]]["status"]["status"])
self.assertEquals(200, election[service_ids[1]]["status"]["status"])
def test_has_base(self):
info = self.admin.has_base('meta2',
account=self.account,
reference=self.container)
for peer, meta in info.iteritems():
self.assertEqual(200, meta['status']['status'])
peer = info.keys()[0]
peer_loc = info[peer]['body']
self.assertTrue(os.path.isfile(peer_loc))
os.remove(peer_loc)
info = self.admin.has_base('meta2',
account=self.account,
reference=self.container)
self.assertNotEquals(200, info[peer]['status']['status'])
del info[peer]
for peer, meta in info.iteritems():
self.assertEqual(200, meta['status']['status'])