def setUp(self):
super(TestAdmin, self).setUp()
self.admin = AdminClient(self.conf)
self.api = ObjectStorageApi(self.ns)
self.account = "test_admin"
self.container = "admin-" + random_str(4)
self.api.container_create(self.account, self.container)
def __init__(self, tool, queue_workers, queue_reply):
super(ContainerRepairerWorker, self).__init__(
tool, queue_workers, queue_reply)
self.rebuild_bases = true_value(self.tool.conf.get(
'rebuild_bases', self.tool.DEFAULT_REBUILD_BASES))
self.sync_bases = true_value(self.tool.conf.get(
'sync_bases', self.tool.DEFAULT_SYNC_BASES))
self.update_account = true_value(self.tool.conf.get(
'update_account', self.tool.DEFAULT_UPDATE_ACCOUNT))
self.admin_client = AdminClient(self.conf, logger=self.logger)
self.container_client = ContainerClient(self.conf, logger=self.logger)
self.meta2_database = Meta2Database(self.conf, logger=self.logger)
class ContentRepairerWorker(ToolWorker):
def __init__(self, tool, queue_workers, queue_reply):
super(ContentRepairerWorker, self).__init__(tool, queue_workers,
queue_reply)
self.admin_client = AdminClient(self.conf, logger=self.logger)
self.meta2_database = Meta2Database(self.conf, logger=self.logger)
def _process_item(self, item):
namespace, container_id = item
if namespace != self.tool.namespace:
raise ValueError('Invalid namespace (actual=%s, expected=%s)' %
(namespace, self.tool.namespace))
self.logger.debug('Rebuilding %s', self.tool.string_from_item(item))
errors = list()
for res in self.meta2_database.rebuild(container_id):
if res['err']:
errors.append('%s: %s' % (res['base'], res['err']))
if errors:
raise Exception(errors)
data = self.admin_client.election_sync(service_type='meta2',
cid=container_id)
for host, info in data.items():
if info['status']['status'] not in (200, 301):
errors.append('%s (%d): %s' % (host, info['status']['status'],
info['status']['message']))
if errors:
raise Exception(errors)
class MetaRebuilderWorker(RebuilderWorker):
def __init__(self, rebuilder, type_, max_attempts=5, **kwargs):
super(MetaRebuilderWorker, self).__init__(rebuilder, **kwargs)
self.type = type_
self.max_attempts = max_attempts
self.admin_client = AdminClient(self.conf, logger=self.logger)
def _rebuild_one(self, cid, **kwargs):
missing_base = False
for attempts in range(self.max_attempts):
try:
if not missing_base:
# Check if the bases exist
info = self.admin_client.has_base(self.type, cid=cid)
for meta in info.values():
if meta['status']['status'] != 200:
missing_base = True
break
if missing_base:
self.admin_client.election_leave(self.type, cid=cid)
# TODO(adu): use self.admin_client.election_sync
# Setting a property will trigger a database replication
properties = {'sys.last_rebuild': str(int(time.time()))}
self.admin_client.set_properties(self.type,
cid=cid,
properties=properties)
break
except Exception as err:
if attempts < self.max_attempts - 1:
if isinstance(err, NotFound):
self.logger.warn('%s: %s', cid, err)
continue
if isinstance(err, OioTimeout) \
or isinstance(err, ServiceBusy):
self.logger.warn('%s: %s', cid, err)
time.sleep(attempts * 0.5)
continue
self.logger.error('ERROR while rebuilding %s: %s', cid, err)
sys.stdout.write(cid + '\n')
break
class ContainerRepairerWorker(ToolWorker):
def __init__(self, tool, queue_workers, queue_reply):
super(ContainerRepairerWorker, self).__init__(
tool, queue_workers, queue_reply)
self.rebuild_bases = true_value(self.tool.conf.get(
'rebuild_bases', self.tool.DEFAULT_REBUILD_BASES))
self.sync_bases = true_value(self.tool.conf.get(
'sync_bases', self.tool.DEFAULT_SYNC_BASES))
self.update_account = true_value(self.tool.conf.get(
'update_account', self.tool.DEFAULT_UPDATE_ACCOUNT))
self.admin_client = AdminClient(self.conf, logger=self.logger)
self.container_client = ContainerClient(self.conf, logger=self.logger)
self.meta2_database = Meta2Database(self.conf, logger=self.logger)
def _process_item(self, item):
namespace, account, container = item
if namespace != self.tool.namespace:
raise ValueError('Invalid namespace (actual=%s, expected=%s)' % (
namespace, self.tool.namespace))
errors = list()
if self.rebuild_bases:
cid = cid_from_name(account, container)
for res in self.meta2_database.rebuild(cid):
if res['err']:
errors.append('%s: %s' % (res['base'], res['err']))
if errors:
raise Exception(errors)
if self.sync_bases:
data = self.admin_client.election_sync(
service_type='meta2', account=account, reference=container)
for host, info in data.items():
if info['status']['status'] not in (200, 301):
errors.append('%s (%d): %s' % (
host, info['status']['status'],
info['status']['message']))
if errors:
raise Exception(errors)
if self.update_account:
self.container_client.container_touch(
account=account, reference=container)
def admin(self):
if not self._admin:
self._admin = AdminClient(self.conf)
return self._admin
def __init__(self, tool, queue_workers, queue_reply):
super(ContentRepairerWorker, self).__init__(tool, queue_workers,
queue_reply)
self.admin_client = AdminClient(self.conf, logger=self.logger)
self.meta2_database = Meta2Database(self.conf, logger=self.logger)
def admin(self):
"""Get a client for admin operations (especially sqliterepo)."""
if not self._admin:
from oio.directory.admin import AdminClient
self._admin = AdminClient(self.conf, pool_manager=self.http_pool)
return self._admin
class TestAdmin(BaseTestCase):
def setUp(self):
super(TestAdmin, self).setUp()
self.admin = AdminClient(self.conf)
self.api = ObjectStorageApi(self.ns)
self.account = "test_admin"
self.container = "admin-" + random_str(4)
self.api.container_create(self.account, self.container)
def tearDown(self):
super(TestAdmin, self).tearDown()
try:
self.api.container_delete(self.account, self.container)
except Exception:
pass
def test_election_leave_service_id(self):
status = self.admin.election_status("meta2",
account=self.account,
reference=self.container)
peers = status["peers"]
service_id = peers.keys()[random.randrange(len(peers))]
election = self.admin.election_leave("meta2",
account=self.account,
reference=self.container,
service_id=service_id)
self.assertEqual(1, len(election))
self.assertEqual(200, election[service_id]["status"]["status"])
def test_election_leave_serveral_service_ids(self):
status = self.admin.election_status("meta2",
account=self.account,
reference=self.container)
peers = status["peers"]
if len(peers) < 2:
self.skipTest('Can only run in a replicated environment')
service_ids = peers.keys()[:2]
election = self.admin.election_leave("meta2",
account=self.account,
reference=self.container,
service_id=service_ids)
self.assertEquals(2, len(election))
self.assertEquals(200, election[service_ids[0]]["status"]["status"])
self.assertEquals(200, election[service_ids[1]]["status"]["status"])
def test_has_base(self):
info = self.admin.has_base('meta2',
account=self.account,
reference=self.container)
for peer, meta in info.iteritems():
self.assertEqual(200, meta['status']['status'])
peer = info.keys()[0]
peer_loc = info[peer]['body']
self.assertTrue(os.path.isfile(peer_loc))
os.remove(peer_loc)
info = self.admin.has_base('meta2',
account=self.account,
reference=self.container)
self.assertNotEquals(200, info[peer]['status']['status'])
del info[peer]
for peer, meta in info.iteritems():
self.assertEqual(200, meta['status']['status'])
def admin(self):
if self._admin_client is None:
from oio.directory.admin import AdminClient
self._admin_client = AdminClient(self.client_conf,
pool_manager=self.pool_manager)
return self._admin_client
def __init__(self, rebuilder, type_, max_attempts=5, **kwargs):
super(MetaRebuilderWorker, self).__init__(rebuilder, **kwargs)
self.type = type_
self.max_attempts = max_attempts
self.admin_client = AdminClient(self.conf, logger=self.logger)
def make_client(instance):
endpoint = instance.get_endpoint('election')
client = AdminClient({"namespace": instance.namespace}, endpoint=endpoint)
return client