def main():
args = make_arg_parser().parse_args()
conf = args.conf_file if args.conf_file else "mover.conf"
cs = ConscienceClient({"namespace": args.namespace})
all_rawx = cs.all_services('rawx', full=True)
# Sort rawx by disk usage
all_rawx.sort(key=lambda c: c['tags']['stat.space'])
dic = dict()
sum_size = 0
nrawx = 0
for rawx in all_rawx:
addr = rawx['addr']
volume = rawx['tags']['tag.vol']
du = float(rawx['tags']['stat.space'])
# Keep only rawx with big disk usage
if nrawx < len(all_rawx)/2:
dic[addr] = volume
sum_size += du
nrawx += 1
if nrawx == 0:
return
av = sum_size/nrawx
# Lock rawx, run blob-mover and unlock rawx
target_use = str(int(av))
for addr in dic:
infos_srv = {"addr": addr, "type": "rawx"}
print("Lock rawx at " + addr)
if not args.dry_run:
cs.lock_score(infos_srv)
print("Run mover on rawx at " + addr +
" to get disk usage under " + str(target_use))
if not args.dry_run:
try:
output = move_rawx(dic[addr], target_use,
args.user, args.namespace, conf)
print(output)
except Exception as err:
print("ERROR: " + str(err))
print("Unlock rawx at " + addr)
if not args.dry_run:
cs.unlock_score(infos_srv)
# Delete mover configuration file
if not args.dry_run:
remove(conf)
class TestPerfectibleContent(BaseTestCase):
def setUp(self):
super(TestPerfectibleContent, self).setUp()
self.api = ObjectStorageApi(self.ns,
endpoint=self.uri,
pool_manager=self.http_pool)
self.cs = ConscienceClient(self.conf, pool_manager=self.http_pool)
self.event = EventClient(self.conf)
self.locked_svc = list()
# Ensure the tube is not clogged
self.event.beanstalk.drain_tube(DEFAULT_IMPROVER_TUBE)
def tearDown(self):
if self.locked_svc:
self.cs.unlock_score(self.locked_svc)
super(TestPerfectibleContent, self).tearDown()
@classmethod
def tearDownClass(cls):
# Be kind with the next test suites
cls._cls_reload_proxy()
time.sleep(3)
cls._cls_reload_meta()
time.sleep(1)
def _aggregate_services(self, type_, key):
"""
Build lists of services indexed by `key`.
"""
all_svcs = self.cs.all_services(type_)
out = defaultdict(list)
for svc in all_svcs:
out[key(svc)].append(svc)
return out
def _lock_services(self, type_, services):
"""
Lock specified services, wait for the score to be propagated.
"""
for svc in services:
self.locked_svc.append({'type': type_, 'addr': svc['addr']})
self.cs.lock_score(self.locked_svc)
# In a perfect world™️ we do not need the time.sleep().
# For mysterious reason, all services are not reloaded immediately.
self._reload_proxy()
time.sleep(0.5)
self._reload_meta()
time.sleep(0.5)
def _wait_for_event(self, timeout=REASONABLE_EVENT_DELAY):
"""
Wait for an event in the oio-improve tube.
"""
bt = self.event.beanstalk
bt.watch(DEFAULT_IMPROVER_TUBE)
try:
job_id, data = bt.reserve(timeout=timeout)
except ResponseError as exc:
logging.warn('No event read from tube %s: %s',
DEFAULT_IMPROVER_TUBE, exc)
self.fail()
bt.delete(job_id)
return Event(json.loads(data))
# This test must be executed first
def test_0_upload_ok(self):
"""Check that no event is emitted when everything is ok."""
# Check we have enough service locations.
by_place = self._aggregate_services(
'rawx', lambda x: x['tags']['tag.loc'].rsplit('.', 2)[0])
if len(by_place) < 3:
self.skip('This test requires 3 different 2nd level locations')
return
# Upload an object.
container = self._random_user()
reqid = request_id('perfectible-')
self.api.object_create(self.account,
container,
obj_name='perfect',
data='whatever',
policy='THREECOPIES',
headers={'X-oio-req-id': reqid})
# Wait on the oio-improve beanstalk tube.
bt = self.event.beanstalk
bt.watch(DEFAULT_IMPROVER_TUBE)
# Ensure we do not receive any event.
self.assertRaises(ResponseError,
bt.reserve,
timeout=REASONABLE_EVENT_DELAY)
def test_upload_warn_dist(self):
"""
Check that an event is emitted when the warning distance is reached.
"""
# Check we have enough service locations.
by_place = self._aggregate_services(
'rawx', lambda x: x['tags']['tag.loc'].rsplit('.', 2)[0])
if len(by_place) < 3:
self.skip('This test requires 3 different 2nd level locations')
return
# Lock all services of the 3rd location.
banned_loc = by_place.keys()[2]
self._lock_services('rawx', by_place[banned_loc])
# Upload an object.
container = self._random_user()
reqid = request_id('perfectible-')
self.api.object_create(self.account,
container,
obj_name='perfectible',
data='whatever',
policy='THREECOPIES',
headers={'X-oio-req-id': reqid})
# Wait on the oio-improve beanstalk tube.
event = self._wait_for_event()
# Check the content of the event.
self.assertEqual('storage.content.perfectible', event.event_type)
self.assertEqual(reqid, event.reqid)
self.assertEqual(self.account, event.url['account'])
self.assertEqual(container, event.url['user'])
self.assertEqual('perfectible', event.url['path'])
mc = event.data
self.assertEqual(0, mc['pos']) # only one metachunk in this test
lowest_dist = 4
warn_dist = 4
for chunk in mc['chunks']:
qual = chunk['quality']
if qual['final_dist'] < lowest_dist:
lowest_dist = qual['final_dist']
if qual['warn_dist'] < warn_dist:
warn_dist = qual['warn_dist']
self.assertEqual(qual['expected_slot'], qual['final_slot'])
self.assertLessEqual(lowest_dist, warn_dist)
def test_upload_fallback(self):
"""
Test that an event is emitted when a fallback service slot is used.
"""
by_slot = self._aggregate_services(
'rawx',
lambda x: x['tags'].get('tag.slots', 'rawx').rsplit(',', 2)[-1])
if len(by_slot) < 2:
self.skip('This test requires 2 different slots for rawx services')
return
elif len(by_slot['rawx-odd']) < 3:
self.skip('This test requires at least 3 services '
'in the "rawx-odd" slot')
# Lock all services of the 'rawx-even' slot.
banned_slot = 'rawx-even'
self._lock_services('rawx', by_slot[banned_slot])
# Upload an object.
container = self._random_user()
reqid = request_id('perfectible-')
self.api.object_create(self.account,
container,
obj_name='perfectible',
data='whatever',
policy='THREECOPIES',
headers={'X-oio-req-id': reqid})
# Wait on the oio-improve beanstalk tube.
event = self._wait_for_event()
# Check the content of the event.
self.assertEqual('storage.content.perfectible', event.event_type)
self.assertEqual(reqid, event.reqid)
self.assertEqual(self.account, event.url['account'])
self.assertEqual(container, event.url['user'])
self.assertEqual('perfectible', event.url['path'])
mc = event.data
self.assertEqual(0, mc['pos']) # only one metachunk in this test
slot_matches = list()
for chunk in mc['chunks']:
qual = chunk['quality']
slot_matches.append(qual['final_slot'] == qual['expected_slot'])
self.assertNotEqual(qual['final_slot'], banned_slot)
self.assertIn(False, slot_matches)