def testQuotaAllocation(self):
local_pool_addr = 'localhost:%d' % get_next_port()
with create_actor_pool(n_process=1, backend='gevent', address=local_pool_addr) as pool:
quota_ref = pool.create_actor(QuotaActor, 300, uid=QuotaActor.default_uid())
end_time = []
finished = set()
with self.run_actor_test(pool) as test_actor:
ref = test_actor.promise_ref(QuotaActor.default_uid())
def actual_exec(x):
ref.release_quota(x)
end_time.append(time.time())
finished.add(x)
if len(finished) == 5:
test_actor.set_result(None)
for idx in range(5):
x = str(idx)
ref.request_quota(x, 100, _promise=True) \
.then(functools.partial(test_actor.run_later, actual_exec, x, _delay=0.5))
self.get_result(10)
self.assertLess(abs(end_time[0] - end_time[1]), 0.1)
self.assertLess(abs(end_time[0] - end_time[2]), 0.1)
self.assertGreater(abs(end_time[0] - end_time[3]), 0.4)
self.assertLess(abs(end_time[3] - end_time[4]), 0.1)
self.assertEqual(quota_ref.get_allocated_size(), 0)
def testBatchQuotaAllocation(self):
local_pool_addr = 'localhost:%d' % get_next_port()
with create_actor_pool(n_process=1, backend='gevent', address=local_pool_addr) as pool:
quota_ref = pool.create_actor(QuotaActor, 300, uid=QuotaActor.default_uid())
end_time = []
for idx in range(2):
x = str(idx)
with self.run_actor_test(pool) as test_actor:
ref = test_actor.promise_ref(QuotaActor.default_uid())
def actual_exec(keys):
for k in keys:
ref.release_quota(k)
end_time.append(time.time())
test_actor.set_result(None)
keys = [x + '_0', x + '_1']
batch = dict((k, 100) for k in keys)
ref.request_batch_quota(batch, _promise=True) \
.then(functools.partial(test_actor.run_later, actual_exec, keys, _delay=0.5))
self.get_result(10)
self.assertGreater(abs(end_time[0] - end_time[1]), 0.4)
self.assertEqual(quota_ref.get_allocated_size(), 0)
def testBatchQuotaAllocation(self):
local_pool_addr = 'localhost:%d' % get_next_port()
with create_actor_pool(n_process=1, backend='gevent', address=local_pool_addr) as pool:
quota_ref = pool.create_actor(QuotaActor, 300, uid=QuotaActor.default_uid())
end_time = []
with self.run_actor_test(pool) as test_actor:
for idx in (0, 1):
x = str(idx)
ref = test_actor.promise_ref(QuotaActor.default_uid())
def actual_exec(b, set_result):
self.assertTrue(ref.request_batch_quota(b, process_quota=True))
self.assertEqual(set(b.keys()), set(quota_ref.dump_data().proc_sizes.keys()))
ref.release_quotas(list(b.keys()))
end_time.append(time.time())
if set_result:
test_actor.set_result(None)
keys = [x + '_0', x + '_1']
batch = dict((k, 100) for k in keys)
ref.request_batch_quota(batch, _promise=True) \
.then(functools.partial(test_actor.run_later, actual_exec, batch,
set_result=(idx == 1), _delay=0.5),
lambda *exc: test_actor.set_result(exc, accept=False))
self.get_result(10)
self.assertGreater(abs(end_time[0] - end_time[1]), 0.4)
self.assertEqual(quota_ref.get_allocated_size(), 0)
def testQuota(self):
local_pool_addr = 'localhost:%d' % get_next_port()
with create_actor_pool(n_process=1,
backend='gevent',
address=local_pool_addr) as pool:
quota_ref = pool.create_actor(QuotaActor,
300,
uid=QuotaActor.default_name())
end_time = []
for idx in range(4):
x = str(idx)
with self.run_actor_test(pool) as test_actor:
ref = test_actor.promise_ref(QuotaActor.default_name())
def actual_exec(x):
test_actor.ctx.sleep(1)
ref.release_quota(x)
end_time.append(time.time())
test_actor.set_result(None)
ref.request_quota(x, 100, _promise=True) \
.then(functools.partial(actual_exec, x))
pool.sleep(2.5)
self.assertLess(abs(end_time[0] - end_time[1]), 0.1)
self.assertLess(abs(end_time[0] - end_time[2]), 0.1)
self.assertGreater(abs(end_time[0] - end_time[3]), 0.9)
self.assertEqual(quota_ref.get_allocated_size(), 0)
def mock_step(self, key):
ref = self.promise_ref(QuotaActor.default_name())
def actual_exec():
gevent.sleep(1)
ref.release_quota(key)
self._end_time = time.time()
ref.request_quota(key, self._alloc_size, _promise=True) \
.then(actual_exec)
def mock_step(self, keys):
ref = self.promise_ref(QuotaActor.default_name())
def actual_exec():
gevent.sleep(1)
for k in keys:
ref.release_quota(k)
self._end_time = time.time()
batch = dict((k, self._alloc_size) for k in keys)
ref.request_batch_quota(batch, _promise=True) \
.then(actual_exec)
def testBatchQuota(self):
with create_actor_pool() as pool:
quota_ref = pool.create_actor(QuotaActor,
300,
uid=QuotaActor.default_name())
test_refs = [
pool.create_actor(BatchQuotaTestActor, 100) for _ in range(2)
]
def test_method():
for ref in test_refs:
ref_str = str(id(ref))
ref.mock_step([ref_str + '_0', ref_str + '_1'])
gevent.sleep(3)
return [ref.get_end_time() for ref in test_refs]
gl = gevent.spawn(test_method)
gl.join()
end_time = gl.value
self.assertGreater(abs(end_time[0] - end_time[1]), 0.9)
self.assertEqual(quota_ref.get_allocated_size(), 0)
def testQuota(self):
def _raiser(*_, **__):
raise ValueError
local_pool_addr = 'localhost:%d' % get_next_port()
with create_actor_pool(n_process=1, backend='gevent', address=local_pool_addr) as pool:
pool.create_actor(WorkerClusterInfoActor, [local_pool_addr],
uid=WorkerClusterInfoActor.default_uid())
pool.create_actor(StatusActor, local_pool_addr, uid=StatusActor.default_uid())
quota_ref = pool.create_actor(QuotaActor, 300, uid=QuotaActor.default_uid())
quota_ref.process_quota('non_exist')
quota_ref.hold_quota('non_exist')
quota_ref.release_quota('non_exist')
with self.assertRaises(ValueError):
quota_ref.request_quota('ERROR', 1000)
self.assertTrue(quota_ref.request_quota('0', 100))
self.assertTrue(quota_ref.request_quota('0', 50))
self.assertTrue(quota_ref.request_quota('0', 200))
quota_ref.process_quota('0')
self.assertIn('0', quota_ref.dump_data().proc_sizes)
quota_ref.alter_allocation('0', 190, new_key=('0', 0))
self.assertEqual(quota_ref.dump_data().allocations[('0', 0)], 190)
quota_ref.hold_quota(('0', 0))
self.assertIn(('0', 0), quota_ref.dump_data().hold_sizes)
quota_ref.alter_allocation(('0', 0), new_key=('0', 1))
self.assertEqual(quota_ref.dump_data().allocations[('0', 1)], 190)
with self.run_actor_test(pool) as test_actor:
ref = test_actor.promise_ref(QuotaActor.default_uid())
ref.request_quota('1', 150, _promise=True) \
.then(lambda *_: test_actor.set_result(True)) \
.catch(lambda *exc: test_actor.set_result(exc, accept=False))
pool.sleep(0.5)
self.assertFalse(quota_ref.request_quota('2', 50))
self.assertFalse(quota_ref.request_quota('3', 200))
self.assertFalse(quota_ref.request_quota('3', 180))
self.assertNotIn('2', quota_ref.dump_data().allocations)
ref.cancel_requests(('1',), reject_exc=build_exc_info(OSError))
with self.assertRaises(OSError):
self.get_result(5)
with patch_method(QuotaActor._request_quota, new=_raiser):
ref.request_quota('err_raise', 1, _promise=True) \
.catch(lambda *exc: test_actor.set_result(exc, accept=False))
with self.assertRaises(ValueError):
self.get_result(5)
ref.request_batch_quota({'err_raise': 1}, _promise=True) \
.catch(lambda *exc: test_actor.set_result(exc, accept=False))
with self.assertRaises(ValueError):
self.get_result(5)
self.assertNotIn('1', quota_ref.dump_data().requests)
self.assertIn('2', quota_ref.dump_data().allocations)
self.assertNotIn('3', quota_ref.dump_data().allocations)
quota_ref.release_quotas([('0', 1)])
self.assertIn('3', quota_ref.dump_data().allocations)
self.assertFalse(quota_ref.request_quota('4', 180))
quota_ref.alter_allocations(['3'], [50])
self.assertIn('4', quota_ref.dump_data().allocations)
with self.run_actor_test(pool) as test_actor:
ref = test_actor.promise_ref(QuotaActor.default_uid())
ref.request_quota('5', 50, _promise=True) \
.catch(lambda *exc: test_actor.set_result(exc, accept=False))
with patch_method(QuotaActor.alter_allocation, new=_raiser):
quota_ref.release_quota('2')
with self.assertRaises(ValueError):
self.get_result(5)