def testApi(self, *_):
timeout = 120 if 'CI' in os.environ else -1
with new_session(self._service_ep) as sess:
self.assertEqual(sess.count_workers(), 1)
a = mt.ones((100, 100), chunk_size=30)
b = mt.ones((100, 100), chunk_size=30)
c = a.dot(b)
result = sess.run(c, timeout=timeout)
np.testing.assert_array_equal(result, np.ones((100, 100)) * 100)
mock_resp = MockResponse(200, text='{"ERR_STR":')
with mock.patch('requests.Session.post',
side_effect=lambda *_, **__: mock_resp):
with self.assertRaises(ResponseMalformed):
sess.run(c, timeout=timeout)
mock_resp = MockResponse(
400,
json_data=dict(exc_info=base64.b64encode(
pickle.dumps(build_exc_info(ValueError))).decode()))
with mock.patch('requests.Session.post',
side_effect=lambda *_, **__: mock_resp):
with self.assertRaises(ValueError):
sess.run(c, timeout=timeout)
d = a * 100
self.assertIsNone(sess.run(d, fetch=False, timeout=120))
np.testing.assert_array_equal(sess.run(d, timeout=120),
np.ones((100, 100)) * 100)
def _create_writer_with_fail(self, session_id, chunk_key, *args, **kwargs):
if chunk_key == fail_key:
if kwargs.get('_promise', True):
return promise.finished(*build_exc_info(ValueError), **dict(_accept=False))
else:
raise ValueError
return old_create_writer(self, session_id, chunk_key, *args, **kwargs)
def actual_exec(self, session_id, graph_key):
if graph_key in self._results:
del self._results[graph_key]
rec = self._graph_records[(session_id, graph_key)]
if graph_key in self._cancels:
exc = build_exc_info(ExecutionInterrupted)
self._results[graph_key] = (exc, dict(_accept=False))
for cb in rec.finish_callbacks:
self.tell_promise(cb, *exc, **dict(_accept=False))
rec.finish_callbacks = []
return
elif self._fail_count and self._retries[graph_key] < self._fail_count:
exc = build_exc_info(ValueError)
logger.debug('Key %r: %r', graph_key, self._retries.get(graph_key))
self._retries[graph_key] += 1
del self._graph_records[(session_id, graph_key)]
self._results[graph_key] = (exc, dict(_accept=False))
for cb in rec.finish_callbacks:
self.tell_promise(cb, *exc, **dict(_accept=False))
rec.finish_callbacks = []
return
chunk_graph = rec.graph
key_to_chunks = defaultdict(list)
for n in chunk_graph:
key_to_chunks[n.key].append(n)
self._chunk_meta_ref.set_chunk_size(session_id, n.key, 0)
for tk in rec.data_targets:
for n in key_to_chunks[tk]:
self._chunk_meta_ref.add_worker(session_id, n.key, 'localhost:12345')
self._results[graph_key] = ((dict(),), dict())
for cb in rec.finish_callbacks:
self.tell_promise(cb, {})
rec.finish_callbacks = []
for succ_key in (rec.succ_keys or ()):
try:
succ_rec = self._graph_records[(session_id, succ_key)]
except KeyError:
continue
succ_rec.undone_pred_keys.difference_update([graph_key])
if not succ_rec.undone_pred_keys:
self.tell_promise(succ_rec.enqueue_callback)
def cancel_receive(self, session_id, chunk_keys, exc_info=None):
if exc_info is None:
exc_info = build_exc_info(ExecutionInterrupted)
for k in chunk_keys:
self._data_metas[(session_id, k)].status = ReceiveStatus.ERROR
if self._receiver_manager_ref:
self._receiver_manager_ref.notify_keys_finish(
session_id, chunk_keys, *exc_info, **dict(_accept=False, _tell=True))
def test_addr_reject(self):
from mars.errors import WorkerDead
self._finished = False
ref = self.promise_ref('ServeActor', address=self.address)
def _rejecter(*exc):
ref.serve(exc[0].__name__)
ref.serve(0, delay=2, _promise=True) \
.catch(_rejecter) \
.then(lambda *_: setattr(self, '_finished', True))
self.reject_dead_endpoints([self.address], *build_exc_info(WorkerDead))
def test_ref_reject(self):
from mars.errors import WorkerProcessStopped
self._finished = False
ref = self.promise_ref('ServeActor')
def _rejecter(*exc):
ref.serve(exc[0].__name__)
ref.serve(0, delay=2, _promise=True) \
.catch(_rejecter) \
.then(lambda *_: setattr(self, '_finished', True))
self.reject_promise_refs([ref], *build_exc_info(WorkerProcessStopped))
def actual_exec(self, session_id, graph_key):
if graph_key in self._results:
del self._results[graph_key]
rec = self._graph_records[(session_id, graph_key)]
if graph_key in self._cancels:
exc = build_exc_info(ExecutionInterrupted)
self._results[graph_key] = (exc, dict(_accept=False))
for cb in rec.finish_callbacks:
self.tell_promise(cb, *exc, _accept=False)
rec.finish_callbacks = []
return
elif self._fail_count and self._retries[graph_key] < self._fail_count:
exc = build_exc_info(ValueError)
logger.debug('Key %r: %r', graph_key, self._retries.get(graph_key))
self._retries[graph_key] += 1
del self._graph_records[(session_id, graph_key)]
self._results[graph_key] = (exc, dict(_accept=False))
for cb in rec.finish_callbacks:
self.tell_promise(cb, *exc, _accept=False)
rec.finish_callbacks = []
return
chunk_graph = rec.graph
key_to_chunks = defaultdict(list)
for n in chunk_graph:
key_to_chunks[n.key].append(n)
self.chunk_meta.set_chunk_size(session_id, n.key, 0)
for tk in rec.data_targets:
for n in key_to_chunks[tk]:
self.chunk_meta.add_worker(session_id, n.key,
'localhost:12345')
self._results[graph_key] = ((dict(), ), dict(), dict())
for cb in rec.finish_callbacks:
self.tell_promise(cb, {}, {})
rec.finish_callbacks = []
def _mock_load_from(*_, **__):
return promise.finished(*build_exc_info(SystemError),
_accept=False)
def testQuota(self):
def _raiser(*_, **__):
raise ValueError
local_pool_addr = 'localhost:%d' % get_next_port()
with create_actor_pool(n_process=1, backend='gevent', address=local_pool_addr) as pool:
pool.create_actor(WorkerClusterInfoActor, [local_pool_addr],
uid=WorkerClusterInfoActor.default_uid())
pool.create_actor(StatusActor, local_pool_addr, uid=StatusActor.default_uid())
quota_ref = pool.create_actor(QuotaActor, 300, uid=QuotaActor.default_uid())
quota_ref.process_quota('non_exist')
quota_ref.hold_quota('non_exist')
quota_ref.release_quota('non_exist')
with self.assertRaises(ValueError):
quota_ref.request_quota('ERROR', 1000)
self.assertTrue(quota_ref.request_quota('0', 100))
self.assertTrue(quota_ref.request_quota('0', 50))
self.assertTrue(quota_ref.request_quota('0', 200))
quota_ref.process_quota('0')
self.assertIn('0', quota_ref.dump_data().proc_sizes)
quota_ref.alter_allocation('0', 190, new_key=('0', 0))
self.assertEqual(quota_ref.dump_data().allocations[('0', 0)], 190)
quota_ref.hold_quota(('0', 0))
self.assertIn(('0', 0), quota_ref.dump_data().hold_sizes)
quota_ref.alter_allocation(('0', 0), new_key=('0', 1))
self.assertEqual(quota_ref.dump_data().allocations[('0', 1)], 190)
with self.run_actor_test(pool) as test_actor:
ref = test_actor.promise_ref(QuotaActor.default_uid())
ref.request_quota('1', 150, _promise=True) \
.then(lambda *_: test_actor.set_result(True)) \
.catch(lambda *exc: test_actor.set_result(exc, accept=False))
pool.sleep(0.5)
self.assertFalse(quota_ref.request_quota('2', 50))
self.assertFalse(quota_ref.request_quota('3', 200))
self.assertFalse(quota_ref.request_quota('3', 180))
self.assertNotIn('2', quota_ref.dump_data().allocations)
ref.cancel_requests(('1',), reject_exc=build_exc_info(OSError))
with self.assertRaises(OSError):
self.get_result(5)
with patch_method(QuotaActor._request_quota, new=_raiser):
ref.request_quota('err_raise', 1, _promise=True) \
.catch(lambda *exc: test_actor.set_result(exc, accept=False))
with self.assertRaises(ValueError):
self.get_result(5)
ref.request_batch_quota({'err_raise': 1}, _promise=True) \
.catch(lambda *exc: test_actor.set_result(exc, accept=False))
with self.assertRaises(ValueError):
self.get_result(5)
self.assertNotIn('1', quota_ref.dump_data().requests)
self.assertIn('2', quota_ref.dump_data().allocations)
self.assertNotIn('3', quota_ref.dump_data().allocations)
quota_ref.release_quotas([('0', 1)])
self.assertIn('3', quota_ref.dump_data().allocations)
self.assertFalse(quota_ref.request_quota('4', 180))
quota_ref.alter_allocations(['3'], [50])
self.assertIn('4', quota_ref.dump_data().allocations)
with self.run_actor_test(pool) as test_actor:
ref = test_actor.promise_ref(QuotaActor.default_uid())
ref.request_quota('5', 50, _promise=True) \
.catch(lambda *exc: test_actor.set_result(exc, accept=False))
with patch_method(QuotaActor.alter_allocation, new=_raiser):
quota_ref.release_quota('2')
with self.assertRaises(ValueError):
self.get_result(5)
def handle_process_down(self, halt_refs):
self.reject_promise_refs(halt_refs,
*build_exc_info(WorkerProcessStopped))