def run_test_cache(self): session_id = str(uuid.uuid4()) chunk_holder_ref = self.promise_ref(ChunkHolderActor.default_name()) chunk_store = self._chunk_store data_list = [] for _ in range(9): data_id = str(uuid.uuid4()) data = np.random.randint(0, 32767, (655360,), np.int16) data_list.append((data_id, data)) def _put_chunk(data_key, data, *_): def _handle_reject(*exc): if issubclass(exc[0], NoDataToSpill): return six.reraise(*exc) try: ref = chunk_store.put(session_id, data_key, data) chunk_holder_ref.register_chunk(session_id, data_key) self.ctx.sleep(0.5) del ref except StoreFull: return chunk_holder_ref.spill_size(calc_data_size(data) * 2, _promise=True) \ .then(partial(_put_chunk, data_key, data), _handle_reject) data_promises = [] for data_id, data in data_list: data_promises.append(promise.Promise(done=True) \ .then(partial(_put_chunk, data_id, data))) def assert_true(v): assert v last_id = data_list[-1][0] p = promise.all_(data_promises) \ .then(lambda *_: assert_true(chunk_store.contains(session_id, last_id))) \ .then(lambda *_: ensure_chunk(self, session_id, last_id)) \ .then(lambda *_: assert_true(chunk_store.contains(session_id, last_id))) first_id = data_list[0][0] p = p.then(lambda *_: assert_true(not chunk_store.contains(session_id, first_id))) \ .then(lambda *_: ensure_chunk(self, session_id, first_id)) \ .then(lambda *_: assert_true(chunk_store.contains(session_id, first_id))) p = p.then(lambda *_: chunk_holder_ref.unregister_chunk(session_id, first_id)) \ .then(lambda *_: self._plasma_client.evict(128)) \ .then(lambda *_: assert_true(not chunk_store.contains(session_id, first_id))) p = p.then(lambda *_: chunk_holder_ref.unregister_chunk(session_id, last_id)) \ .then(lambda *_: self._plasma_client.evict(128)) \ .then(lambda *_: assert_true(not chunk_store.contains(session_id, last_id))) p.catch(lambda *exc: setattr(self, '_exc_info', exc)) \ .then(lambda *_: setattr(self, '_finished', True))
def register(self, session_id, chunk_keys): import numpy as np cache_ref = self.promise_ref(ChunkHolderActor.default_name()) left_keys = set(chunk_keys) def _put_chunk(chunk_key, data, spill_times=1): try: refs = self._chunk_store.put(session_id, chunk_key, data) cache_ref.register_chunk(session_id, chunk_key) del refs left_keys.remove(chunk_key) except StoreFull: return cache_ref.spill_size(2 * spill_times * calc_data_size(data), _promise=True) \ .then(partial(_put_chunk, chunk_key, data, 2 * spill_times)) promises = [] for idx, chunk_key in enumerate(chunk_keys): data = np.ones((640 * 1024,), dtype=np.int16) * idx promises.append(promise.Promise(done=True) \ .then(partial(_put_chunk, chunk_key, data))) promise.all_(promises).then(lambda *_: setattr(self, '_finished', True))
def gen_promise(value, accept=True): p = promise.Promise() promises[p.id] = p req_queue.put((p.id, value + 1, accept)) return p
def gen_promise(value, accept=True): value_list.append(('gen_promise', value)) p = promise.Promise() promises[p.id] = p req_queue.put((p.id, value + 1, accept)) return p
def testPromise(self): promises = weakref.WeakValueDictionary() req_queue = Queue() value_list = [] time_unit = 0.1 def test_thread_body(): while True: idx, v, success = req_queue.get() if v is None: break value_list.append(('thread_body', v)) time.sleep(time_unit) promises[idx].step_next(v, _accept=success) try: thread = threading.Thread(target=test_thread_body) thread.daemon = True thread.start() def gen_promise(value, accept=True): value_list.append(('gen_promise', value)) p = promise.Promise() promises[p.id] = p req_queue.put((p.id, value + 1, accept)) return p # simple promise call value_list = [] p = gen_promise(0) \ .then(lambda v: gen_promise(v)) \ .then(lambda v: gen_promise(v)) p.wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1), ('gen_promise', 1), ('thread_body', 2), ('gen_promise', 2), ('thread_body', 3)] ) # continue accepted call with then value_list = [] p.then(lambda *_: gen_promise(0)) \ .then(lambda v: gen_promise(v)) \ .wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1), ('gen_promise', 1), ('thread_body', 2)] ) # immediate error value_list = [] p = promise.Promise(done=True) \ .then(lambda *_: 5 / 0) p.catch(lambda *_: gen_promise(0)) \ .wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1)] ) # chained errors value_list = [] p = promise.Promise(failed=True) \ .catch(lambda *_: 1 / 0) \ .catch(lambda *_: 2 / 0) \ .catch(lambda *_: gen_promise(0)) \ .catch(lambda *_: gen_promise(1)) p.wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1)] ) # continue error call value_list = [] p = gen_promise(0) \ .then(lambda *_: 5 / 0) \ .then(lambda *_: gen_promise(2)) time.sleep(0.5) value_list = [] p.catch(lambda *_: gen_promise(0)) \ .wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1)] ) value_list = [] gen_promise(0) \ .then(lambda v: gen_promise(v).then(lambda x: x + 1)) \ .then(lambda v: gen_promise(v)) \ .wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1), ('gen_promise', 1), ('thread_body', 2), ('gen_promise', 3), ('thread_body', 4)] ) value_list = [] gen_promise(0) \ .then(lambda v: gen_promise(v)) \ .then(lambda v: gen_promise(v, False)) \ .catch(lambda v: value_list.append(('catch', v))) \ .wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1), ('gen_promise', 1), ('thread_body', 2), ('gen_promise', 2), ('thread_body', 3), ('catch', 3)] ) value_list = [] gen_promise(0) \ .then(lambda v: gen_promise(v, False).then(lambda x: x + 1)) \ .then(lambda v: gen_promise(v)) \ .catch(lambda v: value_list.append(('catch', v))) \ .wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1), ('gen_promise', 1), ('thread_body', 2), ('catch', 2)] ) value_list = [] gen_promise(0) \ .then(lambda v: gen_promise(v)) \ .then(lambda v: v + 1) \ .then(lambda v: gen_promise(v, False)) \ .catch(lambda v: value_list.append(('catch', v))) \ .wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1), ('gen_promise', 1), ('thread_body', 2), ('gen_promise', 3), ('thread_body', 4), ('catch', 4)] ) value_list = [] gen_promise(0) \ .then(lambda v: gen_promise(v)) \ .catch(lambda v: gen_promise(v)) \ .catch(lambda v: gen_promise(v)) \ .then(lambda v: gen_promise(v, False)) \ .catch(lambda v: value_list.append(('catch', v))) \ .wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1), ('gen_promise', 1), ('thread_body', 2), ('gen_promise', 2), ('thread_body', 3), ('catch', 3)] ) value_list = [] gen_promise(0) \ .then(lambda v: gen_promise(v)) \ .catch(lambda v: gen_promise(v)) \ .catch(lambda v: gen_promise(v)) \ .then(lambda v: gen_promise(v, False)) \ .then(lambda v: gen_promise(v), lambda v: gen_promise(v + 1, False)) \ .catch(lambda v: value_list.append(('catch', v))) \ .wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1), ('gen_promise', 1), ('thread_body', 2), ('gen_promise', 2), ('thread_body', 3), ('gen_promise', 4), ('thread_body', 5), ('catch', 5)] ) value_list = [] gen_promise(0) \ .then(lambda v: gen_promise(v)) \ .catch(lambda v: gen_promise(v)) \ .catch(lambda v: gen_promise(v)) \ .then(lambda v: gen_promise(v, False)) \ .then(lambda v: gen_promise(v), lambda v: _raise_exception(ValueError)) \ .catch(lambda *_: value_list.append(('catch',))) \ .wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1), ('gen_promise', 1), ('thread_body', 2), ('gen_promise', 2), ('thread_body', 3), ('catch', )] ) value_list = [] gen_promise(0) \ .then(lambda v: gen_promise(v, False)) \ .catch(lambda v: gen_promise(v, False)) \ .catch(lambda v: gen_promise(v)) \ .then(lambda v: gen_promise(v)) \ .catch(lambda v: value_list.append(('catch', v))) \ .wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1), ('gen_promise', 1), ('thread_body', 2), ('gen_promise', 2), ('thread_body', 3), ('gen_promise', 3), ('thread_body', 4), ('gen_promise', 4), ('thread_body', 5)] ) value_list = [] gen_promise(0) \ .then(lambda v: gen_promise(v, False)) \ .then(lambda v: gen_promise(v)) \ .catch(lambda v: value_list.append(('catch', v))) \ .wait() self.assertListEqual( value_list, [('gen_promise', 0), ('thread_body', 1), ('gen_promise', 1), ('thread_body', 2), ('catch', 2)] ) finally: gc.collect() self.assertDictEqual(promise._promise_pool, {}) req_queue.put((None, None, None))