def run_test_cache(self):
session_id = str(uuid.uuid4())
chunk_holder_ref = self.promise_ref(ChunkHolderActor.default_name())
chunk_store = self._chunk_store
data_list = []
for _ in range(9):
data_id = str(uuid.uuid4())
data = np.random.randint(0, 32767, (655360,), np.int16)
data_list.append((data_id, data))
def _put_chunk(data_key, data, *_):
def _handle_reject(*exc):
if issubclass(exc[0], NoDataToSpill):
return
six.reraise(*exc)
try:
ref = chunk_store.put(session_id, data_key, data)
chunk_holder_ref.register_chunk(session_id, data_key)
self.ctx.sleep(0.5)
del ref
except StoreFull:
return chunk_holder_ref.spill_size(calc_data_size(data) * 2, _promise=True) \
.then(partial(_put_chunk, data_key, data), _handle_reject)
data_promises = []
for data_id, data in data_list:
data_promises.append(promise.Promise(done=True) \
.then(partial(_put_chunk, data_id, data)))
def assert_true(v):
assert v
last_id = data_list[-1][0]
p = promise.all_(data_promises) \
.then(lambda *_: assert_true(chunk_store.contains(session_id, last_id))) \
.then(lambda *_: ensure_chunk(self, session_id, last_id)) \
.then(lambda *_: assert_true(chunk_store.contains(session_id, last_id)))
first_id = data_list[0][0]
p = p.then(lambda *_: assert_true(not chunk_store.contains(session_id, first_id))) \
.then(lambda *_: ensure_chunk(self, session_id, first_id)) \
.then(lambda *_: assert_true(chunk_store.contains(session_id, first_id)))
p = p.then(lambda *_: chunk_holder_ref.unregister_chunk(session_id, first_id)) \
.then(lambda *_: self._plasma_client.evict(128)) \
.then(lambda *_: assert_true(not chunk_store.contains(session_id, first_id)))
p = p.then(lambda *_: chunk_holder_ref.unregister_chunk(session_id, last_id)) \
.then(lambda *_: self._plasma_client.evict(128)) \
.then(lambda *_: assert_true(not chunk_store.contains(session_id, last_id)))
p.catch(lambda *exc: setattr(self, '_exc_info', exc)) \
.then(lambda *_: setattr(self, '_finished', True))
def register(self, session_id, chunk_keys):
import numpy as np
cache_ref = self.promise_ref(ChunkHolderActor.default_name())
left_keys = set(chunk_keys)
def _put_chunk(chunk_key, data, spill_times=1):
try:
refs = self._chunk_store.put(session_id, chunk_key, data)
cache_ref.register_chunk(session_id, chunk_key)
del refs
left_keys.remove(chunk_key)
except StoreFull:
return cache_ref.spill_size(2 * spill_times * calc_data_size(data), _promise=True) \
.then(partial(_put_chunk, chunk_key, data, 2 * spill_times))
promises = []
for idx, chunk_key in enumerate(chunk_keys):
data = np.ones((640 * 1024,), dtype=np.int16) * idx
promises.append(promise.Promise(done=True) \
.then(partial(_put_chunk, chunk_key, data)))
promise.all_(promises).then(lambda *_: setattr(self, '_finished', True))
def gen_promise(value, accept=True):
p = promise.Promise()
promises[p.id] = p
req_queue.put((p.id, value + 1, accept))
return p
def gen_promise(value, accept=True):
value_list.append(('gen_promise', value))
p = promise.Promise()
promises[p.id] = p
req_queue.put((p.id, value + 1, accept))
return p
def testPromise(self):
promises = weakref.WeakValueDictionary()
req_queue = Queue()
value_list = []
time_unit = 0.1
def test_thread_body():
while True:
idx, v, success = req_queue.get()
if v is None:
break
value_list.append(('thread_body', v))
time.sleep(time_unit)
promises[idx].step_next(v, _accept=success)
try:
thread = threading.Thread(target=test_thread_body)
thread.daemon = True
thread.start()
def gen_promise(value, accept=True):
value_list.append(('gen_promise', value))
p = promise.Promise()
promises[p.id] = p
req_queue.put((p.id, value + 1, accept))
return p
# simple promise call
value_list = []
p = gen_promise(0) \
.then(lambda v: gen_promise(v)) \
.then(lambda v: gen_promise(v))
p.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1),
('gen_promise', 1), ('thread_body', 2),
('gen_promise', 2), ('thread_body', 3)]
)
# continue accepted call with then
value_list = []
p.then(lambda *_: gen_promise(0)) \
.then(lambda v: gen_promise(v)) \
.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1),
('gen_promise', 1), ('thread_body', 2)]
)
# immediate error
value_list = []
p = promise.Promise(done=True) \
.then(lambda *_: 5 / 0)
p.catch(lambda *_: gen_promise(0)) \
.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1)]
)
# chained errors
value_list = []
p = promise.Promise(failed=True) \
.catch(lambda *_: 1 / 0) \
.catch(lambda *_: 2 / 0) \
.catch(lambda *_: gen_promise(0)) \
.catch(lambda *_: gen_promise(1))
p.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1)]
)
# continue error call
value_list = []
p = gen_promise(0) \
.then(lambda *_: 5 / 0) \
.then(lambda *_: gen_promise(2))
time.sleep(0.5)
value_list = []
p.catch(lambda *_: gen_promise(0)) \
.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1)]
)
value_list = []
gen_promise(0) \
.then(lambda v: gen_promise(v).then(lambda x: x + 1)) \
.then(lambda v: gen_promise(v)) \
.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1),
('gen_promise', 1), ('thread_body', 2),
('gen_promise', 3), ('thread_body', 4)]
)
value_list = []
gen_promise(0) \
.then(lambda v: gen_promise(v)) \
.then(lambda v: gen_promise(v, False)) \
.catch(lambda v: value_list.append(('catch', v))) \
.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1),
('gen_promise', 1), ('thread_body', 2),
('gen_promise', 2), ('thread_body', 3),
('catch', 3)]
)
value_list = []
gen_promise(0) \
.then(lambda v: gen_promise(v, False).then(lambda x: x + 1)) \
.then(lambda v: gen_promise(v)) \
.catch(lambda v: value_list.append(('catch', v))) \
.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1),
('gen_promise', 1), ('thread_body', 2),
('catch', 2)]
)
value_list = []
gen_promise(0) \
.then(lambda v: gen_promise(v)) \
.then(lambda v: v + 1) \
.then(lambda v: gen_promise(v, False)) \
.catch(lambda v: value_list.append(('catch', v))) \
.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1),
('gen_promise', 1), ('thread_body', 2),
('gen_promise', 3), ('thread_body', 4),
('catch', 4)]
)
value_list = []
gen_promise(0) \
.then(lambda v: gen_promise(v)) \
.catch(lambda v: gen_promise(v)) \
.catch(lambda v: gen_promise(v)) \
.then(lambda v: gen_promise(v, False)) \
.catch(lambda v: value_list.append(('catch', v))) \
.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1),
('gen_promise', 1), ('thread_body', 2),
('gen_promise', 2), ('thread_body', 3),
('catch', 3)]
)
value_list = []
gen_promise(0) \
.then(lambda v: gen_promise(v)) \
.catch(lambda v: gen_promise(v)) \
.catch(lambda v: gen_promise(v)) \
.then(lambda v: gen_promise(v, False)) \
.then(lambda v: gen_promise(v), lambda v: gen_promise(v + 1, False)) \
.catch(lambda v: value_list.append(('catch', v))) \
.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1),
('gen_promise', 1), ('thread_body', 2),
('gen_promise', 2), ('thread_body', 3),
('gen_promise', 4), ('thread_body', 5),
('catch', 5)]
)
value_list = []
gen_promise(0) \
.then(lambda v: gen_promise(v)) \
.catch(lambda v: gen_promise(v)) \
.catch(lambda v: gen_promise(v)) \
.then(lambda v: gen_promise(v, False)) \
.then(lambda v: gen_promise(v), lambda v: _raise_exception(ValueError)) \
.catch(lambda *_: value_list.append(('catch',))) \
.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1),
('gen_promise', 1), ('thread_body', 2),
('gen_promise', 2), ('thread_body', 3),
('catch', )]
)
value_list = []
gen_promise(0) \
.then(lambda v: gen_promise(v, False)) \
.catch(lambda v: gen_promise(v, False)) \
.catch(lambda v: gen_promise(v)) \
.then(lambda v: gen_promise(v)) \
.catch(lambda v: value_list.append(('catch', v))) \
.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1),
('gen_promise', 1), ('thread_body', 2),
('gen_promise', 2), ('thread_body', 3),
('gen_promise', 3), ('thread_body', 4),
('gen_promise', 4), ('thread_body', 5)]
)
value_list = []
gen_promise(0) \
.then(lambda v: gen_promise(v, False)) \
.then(lambda v: gen_promise(v)) \
.catch(lambda v: value_list.append(('catch', v))) \
.wait()
self.assertListEqual(
value_list,
[('gen_promise', 0), ('thread_body', 1),
('gen_promise', 1), ('thread_body', 2),
('catch', 2)]
)
finally:
gc.collect()
self.assertDictEqual(promise._promise_pool, {})
req_queue.put((None, None, None))
