def _impl_test_pool_results_discarded():
"""
After a RequestPool executes, none of its data should linger if the user didn't hang on to it.
"""
import weakref
from functools import partial
import threading
result_refs = []
def workload():
# In this test, all results are discarded immediately after the
# request exits. Therefore, AT NO POINT IN TIME, should more than N requests be alive.
live_result_refs = [w for w in result_refs if w() is not None]
assert (
len(live_result_refs) <= Request.global_thread_pool.num_workers
), "There should not be more than {} result references alive at one time!".format(
Request.global_thread_pool.num_workers
)
return numpy.zeros((10,), dtype=numpy.uint8) + 1
lock = threading.Lock()
def handle_result(req, result):
with lock:
result_refs.append(weakref.ref(result))
def handle_cancelled(req, *args):
assert False
def handle_failed(req, exc, exc_info):
raise exc
pool = RequestPool()
for _ in range(100):
req = Request(workload)
req.notify_finished(partial(handle_result, req))
req.notify_cancelled(partial(handle_cancelled, req))
req.notify_failed(partial(handle_failed, req))
pool.add(req)
del req
pool.wait()
# This test verifies that
# (1) references to all child requests have been discarded once the pool is complete, and
# (2) therefore, all references to the RESULTS in those child requests are also discarded.
# There is a tiny window of time between a request being 'complete' (for all intents and purposes),
# but before its main execute function has exited back to the main ThreadPool._Worker loop.
# The request is not finally discarded until that loop discards it, so let's wait a tiny extra bit of time.
time.sleep(0.01)
# Now check that ALL results are truly lost.
for ref in result_refs:
assert ref() is None, "Some data was not discarded."
def _impl_test_pool_results_discarded():
"""
After a RequestPool executes, none of its data should linger if the user didn't hang on to it.
"""
import weakref
from functools import partial
import threading
result_refs = []
def workload():
# In this test, all results are discarded immediately after the
# request exits. Therefore, AT NO POINT IN TIME, should more than N requests be alive.
live_result_refs = [w for w in result_refs if w() is not None]
assert (
len(live_result_refs) <= Request.global_thread_pool.num_workers
), "There should not be more than {} result references alive at one time!".format(
Request.global_thread_pool.num_workers
)
return numpy.zeros((10,), dtype=numpy.uint8) + 1
lock = threading.Lock()
def handle_result(req, result):
with lock:
result_refs.append(weakref.ref(result))
def handle_cancelled(req, *args):
assert False
def handle_failed(req, exc, exc_info):
raise exc
pool = RequestPool()
for _ in range(100):
req = Request(workload)
req.notify_finished(partial(handle_result, req))
req.notify_cancelled(partial(handle_cancelled, req))
req.notify_failed(partial(handle_failed, req))
pool.add(req)
del req
pool.wait()
# This test verifies that
# (1) references to all child requests have been discarded once the pool is complete, and
# (2) therefore, all references to the RESULTS in those child requests are also discarded.
# There is a tiny window of time between a request being 'complete' (for all intents and purposes),
# but before its main execute function has exited back to the main ThreadPool._Worker loop.
# The request is not finally discarded until that loop discards it, so let's wait a tiny extra bit of time.
time.sleep(0.01)
# Now check that ALL results are truly lost.
for ref in result_refs:
assert ref() is None, "Some data was not discarded."
def test_dont_cancel_shared_request(self):
"""
Test that a request isn't cancelled if it has requests pending for it.
"""
if Request.global_thread_pool.num_workers == 0:
raise nose.SkipTest
cancelled_requests = []
def f1():
time.sleep(1)
return "RESULT"
r1 = Request(f1)
r1.notify_cancelled(partial(cancelled_requests.append, 1))
def f2():
try:
return r1.wait()
except:
cancelled_requests.append(2)
r2 = Request(f2)
def f3():
try:
return r1.wait()
except:
cancelled_requests.append(3)
r3 = Request(f3)
def otherThread():
r2.wait()
t = threading.Thread(target=otherThread)
t.start()
r3.submit()
time.sleep(0.5)
# By now both r2 and r3 are waiting for the result of r1
# Cancelling r3 should not cancel r1.
r3.cancel()
t.join() # Wait for r2 to finish
time.sleep(0.5)
assert r1.started
assert r1.finished
assert not r1.cancelled # Not cancelled, even though we cancelled a request that was waiting for it.
assert 1 not in cancelled_requests
assert r2.started
assert r2.finished
assert not r2.cancelled # Not cancelled.
assert 1 not in cancelled_requests
assert r2.wait() == "RESULT"
assert r3.started
assert r3.finished
assert r3.cancelled # Successfully cancelled.
assert 3 in cancelled_requests
def test_dont_cancel_shared_request(self):
"""
Test that a request isn't cancelled if it has requests pending for it.
"""
cancelled_requests = []
def f1():
time.sleep(1)
return "RESULT"
r1 = Request(f1)
r1.notify_cancelled( partial(cancelled_requests.append, 1) )
def f2():
try:
return r1.wait()
except:
cancelled_requests.append(2)
r2 = Request(f2)
def f3():
try:
return r1.wait()
except:
cancelled_requests.append(3)
r3 = Request(f3)
def otherThread():
r2.wait()
t = threading.Thread(target=otherThread)
t.start()
r3.submit()
time.sleep(0.5)
# By now both r2 and r3 are waiting for the result of r1
# Cancelling r3 should not cancel r1.
r3.cancel()
t.join() # Wait for r2 to finish
time.sleep(0.5)
assert r1.started
assert r1.finished
assert not r1.cancelled # Not cancelled, even though we cancelled a request that was waiting for it.
assert 1 not in cancelled_requests
assert r2.started
assert r2.finished
assert not r2.cancelled # Not cancelled.
assert 1 not in cancelled_requests
assert r2.wait() == "RESULT"
assert r3.started
assert r3.finished
assert r3.cancelled # Successfully cancelled.
assert 3 in cancelled_requests
