def test_get_put_get_then_get_put_get_works_correctly_with_conflicting_keys_multi_thread(
self):
base_cacher = MaxCountCacher(MemoryCacher(), pause_once_on="get")
curr_cacher = ConcurrentCacher(base_cacher, {}, threading.Lock(),
threading.Condition())
curr_cacher.put(1, 1)
def thread_1():
curr_cacher.get_put(1, lambda: 1)
def thread_2():
base_cacher.wait()
curr_cacher.get_put(1, lambda: 2)
base_cacher.release()
t1 = threading.Thread(None, thread_1)
t2 = threading.Thread(None, thread_2)
t1.daemon = True
t2.daemon = True
t1.start()
t2.start()
t1.join()
t2.join()
self.assertEqual(2, base_cacher.max_count)
self.assertEqual(curr_cacher.get(1), 1)
def test_get_put_put_then_get_put_get_iter_works_correctly_sans_conflicting_keys_multi_thread(
self):
base_cacher = MaxCountCacher(IterCacher(), pause_once_on="get_put")
curr_cacher = ConcurrentCacher(base_cacher, {}, threading.Lock(),
threading.Condition())
def thread_1():
curr_cacher.get_put(1, lambda: [1])
def thread_2():
base_cacher.wait()
curr_cacher.get_put(2, lambda: [2])
base_cacher.release()
t1 = threading.Thread(None, thread_1)
t2 = threading.Thread(None, thread_2)
t1.daemon = True
t2.daemon = True
t1.start()
t2.start()
t1.join()
t2.join()
self.assertEqual(2, base_cacher.max_count)
self.assertEqual(list(curr_cacher.get(1)), [1])
self.assertEqual(list(curr_cacher.get(2)), [2])
def test_put_then_put_works_correctly_with_conflicting_keys_multi_thread(
self):
base_cacher = MaxCountCacher(MemoryCacher(), pause_once_on="put")
curr_cacher = ConcurrentCacher(base_cacher, {}, threading.Lock(),
threading.Condition())
def thread_1():
curr_cacher.put(1, 2)
def thread_2():
base_cacher.wait()
curr_cacher.put(1, 3)
t1 = threading.Thread(None, thread_1)
t2 = threading.Thread(None, thread_2)
t1.daemon = True
t2.daemon = True
t1.start()
t2.start()
while curr_cacher.write_waits != 1:
time.sleep(0.01)
base_cacher.release()
t1.join()
t2.join()
self.assertEqual(1, base_cacher.max_count)
self.assertEqual(curr_cacher.get(1), 3)
def test_get_iter_works_correctly_single_thread(self):
cacher = ConcurrentCacher(IterCacher(), {}, threading.Lock(),
threading.Condition())
cacher.put("abc", [1, 2, 3])
self.assertEqual(list(cacher.get("abc")), [1, 2, 3])
self.assertEqual(list(cacher.get("abc")), [1, 2, 3])
self.assertEqual(0, cacher._dict["abc"])
def test_get_during_get_put_same_process_causes_exception(self):
base_cacher = IterCacher()
curr_cacher = ConcurrentCacher(base_cacher, {}, threading.Lock(),
threading.Condition())
def getter():
yield 1
curr_cacher.get(1)
yield 2
with self.assertRaises(CobaException):
curr_cacher.get_put(1, getter)
def test_rmv_works_correctly_single_thread(self):
cacher = ConcurrentCacher(MemoryCacher(), {}, threading.Lock(),
threading.Condition())
cacher.put("abc", "abcd")
self.assertIn("abc", cacher)
cacher.rmv("abc")
self.assertNotIn("abc", cacher)
cacher.rmv("abc")
def filter(self, items: Iterable[Any]) -> Iterable[Any]:
try:
with Manager() as manager:
stdlog = QueueIO(Queue())
stderr = CobaMultiprocessor.PipeStderr()
log_thread = Thread(target=Pipes.join(
stdlog, Foreach(CobaContext.logger.sink)).run)
log_thread.daemon = True
log_thread.start()
logger = CobaContext.logger
cacher = ConcurrentCacher(CobaContext.cacher, manager.dict(),
Lock(), Condition())
store = {"srcsema": Semaphore(2)}
filter = CobaMultiprocessor.ProcessFilter(
self._filter, logger, cacher, store, stdlog)
for item in Multiprocessor(filter, self._processes,
self._maxtasksperchild,
stderr).filter(items):
yield item
stdlog.write(
None
) #attempt to shutdown the logging process gracefully by sending the poison pill
except RuntimeError as e: #pragma: no cover
#This happens when importing main causes this code to run again
coba_exit(str(e))
def test_rmv_during_get_same_process_with_release(self):
base_cacher = IterCacher()
curr_cacher = ConcurrentCacher(base_cacher, {}, threading.Lock(),
threading.Condition())
curr_cacher.put(1, range(4))
try:
iter_1 = curr_cacher.get(1)
next(iter_1)
raise Exception("Something unexpected went wrong while reading")
except:
curr_cacher.release(1)
curr_cacher.rmv(1)
def test_rmv_during_get_same_process_causes_exception(self):
base_cacher = IterCacher()
curr_cacher = ConcurrentCacher(base_cacher, {}, threading.Lock(),
threading.Condition())
curr_cacher.put(1, range(4))
iter_1 = curr_cacher.get(1)
next(iter_1)
with self.assertRaises(CobaException):
curr_cacher.rmv(1)
