def test_synchronization_coordinator_abandon(): mo = MultiObject(range(3)) sync = SynchronizationCoordinator(len(mo)) executed = [] def foo(i): def execute(caption): executed.append((i, caption)) sync.wait_for_everyone() execute('after wait 1') if i == 2: sync.abandon() return # Only two waiters should reach here sync.wait_for_everyone() execute('after wait 2') # Even without explicit call to abandon, sync should only wait for two waiters sync.wait_for_everyone() execute('after wait 3') mo.call(foo) verify_concurrent_order(executed, {(i, 'after wait 1') for i in range(3)}, {(i, 'after wait 2') for i in range(2)}, {(i, 'after wait 3') for i in range(2)})
def test_synchronization_coordinator_wait_for_everyone(): mo = MultiObject(range(3)) sync = SynchronizationCoordinator(len(mo)) executed = [] def foo(i): def execute(caption): executed.append((i, caption)) sleep(i / 10) execute('after sleep') sync.wait_for_everyone() execute('after wait') sync.wait_for_everyone() sleep(i / 10) execute('after sleep 2') sync.wait_for_everyone() execute('after wait 2') mo.call(foo) verify_concurrent_order(executed, {(i, 'after sleep') for i in range(3)}, {(i, 'after wait') for i in range(3)}, {(i, 'after sleep 2') for i in range(3)}, {(i, 'after wait 2') for i in range(3)})
def test_synchronization_coordinator_collect_and_call_once(): mo = MultiObject(range(3)) sync = SynchronizationCoordinator(len(mo)) executed = [] def foo(i): def execute(caption): executed.append((i, caption)) sleep(i / 10) def func_to_call_once(param): executed.append('params = %s' % sorted(param)) return sum(param) result = sync.collect_and_call_once(i + 1, func_to_call_once) execute('result is %s' % result) assert sync.collect_and_call_once( i, len) == 3, 'parameters remain from previous call' mo.call(foo) verify_concurrent_order(executed, {'params = [1, 2, 3]'}, {(i, 'result is 6') for i in range(3)})
def test_synchronization_coordinator_with_multiobject_exception(): mo = MultiObject(range(3)) executed = [] class MyException(Exception): pass def foo(i, _sync=SYNC): def execute(caption): executed.append((i, caption)) _sync.wait_for_everyone() execute('after wait') if i == 2: raise MyException _sync.wait_for_everyone() execute('after wait/abandon') with pytest.raises(MultiException) as exc: mo.call(foo) assert exc.value.count == 1 assert exc.value.common_type is MyException verify_concurrent_order(executed, {(i, 'after wait') for i in range(3)}, {(i, 'after wait/abandon') for i in range(2)})
def test_synchronization_coordinator_with_multiobject(): mo = MultiObject(range(3)) executed = [] def foo(i, _sync=SYNC): def execute(caption): executed.append((i, caption)) sleep(i / 10) _sync.wait_for_everyone() execute('after wait') def func_to_call_once(param): executed.append('params = %s' % sorted(param)) return sum(param) result = _sync.collect_and_call_once(i + 1, func_to_call_once) execute('result is %s' % result) foo(10) assert executed == [(10, 'after wait'), 'params = [11]', (10, 'result is 11')] executed.clear() mo.call(foo) verify_concurrent_order(executed, {(i, 'after wait') for i in range(3)}, {'params = [1, 2, 3]'}, {(i, 'result is 6') for i in range(3)})
def test_multiexception_types(): class OK(Exception): pass class BAD(object): pass class OKBAD(OK, BAD): pass with pytest.raises(AssertionError): MultiException[BAD] def raise_it(typ): raise typ() with pytest.raises(MultiException[OK]): MultiObject([OK]).call(raise_it) with pytest.raises(MultiException[OKBAD]): MultiObject([OKBAD]).call(raise_it) with pytest.raises(MultiException[OK]): MultiObject([OKBAD]).call(raise_it)
def test_multiobject_concurrent_find_proper_shutdown(): executed = [] m = MultiObject(range(10), workers=1) ret = m.concurrent_find(lambda n: [print(n) or executed.append(n) or sleep(.01)]) assert ret sleep(1) # wait for potential stragglers assert max(executed) <= 2
def test_multiobject_exceptions(): assert MultiException[ValueError] is MultiException[ValueError] assert issubclass(MultiException[UnicodeDecodeError], MultiException[UnicodeError]) assert issubclass(MultiException[UnicodeDecodeError], MultiException[ValueError]) with pytest.raises(AssertionError): MultiException[0] with pytest.raises(MultiException): MultiObject(range(5)).call(lambda n: 1 / n) with pytest.raises(MultiException[Exception]): MultiObject(range(5)).call(lambda n: 1 / n) with pytest.raises(MultiException[ZeroDivisionError]): MultiObject(range(5)).call(lambda n: 1 / n) try: MultiObject(range(5)).call(lambda n: 1 / n) except MultiException[ValueError] as exc: assert False except MultiException[ZeroDivisionError] as exc: assert len(exc.actual) == 1 assert isinstance(exc.one, ZeroDivisionError) else: assert False with pytest.raises(MultiException[ArithmeticError]): try: MultiObject(range(5)).call(lambda n: 1 / n) except ZeroDivisionError: assert False # shouldn't be here except MultiException[ValueError]: assert False # shouldn't be here
def test_multiobject_types(): assert isinstance(MultiObject(range(5)), MultiObject[int]) assert not isinstance(MultiObject(range(5)), MultiObject[str]) class A(): ... class B(A): ... assert issubclass(MultiObject[A], MultiObject) assert not issubclass(MultiObject[A], A) assert issubclass(MultiObject[B], MultiObject[A]) assert not issubclass(MultiObject[A], MultiObject[B]) assert isinstance(MultiObject([B()]), MultiObject[A]) assert not isinstance(MultiObject([A()]), MultiObject[B]) assert isinstance(MultiObject[A]([B()]), MultiObject[A]) assert isinstance(MultiObject[A]([B()]), MultiObject[B]) assert isinstance(MultiObject[int](range(5)), MultiObject[int]) with pytest.raises(TypeError): assert MultiObject[str](range(5)) assert isinstance(MultiObject[str]("123").call(int), MultiObject[int])
def test_synchronization_coordinator_with_context_manager(): mo = MultiObject(range(3)) executed = [] @contextmanager def foo(i, _sync=SYNC): def execute(caption): executed.append((i, caption)) sleep(i / 10) execute('after sleep') _sync.wait_for_everyone() execute('before yield') yield _sync.wait_for_everyone() execute('after yield') with mo.call(foo): executed.append('with body') verify_concurrent_order(executed, {(i, 'after sleep') for i in range(3)}, {(i, 'before yield') for i in range(3)}, {'with body'}, {(i, 'after yield') for i in range(3)})
def test_multiobject_enumerate(): m = MultiObject(range(5), log_ctx="abcd") def check(i, j): assert i == j + 1 e = m.enumerate(1) assert e._log_ctx == tuple("abcd") e.call(check)
def __call__(self, **kwargs): if self.log: # log signal for centralized logging analytics. # minimize text message as most of the data is sent in the 'extra' dict # signal fields get prefixed with 'sig_', and the values are repr'ed signal_fields = {("sig_%s" % k): repr(v) for (k, v) in kwargs.items()} _logger.debug("Triggered '%s' (%s) - entering", self.name, self.id, extra=dict(signal_fields, signal=self.name, signal_id=self.id)) for handler in self.iter_handlers(): if handler.times == 0: self.unregister(handler.func) kwargs.setdefault('swallow_exceptions', self.swallow_exceptions) with ExitStack() as handlers_stack: handlers_to_remove = [] def should_run(handler): try: return handler.should_run(**kwargs) except STALE_HANDLER: pass handlers_to_remove.append(handler) return False indexer = count() @contextmanager def run_handler(handler): index = next(indexer) already_yielded = False try: with _logger.context(self.id), _logger.context("%02d" % index): with handler(**kwargs): yield already_yielded = True except STALE_HANDLER: handlers_to_remove.append(handler) if not already_yielded: yield for async_handlers, synced_handlers in self.iter_handlers_by_priority(): async_handlers = MultiObject(filter(should_run, async_handlers)) synced_handlers = list(filter(should_run, synced_handlers)) if async_handlers: handlers_stack.enter_context(MultiObject(async_handlers).call(run_handler)) for handler in synced_handlers: handlers_stack.enter_context(run_handler(handler)) yield self.remove_handlers_if_exist(handlers_to_remove)
def test_synchronization_coordinator_timeout(): mo = MultiObject(range(3)) def foo(i, _sync=SYNC): sleep(i / 10) _sync.wait_for_everyone(timeout=0.1) with pytest.raises(MultiException) as exc: mo.call(foo) assert exc.value.count == len(mo) assert exc.value.common_type is threading.BrokenBarrierError
def test_synchronization_coordinator_with_multiobject_method(): class Foo: def __init__(self, i): self.i = i def foo(self, _sync=SYNC): return (self.i, _sync.collect_and_call_once( self.i, lambda i_values: sorted(i_values))) mo = MultiObject(Foo(i) for i in range(3)) assert mo.foo().T == ((0, [0, 1, 2]), (1, [0, 1, 2]), (2, [0, 1, 2]))
def test_multiobject_logging(): m = MultiObject(range(4), log_ctx="abcd", initial_log_interval=0.1) def check(i): sleep(.2) # we'll mock the logger so we can ensure it logged with patch("easypy.concurrency._logger") as _logger: m.call(check) args_list = (c[0] for c in _logger.info.call_args_list) for args in args_list: assert "test_multiobject_logging.<locals>.check" == args[2] assert "easypy/tests/test_concurrency.py" in args[4]
def _poll(): nonlocal had_results results = self.poll(pending) done_idxes = [] for i, result in enumerate(results): cmd = pending[i] if result is not None: done_idxes.append(i) cmd.raise_on_failure = cmd._should_raise_on_failure yield cmd, result for i in sorted( done_idxes, reverse=True ): # Alternative would be to copy pending list, but this is more efficient pending.pop(i) if pending: if logging_timer.expired: logging_timer.backoff() MultiObject(pending).check_client_timeout() hosts = sorted(set(cmd.hostname for cmd in pending)) _logger.info("Waiting on %s command(s) on %s host(s): %s", len(pending), len(hosts), ", ".join(hosts)) for cmd in pending: since_started = ( "no-ack" if not cmd.ack_supported else "acked {:ago}".format(cmd.since_started) if cmd.ack else "not-started") _logger.debug(" job-id: %s (%s)", cmd.job_id, since_started) if done_idxes: had_results = True
def get_output(self, cmds, decode='utf-8'): """ Get commands' output so far :param [List[Cmd]] cmds: The list of commands to get their output :param [str] decode: Expected output encoding :returns: A MultiObject of tuples containing stdout and stderr :rtype: MultiObject[Tuple[str, str]] """ ret = [] with self.redis.pipeline() as p: for cmd in cmds: cmd._request_outputs(p) self._pipeline_flush_log(p) results = p.execute() p.reset() for i, (stdout, stderr) in enumerate(chunkify(results, 2)): cmd = cmds.L[i] cmd.on_output(cmd.stdout, stdout) cmd.on_output(cmd.stderr, stderr) cmd._trim_outputs(stdout, stderr, pipeline=p) if decode: (stdout, stderr) = (cmd._decode_output(out, decode) for out in (stdout, stderr)) ret.append((stdout, stderr)) self._pipeline_flush_log(p) p.execute() return MultiObject(ret)
def test_tag_along_thread(n): from random import random data = set() counter = 0 called = 0 def _get_data(): nonlocal called called += 1 sleep(0.05 * random() + 0.05) return data get_data = TagAlongThread(_get_data, 'counter-incrementer') def change_and_verify(): nonlocal counter counter += 1 data.add(counter) # ensure that the data we get from TagAlongThread includes our update assert counter in get_data() # ensure that regardless of how many threads there were, # we do not trigger more than 2 invocations of _get_data MultiObject(range(n)).call(lambda _: change_and_verify()) assert called <= 2
def test_sync_singleton(): class S(metaclass=SynchronizedSingleton): def __init__(self): sleep(1) a, b = MultiObject(range(2)).call(lambda _: S()) assert a is b
def test_multiobject_zip_with(): m = MultiObject(range(4)) with pytest.raises(AssertionError): m.zip_with(range(3), range(5)) # too few objects m.zip_with(range(5), range(6)) # too many objects ret = m.zip_with(range(1, 5)).call(lambda a, b: a + b).T assert ret == (1, 3, 5, 7)
def test_synchronization_coordinator_exception_in_collect_and_call_once(): mo = MultiObject(range(3)) sync = SynchronizationCoordinator(len(mo)) times_called = 0 class MyException(Exception): pass def foo(i): def func_to_call_once(_): nonlocal times_called times_called += 1 raise MyException with pytest.raises(MyException): sync.collect_and_call_once(i, func_to_call_once) assert sync.collect_and_call_once(i + 1, sum) == 6 mo.call(foo) assert times_called == 1, 'collect_and_call_once with exception called the function more than once'
def test_multiobject_namedtuples(): from collections import namedtuple class Something(namedtuple("Something", "a b")): pass def ensure_not_expanded(something): # This will probably fail before these asserts assert hasattr(something, 'a') assert hasattr(something, 'b') objects = [Something(1, 2), Something(2, 3), Something(3, 4)] MultiObject(objects).call(ensure_not_expanded)
def test_multiobject_concurrent_find_not_found(): m = MultiObject(range(10)) ret = m.concurrent_find(lambda n: n < 0) assert ret is False m = MultiObject([0] * 5) ret = m.concurrent_find(lambda n: n) assert ret is 0
def __call__(self, **kwargs): if self.log: # log signal for centralized logging analytics. # minimize text message as most of the data is sent in the 'extra' dict # signal fields get prefixed with 'sig_', and the values are repr'ed signal_fields = {("sig_%s" % k): repr(v) for (k, v) in kwargs.items()} _logger.debug("Triggered '%s' (%s) - entering", self.name, self.id, extra=dict(signal_fields, signal=self.name, signal_id=self.id)) for handler in self.iter_handlers(): if handler.times == 0: self.unregister(handler.func) kwargs.setdefault('swallow_exceptions', self.swallow_exceptions) with ExitStack() as handlers_stack: async_handlers = MultiObject() handlers = [] for index, handler in enumerate(self.iter_handlers()): # allow handler to use our async context handler = _logger.context("%02d" % index)(handler) handler = _logger.context(self.id)(handler) if handler. async: async_handlers.append(handler) else: handlers.append(handler) if async_handlers: handlers_stack.enter_context(async_handlers(**kwargs)) for handler in handlers: res = handler(**kwargs) if res: handlers_stack.enter_context(res) yield
def test_multiexception_api(): with pytest.raises(MultiException) as exc: MultiObject([0, 5]).call(lambda i: 10 // i) failed, sucsessful = exc.value.futures assert failed.done() with pytest.raises(ZeroDivisionError): failed.result() assert isinstance(failed.exception(), ZeroDivisionError) assert sucsessful.done() assert sucsessful.result() == 2 assert sucsessful.exception() is None
def test_synchronization_coordinator_with_multiobject_early_return(): mo = MultiObject(range(3)) executed = [] def foo(i, _sync=SYNC): def execute(caption): executed.append((i, caption)) _sync.wait_for_everyone() execute('after wait') if i == 2: return _sync.wait_for_everyone() execute('after wait/abandon') mo.call(foo) verify_concurrent_order(executed, {(i, 'after wait') for i in range(3)}, {(i, 'after wait/abandon') for i in range(2)})
def test_thread_contexts_counters_multiobject(): TC = ThreadContexts(counters=('i', )) assert TC.i == 0 print("---") @TC(i=True) def test(n): print(n, TC._context_data) sleep(.1) return TC.i test(0) ret = MultiObject(range(10)).call(test) assert set(ret) == {1}
def test_multiobject_1(): m = MultiObject(range(10)) def mul(a, b, *c): return a * b + sum(c) assert sum(m.call(mul, 2)) == 90 assert sum(m.call(mul, b=10)) == 450 assert sum(m.call(mul, 1, 1, 1)) == 65 assert m.filter(None).T == (1, 2, 3, 4, 5, 6, 7, 8, 9) assert sum(m.denominator) == 10 with pytest.raises(MultiException) as info: m.call(lambda i: 1 / (i % 2)) assert info.value.count == 5 assert info.value.common_type == ZeroDivisionError assert not info.value.complete
def test_synchronization_coordinator_failing_context_manager(): class MyException(Exception): pass @contextmanager def foo(should_fail, _sync=SYNC): if should_fail: raise MyException() else: yield inside_executed = False with pytest.raises(MultiException[MyException]): with MultiObject([False, True]).call(foo): inside_executed = True assert not inside_executed, 'CM body executed even though __enter__ failed in one thread'
def test_locking_timecache(): from easypy.concurrency import MultiObject # Cached func should be called only once value_generated = False class UnnecessaryFunctionCall(Exception): pass @timecache(ignored_keywords='x') def test(x): nonlocal value_generated if value_generated: raise UnnecessaryFunctionCall() value_generated = True return True MultiObject(range(10)).call(lambda x: test(x=x))