def destroy(self):
if self._ref and self._ref():
ref = self._ref() # grab the ref before we stop, otherwise ref() returns a dead ref
self.stopped = True
ref._cell = None
self._ref = None
else:
self.stopped = True
ref = self.ref
while True:
try:
sender, m = self.queue.get_nowait()
except gevent.queue.Empty:
break
if m == ('_watched', ANY):
self._watched(m[1])
elif m == ('__error', ANY, ANY):
_, exc, tb = m
self.report((exc, tb))
elif not (m == ('terminated', ANY) or m == ('_unwatched', ANY) or m == ('_node_down', ANY) or m == '_stop' or m == '_kill' or m == '__done' or m == '__undone'):
Events.log(DeadLetter(ref, m, sender))
self.parent_actor.send(('_child_terminated', ref))
for watcher in (self.watchers or []):
watcher << ('terminated', ref)
self.actor = self.inbox = self.queue = self.parent_actor = None
def _remote_dead_letter(self, path, msg, sender):
ref = Ref(cell=None,
uri=Uri.parse(self.nid + path),
node=self,
is_local=True)
if not (msg == ('_unwatched', ANY) or msg == ('_watched', ANY)):
Events.log(DeadLetter(ref, msg, sender))
def send(self, message, _sender=None):
"""Sends a message to the actor represented by this `Ref`."""
if not _sender:
context = get_context()
if context:
_sender = context.ref
if self._cell:
if not self._cell.stopped:
self._cell.receive(message, _sender)
return
else:
self._cell = None
if not self.is_local:
if self.uri.node != self.node.nid:
self.node.send_message(message, remote_ref=self, sender=_sender)
else:
self._cell = self.node.guardian.lookup_cell(self.uri)
self.is_local = True
self._cell.receive(message, _sender)
else:
if self.node and self.node.guardian:
cell = self.node.guardian.lookup_cell(self.uri)
if cell:
cell.receive(message, _sender) # do NOT set self._cell--it will never be unset and will cause a memleak
return
if ('_watched', ANY) == message:
message[1].send(('terminated', self), _sender=self)
elif (message == ('terminated', ANY) or message == ('_unwatched', ANY) or message == ('_node_down', ANY) or
message == '_stop' or message == '_kill' or message == '__done'):
pass
else:
Events.log(DeadLetter(self, message, _sender))
def destroy(self):
if self._ref and self._ref():
ref = self._ref(
) # grab the ref before we stop, otherwise ref() returns a dead ref
self.stopped = True
ref._cell = None
self._ref = None
else:
self.stopped = True
ref = self.ref
while True:
try:
sender, m = self.queue.get_nowait()
except gevent.queue.Empty:
break
if m == ('_watched', ANY):
self._watched(m[1])
elif m == ('__error', ANY, ANY):
_, exc, tb = m
self.report((exc, tb))
elif not (m == ('terminated', ANY) or m == ('_unwatched', ANY)
or m == ('_node_down', ANY) or m == '_stop'
or m == '_kill' or m == '__done' or m == '__undone'):
Events.log(DeadLetter(ref, m, sender))
self.parent_actor.send(('_child_terminated', ref))
for watcher in (self.watchers or []):
watcher << ('terminated', ref)
self.actor = self.inbox = self.queue = self.parent_actor = None
def start_actor():
if self._nodeid:
Events.log(Message("Setting up remoting; node ID = %s" % (self._nodeid,)))
try:
f1 = ZmqFactory()
insock = ZmqPullConnection(f1)
outsock = lambda: ZmqPushConnection(f1, linger=0)
hub = Hub(insock, outsock, nodeid=self._nodeid)
except Exception:
err("Could not set up remoting")
traceback.print_exc()
reactor.stop()
return
else:
Events.log(Message("No remoting requested; specify `--remoting/-r <nodeid>` (nodeid=host:port) to set up remoting"))
hub = HubWithNoRemoting()
supervision = {'stop': Stop, 'restart': Restart, 'resume': Resume}[self._supervise]
node = Node(hub=hub, root_supervision=supervision)
try:
self._wrapper = node.spawn(Wrapper.using(
self._actor_cls.using(**self._init_params),
spawn_at=self._name, keep_running=self._keep_running
), name='_runner')
except Exception:
panic("Failed to start wrapper for %s\n" % (actor_path,),
Failure().getTraceback())
reactor.stop()
else:
if self._initial_message is not _EMPTY:
self._wrapper << ('_forward', self._initial_message)
def send(self, ref, msg):
if self.queue is not None:
self.queue.append((ref, msg))
else:
if self.sock:
self._do_send(dumps((ref.uri.path, msg), protocol=2))
else:
Events.log(DeadLetter(ref, msg))
def _send_local(self, msg, ref):
cell = self.guardian.lookup_cell(ref.uri)
if cell:
ref._cell = cell
ref.is_local = True
ref << msg
else:
ref.is_local = True # next time, just put it straight to DeadLetters
if msg not in (("terminated", ANY), ("_watched", ANY), ("_unwatched", ANY)):
Events.log(DeadLetter(ref, msg))
def receive(self, message):
if message == ('_forward', ANY):
_, payload = message
self.actor << payload
elif message == ('terminated', self.actor):
_, actor = message
if self.keep_running:
after(1.0).do(self._do_spawn)
else:
log("actor terminated but not re-spawning actor; pass --keepruning/-k to change this behaviour")
self.stop()
else:
Events.log(Message("Contained actor sent a message to parent: %r" % (message,)))
def ret():
# dbg("\n============================================\n")
import spinoff.actor._actor
spinoff.actor._actor.TESTING = True
Actor.reset_flags(debug=True)
# TODO: once the above TODO (fresh Node for each test fn) is complete, consider making Events non-global by
# having each Node have its own Events instance.
Events.reset()
def check_memleaks():
if '__pypy__' not in sys.builtin_module_names:
gc.collect()
for trash in gc.garbage[:]:
if isinstance(trash, DebugInfo):
# dbg("DEBUGINFO: __del__")
if trash.failResult is not None:
exc = Unclean(repr(trash.failResult.value) + '\n' + str(trash._getDebugTracebacks()))
trash.__dict__.clear()
raise exc
gc.garbage.remove(trash)
assert not gc.garbage, "Memory leak detected: %r" % (gc.garbage,)
# if gc.garbage:
# dbg("GARGABE: detected after %s:" % (fn.__name__,), len(gc.garbage))
# import objgraph as ob
# import os
# def dump_chain(g_):
# def calling_test(x):
# if not isframework(x):
# return None
# import spinoff
# isframework = lambda x: type(x).__module__.startswith(spinoff.__name__)
# ob.show_backrefs([g_], filename='backrefs.png', max_depth=100, highlight=isframework)
# for gen in gc.garbage:
# dump_chain(gen)
# dbg(" TESTWRAP: mem-debuggin", gen)
# import pdb; pdb.set_trace()
# os.remove('backrefs.png')
return (
deferred_with(ErrorCollector(), fn)
.addBoth(lambda result: Node.stop_all().addCallback(lambda _: result))
.addBoth(lambda result: (_process_idle_calls(), result)[-1])
.addBoth(lambda result: (check_memleaks(), result)[-1])
)
def receive(self, msg):
# dbg("recv %r" % (msg,))
if self.__get_d and self.__get_d.wants(msg):
# dbg("injecting %r" % (msg,))
self.__get_d.callback(msg)
# dbg("...injectng %r OK" % (msg,))
else:
# dbg("queueing")
if not self.__queue:
self.__queue = []
self.__queue.append(msg)
l = len(self.__queue)
if l and l % self.hwm == 0:
Events.log(HighWaterMarkReached(self.ref, l))
def report(self, exc_and_tb=None):
if not exc_and_tb:
_, exc, tb = sys.exc_info()
else:
exc, tb = exc_and_tb
if not isinstance(exc, UnhandledTermination):
if isinstance(tb, str):
exc_fmt = tb
else:
exc_fmt = ''.join(traceback.format_exception(type(exc), exc, tb))
print(exc_fmt.strip(), file=sys.stderr)
else:
fail("Died because a watched actor (%r) died" % (exc.watchee,))
Events.log(Error(self.ref, exc, tb)),
def report(self, exc_and_tb=None):
if not exc_and_tb:
_, exc, tb = sys.exc_info()
else:
exc, tb = exc_and_tb
if not isinstance(exc, UnhandledTermination):
if isinstance(tb, str):
exc_fmt = tb
else:
exc_fmt = ''.join(
traceback.format_exception(type(exc), exc, tb))
print(exc_fmt.strip(), file=sys.stderr)
else:
fail("Died because a watched actor (%r) died" % (exc.watchee, ))
Events.log(Error(self.ref, exc, tb)),
def run(self, req, responder, args, kwargs):
try:
self.responder = responder = self.spawn(responder.using(req, *args, **kwargs))
self.error = None
try:
Events.subscribe(Error, self.check_error) # XXX: it would be nicer if Events.subscribe accepted an actor ref to filter on
responder.join()
if not req.closed:
if self.error:
_send_500(req, extra='\n<pre>\n%s</pre>\n' % (''.join(traceback.format_exception(type(self.error.exc), self.error.exc, self.error.tb)),))
req.close()
finally:
Events.unsubscribe(Error, self.check_error)
except:
_send_500(req)
raise
def test_queued_messages_are_logged_as_deadletters_after_stop(defer):
node = DummyNode()
defer(node.stop)
deadletter_event_emitted = Events.consume_one(DeadLetter)
a = node.spawn(Actor)
a.stop()
a << 'dummy'
eq_(deadletter_event_emitted.get(), DeadLetter(a, 'dummy', sender=None))
def test_queued_messages_are_logged_as_deadletters_after_stop(defer):
node = DummyNode()
defer(node.stop)
deadletter_event_emitted = Events.consume_one(DeadLetter)
a = node.spawn(Actor)
a.stop()
a << 'dummy'
eq_(deadletter_event_emitted.get(), DeadLetter(a, 'dummy', sender=None))
def expect_event_not_emitted(ev, during=0.001):
result = Events.consume_one(type(ev) if not isinstance(ev, type) else ev)
yield
sleep(during)
ok_(
not result.ready() or (not isinstance(result.get(), ev) if isinstance(
ev, type) else result.get() != ev),
"Event %s should not have been emitted" %
(" of type %s" % (ev.__name__, ) if isinstance(ev, type) else ev, ))
def test_stop_and_kill_messages_to_dead_actorrefs_are_discarded(defer):
node = DummyNode()
defer(node.stop)
a = node.spawn(Actor)
a.stop()
for event in ['_stop', '_kill']:
d = Events.consume_one(DeadLetter)
a << event
ok_(not d.ready(), "message %r sent to a dead actor should be discarded" % (event,))
def test_subscribe_and_unsubscribe():
errors = []
Events.subscribe(UnhandledError, errors.append)
Events.log(UnhandledError('actor', 1, 2))
assert errors == [UnhandledError('actor', 1, 2)]
errors[:] = []
Events.unsubscribe(UnhandledError, errors.append)
event = UnhandledError('actor', 1, 2)
Events.log(event)
assert errors == []
def test_subscribe_and_unsubscribe():
errors = []
Events.subscribe(Error, errors.append)
Events.log(Error('actor', 1, 2))
assert errors == [Error('actor', 1, 2)]
errors[:] = []
Events.unsubscribe(Error, errors.append)
event = Error('actor', 1, 2)
Events.log(event)
assert errors == []
def expect_failure(exc, message=None, timeout=None):
with assert_raises(exc, message=message) as basket:
err = Events.consume_one(Error)
yield basket
try:
_, exc, tb = err.get(timeout=timeout)
except Timeout:
pass
else:
raise exc, None, tb
def expect_failure(exc, message=None, timeout=None):
with assert_raises(exc, message=message) as basket:
err = Events.consume_one(Error)
yield basket
try:
_, exc, tb = err.get(timeout=timeout)
except Timeout:
pass
else:
raise exc, None, tb
def test_stop_and_kill_messages_to_dead_actorrefs_are_discarded(defer):
node = DummyNode()
defer(node.stop)
a = node.spawn(Actor)
a.stop()
for event in ['_stop', '_kill']:
d = Events.consume_one(DeadLetter)
a << event
ok_(not d.ready(),
"message %r sent to a dead actor should be discarded" % (event, ))
def send(self, message, _sender=None):
if ('_child_terminated', ANY) == message:
_, sender = message
self._child_gone(sender)
if not self._children and self.all_children_stopped:
self.all_children_stopped.set(None)
# # XXX: find a better way to avoid Terminated messages for TempActors,
# # possibly by using a /tmp container for them
# if not str(sender.uri).startswith('/tempactor'):
# Events.log(Terminated(sender))
elif message == '_stop':
return self._do_stop()
elif message == '_kill':
return self._do_stop(kill=True)
else:
if not _sender:
context = get_context()
if context:
_sender = context.ref
Events.log(UnhandledMessage(self, message, _sender))
def test_termination_message_to_dead_actor_is_discarded(defer):
class Parent(Actor):
def pre_start(self):
self.watch(self.spawn(Actor)).stop()
self.stop()
d = Events.consume_one(DeadLetter)
node = DummyNode()
defer(node.stop)
node.spawn(Parent)
idle()
ok_(not d.ready())
def send(self, message, _sender=None):
if ('_child_terminated', ANY) == message:
_, sender = message
self._child_gone(sender)
if not self._children and self.all_children_stopped:
self.all_children_stopped.set(None)
# # XXX: find a better way to avoid Terminated messages for TempActors,
# # possibly by using a /tmp container for them
# if not str(sender.uri).startswith('/tempactor'):
# Events.log(Terminated(sender))
elif message == '_stop':
return self._do_stop()
elif message == '_kill':
return self._do_stop(kill=True)
else:
if not _sender:
context = get_context()
if context:
_sender = context.ref
Events.log(UnhandledMessage(self, message, _sender))
def test_termination_message_to_dead_actor_is_discarded(defer):
class Parent(Actor):
def pre_start(self):
self.watch(self.spawn(Actor)).stop()
self.stop()
d = Events.consume_one(DeadLetter)
node = DummyNode()
defer(node.stop)
node.spawn(Parent)
idle()
ok_(not d.ready())
def assert_event_not_emitted(ev):
d = Events.consume_one(type(ev) if not isinstance(ev, type) else ev)
try:
yield
except:
raise
else:
assert not d.called or deferred_result(d) != ev, \
"Event %s should not have been emitted" % (
(" of type %s" % (ev.__name__,)) if isinstance(ev, type) else ev,)
finally:
d.addErrback(lambda f: f.trap(CancelledError)).cancel()
def send(self, message, _sender=None):
"""Sends a message to the actor represented by this `Ref`."""
if not _sender:
context = get_context()
if context:
_sender = context.ref
if self._cell:
if not self._cell.stopped:
self._cell.receive(message, _sender)
return
else:
self._cell = None
if not self.is_local:
if self.uri.node != self.node.nid:
self.node.send_message(message,
remote_ref=self,
sender=_sender)
else:
self._cell = self.node.guardian.lookup_cell(self.uri)
self.is_local = True
self._cell.receive(message, _sender)
else:
if self.node and self.node.guardian:
cell = self.node.guardian.lookup_cell(self.uri)
if cell:
cell.receive(
message, _sender
) # do NOT set self._cell--it will never be unset and will cause a memleak
return
if ('_watched', ANY) == message:
message[1].send(('terminated', self), _sender=self)
elif (message == ('terminated', ANY)
or message == ('_unwatched', ANY)
or message == ('_node_down', ANY) or message == '_stop'
or message == '_kill' or message == '__done'):
pass
else:
Events.log(DeadLetter(self, message, _sender))
def expect_one_event(ev, timeout=None):
result = Events.consume_one(type(ev) if not isinstance(ev, type) else ev)
yield
try:
result = result.get(timeout=timeout)
except Timeout:
ok_(False,
"Event %r should have been emitted but was not" % (ev,)
if not isinstance(ev, type) else
"Event of type %s should have been emitted but was not" % (ev.__name__,))
if isinstance(ev, type):
ok_(isinstance(result, ev), "Event of type %s.%s should have been emitted but was not" % (ev.__module__, ev.__name__))
else:
eq_(result, ev, "Event %r should have been emitted but %s was" % (ev, result))
def run(self, req, responder, args, kwargs):
try:
self.responder = responder = self.spawn(
responder.using(req, *args, **kwargs))
self.error = None
try:
Events.subscribe(
Error, self.check_error
) # XXX: it would be nicer if Events.subscribe accepted an actor ref to filter on
responder.join()
if not req.closed:
if self.error:
_send_500(req,
extra='\n<pre>\n%s</pre>\n' % (''.join(
traceback.format_exception(
type(self.error.exc), self.error.exc,
self.error.tb)), ))
req.close()
finally:
Events.unsubscribe(Error, self.check_error)
except:
_send_500(req)
raise
def test_watching_self_is_noop_and_returns_self(defer):
class MyActor(Actor):
def pre_start(self):
eq_(self.watch(self.ref), self.ref)
def receive(self, message):
ok_(False)
node = DummyNode()
defer(node.stop)
a = node.spawn(MyActor)
dead_letter_emitted = Events.consume_one(DeadLetter)
a.stop()
idle()
ok_(not dead_letter_emitted.ready())
def test_watching_self_is_noop_and_returns_self(defer):
class MyActor(Actor):
def pre_start(self):
eq_(self.watch(self.ref), self.ref)
def receive(self, message):
ok_(False)
node = DummyNode()
defer(node.stop)
a = node.spawn(MyActor)
dead_letter_emitted = Events.consume_one(DeadLetter)
a.stop()
idle()
ok_(not dead_letter_emitted.ready())
def assert_one_event(ev):
d = Events.consume_one(type(ev) if not isinstance(ev, type) else ev)
try:
yield
except:
raise
else:
assert d.called, ("Event %r should have been emitted but was not" % (ev,)
if not isinstance(ev, type) else
"Event of type %s should have been emitted but was not" % (ev.__name__,))
result = deferred_result(d)
if isinstance(ev, type):
assert isinstance(result, ev), "Event of type %s.%s should have been emitted but was not" % (ev.__module__, ev.__name__)
else:
assert result == ev, "Event %r should have been emitted but %s was" % (ev, result)
finally:
d.addErrback(lambda f: f.trap(CancelledError)).cancel()
def expect_one_event(ev, timeout=None):
result = Events.consume_one(type(ev) if not isinstance(ev, type) else ev)
yield
try:
result = result.get(timeout=timeout)
except Timeout:
ok_(
False, "Event %r should have been emitted but was not" %
(ev, ) if not isinstance(ev, type) else
"Event of type %s should have been emitted but was not" %
(ev.__name__, ))
if isinstance(ev, type):
ok_(
isinstance(result, ev),
"Event of type %s.%s should have been emitted but was not" %
(ev.__module__, ev.__name__))
else:
eq_(result, ev,
"Event %r should have been emitted but %s was" % (ev, result))
def ret(*args, **kwargs):
# dbg("\n============================================\n")
# TODO: once the above TODO (fresh Node for each test fn) is complete, consider making Events non-global by
# having each Node have its own Events instance.
Events.reset()
def check_memleaks():
if '__pypy__' not in sys.builtin_module_names:
gc.collect()
assert not gc.garbage, "Memory leak detected: %r" % (
gc.garbage, )
# if gc.garbage:
# dbg("GARGABE: detected after %s:" % (fn.__name__,), len(gc.garbage))
# import objgraph as ob
# import os
# def dump_chain(g_):
# isframework = lambda x: type(x).__module__.startswith(spinoff.__name__)
# ob.show_backrefs([g_], filename='backrefs.png', max_depth=100, highlight=isframework)
# for gen in gc.garbage:
# dump_chain(gen)
# dbg(" TESTWRAP: mem-debuggin", gen)
# import pdb; pdb.set_trace()
# os.remove('backrefs.png')
errors = []
Events.subscribe(Error, errors.append)
try:
return with_timeout(
fn.timeout if hasattr(fn, 'timeout') else 0.5, fn, *args,
**kwargs)
finally:
Events.unsubscribe(Error, errors.append)
if errors:
sender, exc, tb = errors[0]
raise exc, None, tb
del errors[:]
check_memleaks()
def ret(*args, **kwargs):
# dbg("\n============================================\n")
# TODO: once the above TODO (fresh Node for each test fn) is complete, consider making Events non-global by
# having each Node have its own Events instance.
Events.reset()
def check_memleaks():
if '__pypy__' not in sys.builtin_module_names:
gc.collect()
assert not gc.garbage, "Memory leak detected: %r" % (gc.garbage,)
# if gc.garbage:
# dbg("GARGABE: detected after %s:" % (fn.__name__,), len(gc.garbage))
# import objgraph as ob
# import os
# def dump_chain(g_):
# isframework = lambda x: type(x).__module__.startswith(spinoff.__name__)
# ob.show_backrefs([g_], filename='backrefs.png', max_depth=100, highlight=isframework)
# for gen in gc.garbage:
# dump_chain(gen)
# dbg(" TESTWRAP: mem-debuggin", gen)
# import pdb; pdb.set_trace()
# os.remove('backrefs.png')
errors = []
Events.subscribe(Error, errors.append)
try:
return with_timeout(fn.timeout if hasattr(fn, 'timeout') else 0.5, fn, *args, **kwargs)
finally:
Events.unsubscribe(Error, errors.append)
if errors:
sender, exc, tb = errors[0]
raise exc, None, tb
del errors[:]
check_memleaks()
def _remote_dead_letter(self, path, msg, from_):
uri = Uri.parse(self.nodeid + path)
ref = Ref(cell=self.guardian.lookup_cell(uri), uri=uri, is_local=True)
Events.log(RemoteDeadLetter(ref, msg, from_))
def expect_event_not_emitted(ev, during=0.001):
result = Events.consume_one(type(ev) if not isinstance(ev, type) else ev)
yield
sleep(during)
ok_(not result.ready() or (not isinstance(result.get(), ev) if isinstance(ev, type) else result.get() != ev),
"Event %s should not have been emitted" % (" of type %s" % (ev.__name__,) if isinstance(ev, type) else ev,))
def send_failed(self):
if not (self.msg == ('_unwatched', ANY) or self.msg == ('_watched', ANY)):
Events.log(DeadLetter(self.ref, self.msg, self.sender))
def unhandled(self, m, sender):
if ('terminated', ANY) == m:
raise UnhandledTermination(watcher=self.ref, watchee=m[1])
else:
Events.log(UnhandledMessage(self.ref, m, sender))
def send_failed(self):
if not (self.msg == ('_unwatched', ANY)
or self.msg == ('_watched', ANY)):
Events.log(DeadLetter(self.ref, self.msg, self.sender))
def _do_spawn(self):
self.actor = self.watch(self.node.spawn(self.actor_factory, name=self.spawn_at))
Events.log(Message("Spawned %s" % (self.actor,)))
def unhandled(self, m, sender):
if ('terminated', ANY) == m:
raise UnhandledTermination(watcher=self.ref, watchee=m[1])
else:
Events.log(UnhandledMessage(self.ref, m, sender))
def unsubscribe(cls):
for event_type in _ERROR_EVENTS:
Events.unsubscribe(event_type, ErrorCollector.collect)
def _remote_dead_letter(self, path, msg, sender):
ref = Ref(cell=None, uri=Uri.parse(self.nid + path), node=self, is_local=True)
if not (msg == ('_unwatched', ANY) or msg == ('_watched', ANY)):
Events.log(DeadLetter(ref, msg, sender))
def _kill_queue(self):
q, self.queue = self.queue, None
while q:
ref, msg = q.popleft()
if (IN(["_watched", "_unwatched", "terminated"]), ANY) != msg:
Events.log(DeadLetter(ref, msg))
