def fmain(seen):
try:
current().parent.switch()
except:
seen.append(sys.exc_info()[0])
raise
raise SomeError
def fmain(seen):
try:
current().parent.switch()
except:
seen.append(sys.exc_info()[0])
raise
raise SomeError
def test_threaded_adv_leak(self):
if is_pypy:
return
gg = []
def worker():
# main and additional *finished* greenlets
ll = current().ll = []
def additional():
ll.append(current())
for i in range(2):
Fiber(additional).switch()
gg.append(weakref.ref(current()))
for i in range(2):
t = threading.Thread(target=worker)
t.start()
t.join()
current() # update ts_current
gc.collect()
gc.collect()
for g in gg:
self.assertTrue(g() is None)
def switch(self):
"""Switch to the hub.
This method pauses the current fiber and runs the event loop. The
caller should ensure that it has set up appropriate callbacks so that
it will get scheduled again, preferably using :class:`switch_back`. In
this case then return value of this method will be an ``(args,
kwargs)`` tuple containing the arguments passed to the switch back
instance.
If this method is called from the root fiber then there are two
additional cases. If the hub exited due to a call to :meth:`close`,
then this method returns None. And if the hub exited due to a
exception, that exception is re-raised here.
"""
if self._loop is None or not self.is_alive():
raise RuntimeError('hub is closed/dead')
elif fibers.current() is self:
raise RuntimeError('cannot switch to myself')
elif compat.get_thread_ident() != self._thread:
raise RuntimeError('cannot switch from a different thread')
value = super(Hub, self).switch()
# A fiber exit will cause its parent to be switched to. All fibers in
# the system should be children of the Hub, *except* the Hub itself
# which is a child of the root fiber. So do an explicit check here to
# see if the Hub exited unexpectedly, and if so raise an error.
if fibers.current().parent is None and not self.is_alive() \
and self._loop is not None:
raise RuntimeError('hub exited unexpectedly')
return value
def worker():
# main and additional *finished* greenlets
ll = current().ll = []
def additional():
ll.append(current())
for i in range(2):
Fiber(additional).switch()
gg.append(weakref.ref(current()))
def fthread():
lock2.acquire()
current()
del g[0]
lock1.release()
lock2.acquire()
current()
lock1.release()
def fthread():
lock2.acquire()
current()
del g[0]
lock1.release()
lock2.acquire()
current()
lock1.release()
def worker():
# main and additional *finished* greenlets
ll = current().ll = []
def additional():
ll.append(current())
for i in range(2):
Fiber(additional).switch()
gg.append(weakref.ref(current()))
def acquire(self, blocking=True, timeout=None):
"""Acquire the lock.
If *blocking* is true (the default), then this will block until the
lock can be acquired. The *timeout* parameter specifies an optional
timeout in seconds.
The return value is a boolean indicating whether the lock was acquired.
"""
hub = get_hub()
try:
# switcher.__call__ needs to be synchronized with a lock IF it can
# be called from different threads. This is the case here because
# this method may be called from multiple threads and the callbacks
# are run in the calling thread. So pass it our _lock.
with switch_back(timeout, lock=self._lock) as switcher:
with self._lock:
if not self._locked:
self._locked = 1
self._owner = fibers.current()
return True
elif self._reentrant and self._owner is fibers.current():
self._locked += 1
return True
elif not blocking:
return False
handle = add_callback(self, switcher)
# It is safe to call hub.switch() outside the lock. Another
# thread could have called acquire()+release(), thereby firing
# the switchback. However the switchback only schedules the
# switchback in our hub, it won't execute it yet. So the
# switchback won't actually happen until we switch to the hub.
hub.switch()
# Here the lock should be ours because _release() wakes up only
# the fiber that it passed the lock.
assert self._locked > 0
assert self._owner is fibers.current()
except BaseException as e:
# Likely a Timeout but could also be e.g. Cancelled
with self._lock:
# Clean up the callback. It might have been popped by
# _release() but that is OK.
remove_callback(self, handle)
# This fiber was passed the lock but before that an exception
# was already scheduled with run_callback() (likely through
# Fiber.throw())
if self._owner is fibers.current():
self._release()
if e is switcher.timeout:
return False
raise
return True
def acquire(self, blocking=True, timeout=None):
"""Acquire the lock.
If *blocking* is true (the default), then this will block until the
lock can be acquired. The *timeout* parameter specifies an optional
timeout in seconds.
The return value is a boolean indicating whether the lock was acquired.
"""
hub = get_hub()
try:
# switcher.__call__ needs to be synchronized with a lock IF it can
# be called from different threads. This is the case here because
# this method may be called from multiple threads and the callbacks
# are run in the calling thread. So pass it our _lock.
with switch_back(timeout, lock=self._lock) as switcher:
with self._lock:
if not self._locked:
self._locked = 1
self._owner = fibers.current()
return True
elif self._reentrant and self._owner is fibers.current():
self._locked += 1
return True
elif not blocking:
return False
handle = add_callback(self, switcher)
# It is safe to call hub.switch() outside the lock. Another
# thread could have called acquire()+release(), thereby firing
# the switchback. However the switchback only schedules the
# switchback in our hub, it won't execute it yet. So the
# switchback won't actually happen until we switch to the hub.
hub.switch()
# Here the lock should be ours because _release() wakes up only
# the fiber that it passed the lock.
assert self._locked > 0
assert self._owner is fibers.current()
except BaseException as e:
# Likely a Timeout but could also be e.g. Cancelled
with self._lock:
# Clean up the callback. It might have been popped by
# _release() but that is OK.
remove_callback(self, handle)
# This fiber was passed the lock but before that an exception
# was already scheduled with run_callback() (likely through
# Fiber.throw())
if self._owner is fibers.current():
self._release()
if e is switcher.timeout:
return False
raise
return True
def test_threaded_leak(self):
gg = []
def worker():
# only main greenlet present
gg.append(weakref.ref(current()))
for i in range(2):
t = threading.Thread(target=worker)
t.start()
t.join()
current() # update ts_current
gc.collect()
for g in gg:
self.assertTrue(g() is None)
def test_threaded_updatecurrent(self):
# FIXME (hangs?)
return
# released when main thread should execute
lock1 = threading.Lock()
lock1.acquire()
# released when another thread should execute
lock2 = threading.Lock()
lock2.acquire()
class finalized(object):
def __del__(self):
# happens while in green_updatecurrent() in main greenlet
# should be very careful not to accidentally call it again
# at the same time we must make sure another thread executes
lock2.release()
lock1.acquire()
# now ts_current belongs to another thread
def deallocator():
current().parent.switch()
def fthread():
lock2.acquire()
current()
del g[0]
lock1.release()
lock2.acquire()
current()
lock1.release()
main = current()
g = [Fiber(deallocator)]
g[0].bomb = finalized()
g[0].switch()
t = threading.Thread(target=fthread)
t.start()
# let another thread grab ts_current and deallocate g[0]
lock2.release()
lock1.acquire()
# this is the corner stone
# getcurrent() will notice that ts_current belongs to another thread
# and start the update process, which would notice that g[0] should
# be deallocated, and that will execute an object's finalizer. Now,
# that object will let another thread run so it can grab ts_current
# again, which would likely crash the interpreter if there's no
# check for this case at the end of green_updatecurrent(). This test
# passes if getcurrent() returns correct result, but it's likely
# to randomly crash if it's not anyway.
self.assertEqual(current(), main)
# wait for another thread to complete, just in case
t.join()
def test_threaded_updatecurrent(self):
# FIXME (hangs?)
return
# released when main thread should execute
lock1 = threading.Lock()
lock1.acquire()
# released when another thread should execute
lock2 = threading.Lock()
lock2.acquire()
class finalized(object):
def __del__(self):
# happens while in green_updatecurrent() in main greenlet
# should be very careful not to accidentally call it again
# at the same time we must make sure another thread executes
lock2.release()
lock1.acquire()
# now ts_current belongs to another thread
def deallocator():
current().parent.switch()
def fthread():
lock2.acquire()
current()
del g[0]
lock1.release()
lock2.acquire()
current()
lock1.release()
main = current()
g = [Fiber(deallocator)]
g[0].bomb = finalized()
g[0].switch()
t = threading.Thread(target=fthread)
t.start()
# let another thread grab ts_current and deallocate g[0]
lock2.release()
lock1.acquire()
# this is the corner stone
# getcurrent() will notice that ts_current belongs to another thread
# and start the update process, which would notice that g[0] should
# be deallocated, and that will execute an object's finalizer. Now,
# that object will let another thread run so it can grab ts_current
# again, which would likely crash the interpreter if there's no
# check for this case at the end of green_updatecurrent(). This test
# passes if getcurrent() returns correct result, but it's likely
# to randomly crash if it's not anyway.
self.assertEqual(current(), main)
# wait for another thread to complete, just in case
t.join()
def switchpoint(*args, **kwargs):
hub = get_hub()
if fibers.current() is hub:
raise RuntimeError('cannot call switchpoint from the Hub')
if hub._noswitch_depth:
raise AssertionError('switchpoint called from no-switch section')
return func(*args, **kwargs)
def Yield(value):
g = fibers.current()
while not isinstance(g, genlet):
if g is None:
raise RuntimeError('yield outside a genlet')
g = g.parent
g.parent.switch(value)
def __next__(self):
self.parent = fibers.current()
result = self.switch()
if self.is_alive():
return result
else:
raise StopIteration
def thread_info(self):
tid = threading.current_thread().name
if tid == 'MainThread':
tid = 'Main'
current = fibers.current()
fid = getattr(current, 'name', util.objref(current)) if current.parent else 'Root'
return '{}:{}'.format(tid, fid)
def release(self):
"""Release the lock."""
with self._lock:
if not self._locked:
raise RuntimeError('lock not currently held')
elif self._reentrant and self._owner is not fibers.current():
raise RuntimeError('lock not owned by this fiber')
self._release()
def test_threaded_leak(self):
if is_pypy:
return
gg = []
def worker():
# only main greenlet present
gg.append(weakref.ref(current()))
for i in range(2):
t = threading.Thread(target=worker)
t.start()
t.join()
current() # update ts_current
gc.collect()
for g in gg:
self.assertTrue(g() is None)
def thread_info(self):
tid = threading.current_thread().name
if tid == 'MainThread':
tid = 'Main'
current = fibers.current()
fid = getattr(current, 'name',
util.objref(current)) if current.parent else 'Root'
return '{}:{}'.format(tid, fid)
def release(self):
"""Release the lock."""
with self._lock:
if not self._locked:
raise RuntimeError('lock not currently held')
elif self._reentrant and self._owner is not fibers.current():
raise RuntimeError('lock not owned by this fiber')
self._release()
def current_fiber():
"""Return the current fiber.
Note: The root and hub fiber are "bare" :class:`fibers.Fiber` instances.
Calling this method there returns the bare instance, not a
:class:`gruvi.Fiber` instance.
"""
return fibers.current()
def thread_info(self):
"""Return a string identifying the current thread and fiber."""
tid = threading.current_thread().name
if tid == 'MainThread':
tid = 'Main'
current = fibers.current()
fid = getattr(current, 'name', util.objref(current)) if current.parent else 'Root'
return '{}/{}'.format(tid, fid)
def _create_main_proc():
main_proc = Proc.__new__(Proc)
main_proc.fiber = fibers.current()
main_proc._is_started = True
main_proc.sleeping = True
_tls.main_proc = main_proc
_tls.current_proc = main_proc
def _create_main_proc():
main_proc = Proc.__new__(Proc)
main_proc.fiber = fibers.current()
main_proc._is_started = True
main_proc.sleeping = True
_tls.main_proc = main_proc
_tls.current_proc = main_proc
def thread_info(self):
"""Return a string identifying the current thread and fiber."""
tid = threading.current_thread().name
if tid == 'MainThread':
tid = 'Main'
current = fibers.current()
fid = getattr(current, 'name',
util.objref(current)) if current.parent else 'Root'
return '{}/{}'.format(tid, fid)
def test_threaded_adv_leak(self):
gg = []
def worker():
# main and additional *finished* greenlets
ll = current().ll = []
def additional():
ll.append(current())
for i in range(2):
Fiber(additional).switch()
gg.append(weakref.ref(current()))
for i in range(2):
t = threading.Thread(target=worker)
t.start()
t.join()
current() # update ts_current
gc.collect()
for g in gg:
self.assertTrue(g() is None)
def get_hub():
"""Return the instance of the hub."""
try:
hub = _local.hub
except AttributeError:
# The Hub can only be instantiated from the root fiber. No other fibers
# can run until the Hub is there, so the root will always be the first
# one to call get_hub().
assert fibers.current().parent is None
hub = _local.hub = Hub()
return hub
def Yield(value, level=1):
g = fibers.current()
while level != 0:
if not isinstance(g, genlet):
raise RuntimeError('yield outside a genlet')
if level > 1:
g.parent.set_child(g)
g = g.parent
level -= 1
g.switch(value)
def Yield(value, level=1):
g = fibers.current()
while level != 0:
if not isinstance(g, genlet):
raise RuntimeError('yield outside a genlet')
if level > 1:
g.parent.set_child(g)
g = g.parent
level -= 1
g.switch(value)
def test_finalizer_crash(self):
# This test is designed to crash when active greenlets
# are made garbage collectable, until the underlying
# problem is resolved. How does it work:
# - order of object creation is important
# - array is created first, so it is moved to unreachable first
# - we create a cycle between a greenlet and this array
# - we create an object that participates in gc, is only
# referenced by a greenlet, and would corrupt gc lists
# on destruction, the easiest is to use an object with
# a finalizer
# - because array is the first object in unreachable it is
# cleared first, which causes all references to greenlet
# to disappear and causes greenlet to be destroyed, but since
# it is still live it causes a switch during gc, which causes
# an object with finalizer to be destroyed, which causes stack
# corruption and then a crash
class object_with_finalizer(object):
def __del__(self):
pass
array = []
parent = current()
def greenlet_body():
current().object = object_with_finalizer()
try:
parent.switch()
finally:
del current().object
g = Fiber(greenlet_body)
g.array = array
array.append(g)
g.switch()
del array
del g
current()
gc.collect()
def test_finalizer_crash(self):
# This test is designed to crash when active greenlets
# are made garbage collectable, until the underlying
# problem is resolved. How does it work:
# - order of object creation is important
# - array is created first, so it is moved to unreachable first
# - we create a cycle between a greenlet and this array
# - we create an object that participates in gc, is only
# referenced by a greenlet, and would corrupt gc lists
# on destruction, the easiest is to use an object with
# a finalizer
# - because array is the first object in unreachable it is
# cleared first, which causes all references to greenlet
# to disappear and causes greenlet to be destroyed, but since
# it is still live it causes a switch during gc, which causes
# an object with finalizer to be destroyed, which causes stack
# corruption and then a crash
class object_with_finalizer(object):
def __del__(self):
pass
array = []
parent = current()
def greenlet_body():
current().object = object_with_finalizer()
try:
parent.switch()
finally:
del current().object
g = Fiber(greenlet_body)
g.array = array
array.append(g)
g.switch()
del array
del g
current()
gc.collect()
def main_loop(self):
self._main_fiber = fibers.current()
while 1:
if self._fibers:
s = self._fibers.pop(0)
if not s.started:
s.start()
self._fibers.append(s)
elif s.finished:
# script has ended, remove it from the list
self._fibers.pop(s)
else:
s.run_to_next_switch()
self._fibers.append(s)
self._on_idle()
def main_loop(self):
self._main_fiber = fibers.current()
while 1:
if self._fibers:
s = self._fibers.pop(0)
if not s.started:
s.start()
self._fibers.append(s)
elif s.finished:
# script has ended, remove it from the list
self._fibers.pop(s)
else:
s.run_to_next_switch()
self._fibers.append(s)
self._on_idle()
def __init__(self, timeout=None, hub=None, lock=None):
"""
The *timeout* argument can be used to force a timeout after this many
seconds. It can be an int or a float. If a timeout happens,
:meth:`Hub.switch` will raise a :class:`Timeout` exception in the
origin fiber. The default is None, meaning there is no timeout.
The *hub* argument can be used to specify an alternate hub to use.
This argument is used by the unit tests and should normally not be
needed.
"""
self._timeout = timeout
self._hub = hub or get_hub()
self._fiber = fibers.current()
self._callbacks = None
self._lock = lock
def __init__(self, timeout=None, hub=None, lock=None):
"""
The *timeout* argument can be used to force a timeout after this many
seconds. It can be an int or a float. If a timeout happens,
:meth:`Hub.switch` will raise a :class:`Timeout` exception in the
origin fiber. The default is None, meaning there is no timeout.
The *hub* argument can be used to specify an alternate hub to use.
This argument is used by the unit tests and should normally not be
needed.
"""
self._timeout = timeout
self._hub = hub or get_hub()
self._fiber = fibers.current()
self._callbacks = None
self._lock = lock
def __next__(self):
if self.child:
child = self.child
while child.child:
tmp = child
child = child.child
tmp.child = None
result = child.switch()
else:
self.parent = fibers.current()
result = self.switch()
if self.is_alive():
return result
else:
raise StopIteration
def __next__(self):
if self.child:
child = self.child
while child.child:
tmp = child
child = child.child
tmp.child = None
result = child.switch()
else:
self.parent = fibers.current()
result = self.switch()
if self.is_alive():
return result
else:
raise StopIteration
def test_throw_goes_to_original_parent(self):
main = fibers.current()
def f1():
try:
main.switch("f1 ready to catch")
except IndexError:
return "caught"
else:
return "normal exit"
def f2():
main.switch("from f2")
g1 = Fiber(f1)
g2 = Fiber(target=f2, parent=g1)
self.assertRaises(IndexError, g2.throw, IndexError)
self.assertFalse(g2.is_alive())
self.assertTrue(g1.is_alive()) # g1 is skipped because it was not started
def test_throw_goes_to_original_parent(self):
main = fibers.current()
def f1():
try:
main.switch("f1 ready to catch")
except IndexError:
return "caught"
else:
return "normal exit"
def f2():
main.switch("from f2")
g1 = Fiber(f1)
g2 = Fiber(target=f2, parent=g1)
self.assertRaises(IndexError, g2.throw, IndexError)
self.assertFalse(g2.is_alive())
self.assertTrue(
g1.is_alive()) # g1 is skipped because it was not started
def close(self):
"""Close the hub and wait for it to be closed.
This may only be called in the root fiber. After this call returned,
Gruvi cannot be used anymore in the current thread. The main use case
for calling this method is to clean up resources in a multi-threaded
program where you want to exit a thead but not yet the entire process.
"""
if self._loop is None:
return
if fibers.current().parent is not None:
raise RuntimeError('close() may only be called in the root fiber')
elif compat.get_thread_ident() != self._thread:
raise RuntimeError('cannot close() from a different thread')
self._closing = True
self._interrupt_loop()
# Note how we are switching to the Hub without a switchback condition
# being in place. This works because the hub is our child and upon
# a child fiber exit its parent is switched in.
self.switch()
def run(self):
# Target of Hub.switch().
if fibers.current() is not self:
raise RuntimeError('run() may only be called from the Hub')
self._log.debug('starting hub fiber')
# This is where the loop runs. There are two loops:
#
# * An outer loop, in Python. This is where callbacks scheduled via
# run_callback() run. We call these "Python" callbacks, and they are
# alowed to switch between fibers.
# * An inner loop, in libuv. This is where we block for input. Fiber
# switches are not allowed here, primarily to enforce that protocol
# callbacks stay fully event based and do not switch.
#
# The libuv loop is the main loop, and is interrupted only when Python
# callbacks need to run (via _interrupt_loop()). After those are run,
# we enter the libuv loop again.
while True:
self._run_callbacks()
if self._closing:
break
# If the Python callbacks run above scheduled further callbacks, do
# not wait for new events in the libuv loop. This would cause the
# Pyton callbacks to potentially be delayed indefinitely.
mode = pyuv.UV_RUN_NOWAIT if len(self._callbacks) else pyuv.UV_RUN_DEFAULT
with assert_no_switchpoints(self):
self._loop.run(mode)
# Hub is going to exit at this point. Clean everyting up.
self._poll.close()
for handle in self._loop.handles:
if not handle.closed:
handle.close()
# Run the loop until all asynchronous closes are handled.
# For some reason it appears this needs to be run twice.
while self._loop.run():
self._log.debug('run loop another time to close handles')
self._loop = None
self._callbacks.clear()
self._async = None
self._sigint = None
self._log.debug('hub fiber terminated')
def log(self, level, exc, msg, *args, **kwargs):
if not self.logger.isEnabledFor(level):
return
current = fibers.current()
from .util import objref
prefix = getattr(current, 'name', objref(current))
if self.logger.isEnabledFor(logging.DEBUG):
target = getattr(current, 'target', None)
if target:
prefix += '{0}()'.format(compat.getqualname(target))
elif current.parent is None:
prefix += '(root)'
f = sys._getframe(2)
fname = os.path.split(f.f_code.co_filename)[1]
prefix += '@{0}:{1}:{2}'.format(fname, f.f_code.co_name, f.f_lineno)
if self.context:
prefix += '; {0}'.format(self.context)
if args or kwargs:
msg = msg.format(*args, **kwargs)
msg = '[{0}] {1}'.format(prefix, msg)
self.logger._log(level, msg, (), exc_info=exc)
def log(self, level, exc, msg, *args, **kwargs):
if not self.logger.isEnabledFor(level):
return
current = fibers.current()
from .util import objref
prefix = getattr(current, 'name', objref(current))
if self.logger.isEnabledFor(logging.DEBUG):
target = getattr(current, 'target', None)
if target:
prefix += '{0}()'.format(compat.getqualname(target))
elif current.parent is None:
prefix += '(root)'
f = sys._getframe(2)
fname = os.path.split(f.f_code.co_filename)[1]
prefix += '@{0}:{1}:{2}'.format(fname, f.f_code.co_name,
f.f_lineno)
if self.context:
prefix += '; {0}'.format(self.context)
if args or kwargs:
msg = msg.format(*args, **kwargs)
msg = '[{0}] {1}'.format(prefix, msg)
self.logger._log(level, msg, (), exc_info=exc)
def test_throw_goes_to_original_parent2(self):
main = fibers.current()
def f1():
try:
main.switch("f1 ready to catch")
except IndexError:
return "caught"
else:
return "normal exit"
def f2():
main.switch("from f2")
g1 = Fiber(f1)
g2 = Fiber(target=f2, parent=g1)
res = g1.switch()
self.assertEqual(res, "f1 ready to catch")
res = g2.throw(IndexError)
self.assertEqual(res, "caught")
self.assertFalse(g2.is_alive())
self.assertFalse(g1.is_alive())
def test_throw_goes_to_original_parent2(self):
main = fibers.current()
def f1():
try:
main.switch("f1 ready to catch")
except IndexError:
return "caught"
else:
return "normal exit"
def f2():
main.switch("from f2")
g1 = Fiber(f1)
g2 = Fiber(target=f2, parent=g1)
res = g1.switch()
self.assertEqual(res, "f1 ready to catch")
res = g2.throw(IndexError)
self.assertEqual(res, "caught")
self.assertFalse(g2.is_alive())
self.assertFalse(g1.is_alive())
def __setattr__(self, key, value):
current = fibers.current()
self._keys.setdefault(current, {})[key] = value
def __delattr__(self, key):
current = fibers.current()
try:
del self._keys[current][key]
except KeyError:
raise AttributeError(key)
def __init__(self):
self._is_started = -1
self.param = None
self.fiber = fibers.current()
self.sleeping = True
def __getattr__(self, key):
current = fibers.current()
try:
return self._keys[current][key]
except KeyError:
raise AttributeError(key)
def f():
lst.append(1)
current().parent.switch()
lst.extend([1, 1])
def switch(val):
return fibers.current().parent.switch(val)
def __delattr__(self, key):
current = fibers.current()
try:
del self._keys[current][key]
except KeyError:
raise AttributeError(key)
def __setattr__(self, key, value):
current = fibers.current()
self._keys.setdefault(current, {})[key] = value
def f():
lst.append(1)
current().parent.switch()
def greenlet_body():
current().object = object_with_finalizer()
try:
parent.switch()
finally:
del current().object
def __getattr__(self, key):
current = fibers.current()
try:
return self._keys[current][key]
except KeyError:
raise AttributeError(key)
