def fmain(seen): try: current().parent.switch() except: seen.append(sys.exc_info()[0]) raise raise SomeError
def test_threaded_adv_leak(self): if is_pypy: return gg = [] def worker(): # main and additional *finished* greenlets ll = current().ll = [] def additional(): ll.append(current()) for i in range(2): Fiber(additional).switch() gg.append(weakref.ref(current())) for i in range(2): t = threading.Thread(target=worker) t.start() t.join() current() # update ts_current gc.collect() gc.collect() for g in gg: self.assertTrue(g() is None)
def switch(self): """Switch to the hub. This method pauses the current fiber and runs the event loop. The caller should ensure that it has set up appropriate callbacks so that it will get scheduled again, preferably using :class:`switch_back`. In this case then return value of this method will be an ``(args, kwargs)`` tuple containing the arguments passed to the switch back instance. If this method is called from the root fiber then there are two additional cases. If the hub exited due to a call to :meth:`close`, then this method returns None. And if the hub exited due to a exception, that exception is re-raised here. """ if self._loop is None or not self.is_alive(): raise RuntimeError('hub is closed/dead') elif fibers.current() is self: raise RuntimeError('cannot switch to myself') elif compat.get_thread_ident() != self._thread: raise RuntimeError('cannot switch from a different thread') value = super(Hub, self).switch() # A fiber exit will cause its parent to be switched to. All fibers in # the system should be children of the Hub, *except* the Hub itself # which is a child of the root fiber. So do an explicit check here to # see if the Hub exited unexpectedly, and if so raise an error. if fibers.current().parent is None and not self.is_alive() \ and self._loop is not None: raise RuntimeError('hub exited unexpectedly') return value
def worker(): # main and additional *finished* greenlets ll = current().ll = [] def additional(): ll.append(current()) for i in range(2): Fiber(additional).switch() gg.append(weakref.ref(current()))
def fthread(): lock2.acquire() current() del g[0] lock1.release() lock2.acquire() current() lock1.release()
def acquire(self, blocking=True, timeout=None): """Acquire the lock. If *blocking* is true (the default), then this will block until the lock can be acquired. The *timeout* parameter specifies an optional timeout in seconds. The return value is a boolean indicating whether the lock was acquired. """ hub = get_hub() try: # switcher.__call__ needs to be synchronized with a lock IF it can # be called from different threads. This is the case here because # this method may be called from multiple threads and the callbacks # are run in the calling thread. So pass it our _lock. with switch_back(timeout, lock=self._lock) as switcher: with self._lock: if not self._locked: self._locked = 1 self._owner = fibers.current() return True elif self._reentrant and self._owner is fibers.current(): self._locked += 1 return True elif not blocking: return False handle = add_callback(self, switcher) # It is safe to call hub.switch() outside the lock. Another # thread could have called acquire()+release(), thereby firing # the switchback. However the switchback only schedules the # switchback in our hub, it won't execute it yet. So the # switchback won't actually happen until we switch to the hub. hub.switch() # Here the lock should be ours because _release() wakes up only # the fiber that it passed the lock. assert self._locked > 0 assert self._owner is fibers.current() except BaseException as e: # Likely a Timeout but could also be e.g. Cancelled with self._lock: # Clean up the callback. It might have been popped by # _release() but that is OK. remove_callback(self, handle) # This fiber was passed the lock but before that an exception # was already scheduled with run_callback() (likely through # Fiber.throw()) if self._owner is fibers.current(): self._release() if e is switcher.timeout: return False raise return True
def test_threaded_leak(self): gg = [] def worker(): # only main greenlet present gg.append(weakref.ref(current())) for i in range(2): t = threading.Thread(target=worker) t.start() t.join() current() # update ts_current gc.collect() for g in gg: self.assertTrue(g() is None)
def test_threaded_updatecurrent(self): # FIXME (hangs?) return # released when main thread should execute lock1 = threading.Lock() lock1.acquire() # released when another thread should execute lock2 = threading.Lock() lock2.acquire() class finalized(object): def __del__(self): # happens while in green_updatecurrent() in main greenlet # should be very careful not to accidentally call it again # at the same time we must make sure another thread executes lock2.release() lock1.acquire() # now ts_current belongs to another thread def deallocator(): current().parent.switch() def fthread(): lock2.acquire() current() del g[0] lock1.release() lock2.acquire() current() lock1.release() main = current() g = [Fiber(deallocator)] g[0].bomb = finalized() g[0].switch() t = threading.Thread(target=fthread) t.start() # let another thread grab ts_current and deallocate g[0] lock2.release() lock1.acquire() # this is the corner stone # getcurrent() will notice that ts_current belongs to another thread # and start the update process, which would notice that g[0] should # be deallocated, and that will execute an object's finalizer. Now, # that object will let another thread run so it can grab ts_current # again, which would likely crash the interpreter if there's no # check for this case at the end of green_updatecurrent(). This test # passes if getcurrent() returns correct result, but it's likely # to randomly crash if it's not anyway. self.assertEqual(current(), main) # wait for another thread to complete, just in case t.join()
def switchpoint(*args, **kwargs): hub = get_hub() if fibers.current() is hub: raise RuntimeError('cannot call switchpoint from the Hub') if hub._noswitch_depth: raise AssertionError('switchpoint called from no-switch section') return func(*args, **kwargs)
def Yield(value): g = fibers.current() while not isinstance(g, genlet): if g is None: raise RuntimeError('yield outside a genlet') g = g.parent g.parent.switch(value)
def __next__(self): self.parent = fibers.current() result = self.switch() if self.is_alive(): return result else: raise StopIteration
def thread_info(self): tid = threading.current_thread().name if tid == 'MainThread': tid = 'Main' current = fibers.current() fid = getattr(current, 'name', util.objref(current)) if current.parent else 'Root' return '{}:{}'.format(tid, fid)
def release(self): """Release the lock.""" with self._lock: if not self._locked: raise RuntimeError('lock not currently held') elif self._reentrant and self._owner is not fibers.current(): raise RuntimeError('lock not owned by this fiber') self._release()
def test_threaded_leak(self): if is_pypy: return gg = [] def worker(): # only main greenlet present gg.append(weakref.ref(current())) for i in range(2): t = threading.Thread(target=worker) t.start() t.join() current() # update ts_current gc.collect() for g in gg: self.assertTrue(g() is None)
def current_fiber(): """Return the current fiber. Note: The root and hub fiber are "bare" :class:`fibers.Fiber` instances. Calling this method there returns the bare instance, not a :class:`gruvi.Fiber` instance. """ return fibers.current()
def thread_info(self): """Return a string identifying the current thread and fiber.""" tid = threading.current_thread().name if tid == 'MainThread': tid = 'Main' current = fibers.current() fid = getattr(current, 'name', util.objref(current)) if current.parent else 'Root' return '{}/{}'.format(tid, fid)
def _create_main_proc(): main_proc = Proc.__new__(Proc) main_proc.fiber = fibers.current() main_proc._is_started = True main_proc.sleeping = True _tls.main_proc = main_proc _tls.current_proc = main_proc
def test_threaded_adv_leak(self): gg = [] def worker(): # main and additional *finished* greenlets ll = current().ll = [] def additional(): ll.append(current()) for i in range(2): Fiber(additional).switch() gg.append(weakref.ref(current())) for i in range(2): t = threading.Thread(target=worker) t.start() t.join() current() # update ts_current gc.collect() for g in gg: self.assertTrue(g() is None)
def get_hub(): """Return the instance of the hub.""" try: hub = _local.hub except AttributeError: # The Hub can only be instantiated from the root fiber. No other fibers # can run until the Hub is there, so the root will always be the first # one to call get_hub(). assert fibers.current().parent is None hub = _local.hub = Hub() return hub
def Yield(value, level=1): g = fibers.current() while level != 0: if not isinstance(g, genlet): raise RuntimeError('yield outside a genlet') if level > 1: g.parent.set_child(g) g = g.parent level -= 1 g.switch(value)
def test_finalizer_crash(self): # This test is designed to crash when active greenlets # are made garbage collectable, until the underlying # problem is resolved. How does it work: # - order of object creation is important # - array is created first, so it is moved to unreachable first # - we create a cycle between a greenlet and this array # - we create an object that participates in gc, is only # referenced by a greenlet, and would corrupt gc lists # on destruction, the easiest is to use an object with # a finalizer # - because array is the first object in unreachable it is # cleared first, which causes all references to greenlet # to disappear and causes greenlet to be destroyed, but since # it is still live it causes a switch during gc, which causes # an object with finalizer to be destroyed, which causes stack # corruption and then a crash class object_with_finalizer(object): def __del__(self): pass array = [] parent = current() def greenlet_body(): current().object = object_with_finalizer() try: parent.switch() finally: del current().object g = Fiber(greenlet_body) g.array = array array.append(g) g.switch() del array del g current() gc.collect()
def main_loop(self): self._main_fiber = fibers.current() while 1: if self._fibers: s = self._fibers.pop(0) if not s.started: s.start() self._fibers.append(s) elif s.finished: # script has ended, remove it from the list self._fibers.pop(s) else: s.run_to_next_switch() self._fibers.append(s) self._on_idle()
def __init__(self, timeout=None, hub=None, lock=None): """ The *timeout* argument can be used to force a timeout after this many seconds. It can be an int or a float. If a timeout happens, :meth:`Hub.switch` will raise a :class:`Timeout` exception in the origin fiber. The default is None, meaning there is no timeout. The *hub* argument can be used to specify an alternate hub to use. This argument is used by the unit tests and should normally not be needed. """ self._timeout = timeout self._hub = hub or get_hub() self._fiber = fibers.current() self._callbacks = None self._lock = lock
def __next__(self): if self.child: child = self.child while child.child: tmp = child child = child.child tmp.child = None result = child.switch() else: self.parent = fibers.current() result = self.switch() if self.is_alive(): return result else: raise StopIteration
def test_throw_goes_to_original_parent(self): main = fibers.current() def f1(): try: main.switch("f1 ready to catch") except IndexError: return "caught" else: return "normal exit" def f2(): main.switch("from f2") g1 = Fiber(f1) g2 = Fiber(target=f2, parent=g1) self.assertRaises(IndexError, g2.throw, IndexError) self.assertFalse(g2.is_alive()) self.assertTrue(g1.is_alive()) # g1 is skipped because it was not started
def test_throw_goes_to_original_parent(self): main = fibers.current() def f1(): try: main.switch("f1 ready to catch") except IndexError: return "caught" else: return "normal exit" def f2(): main.switch("from f2") g1 = Fiber(f1) g2 = Fiber(target=f2, parent=g1) self.assertRaises(IndexError, g2.throw, IndexError) self.assertFalse(g2.is_alive()) self.assertTrue( g1.is_alive()) # g1 is skipped because it was not started
def close(self): """Close the hub and wait for it to be closed. This may only be called in the root fiber. After this call returned, Gruvi cannot be used anymore in the current thread. The main use case for calling this method is to clean up resources in a multi-threaded program where you want to exit a thead but not yet the entire process. """ if self._loop is None: return if fibers.current().parent is not None: raise RuntimeError('close() may only be called in the root fiber') elif compat.get_thread_ident() != self._thread: raise RuntimeError('cannot close() from a different thread') self._closing = True self._interrupt_loop() # Note how we are switching to the Hub without a switchback condition # being in place. This works because the hub is our child and upon # a child fiber exit its parent is switched in. self.switch()
def run(self): # Target of Hub.switch(). if fibers.current() is not self: raise RuntimeError('run() may only be called from the Hub') self._log.debug('starting hub fiber') # This is where the loop runs. There are two loops: # # * An outer loop, in Python. This is where callbacks scheduled via # run_callback() run. We call these "Python" callbacks, and they are # alowed to switch between fibers. # * An inner loop, in libuv. This is where we block for input. Fiber # switches are not allowed here, primarily to enforce that protocol # callbacks stay fully event based and do not switch. # # The libuv loop is the main loop, and is interrupted only when Python # callbacks need to run (via _interrupt_loop()). After those are run, # we enter the libuv loop again. while True: self._run_callbacks() if self._closing: break # If the Python callbacks run above scheduled further callbacks, do # not wait for new events in the libuv loop. This would cause the # Pyton callbacks to potentially be delayed indefinitely. mode = pyuv.UV_RUN_NOWAIT if len(self._callbacks) else pyuv.UV_RUN_DEFAULT with assert_no_switchpoints(self): self._loop.run(mode) # Hub is going to exit at this point. Clean everyting up. self._poll.close() for handle in self._loop.handles: if not handle.closed: handle.close() # Run the loop until all asynchronous closes are handled. # For some reason it appears this needs to be run twice. while self._loop.run(): self._log.debug('run loop another time to close handles') self._loop = None self._callbacks.clear() self._async = None self._sigint = None self._log.debug('hub fiber terminated')
def log(self, level, exc, msg, *args, **kwargs): if not self.logger.isEnabledFor(level): return current = fibers.current() from .util import objref prefix = getattr(current, 'name', objref(current)) if self.logger.isEnabledFor(logging.DEBUG): target = getattr(current, 'target', None) if target: prefix += '{0}()'.format(compat.getqualname(target)) elif current.parent is None: prefix += '(root)' f = sys._getframe(2) fname = os.path.split(f.f_code.co_filename)[1] prefix += '@{0}:{1}:{2}'.format(fname, f.f_code.co_name, f.f_lineno) if self.context: prefix += '; {0}'.format(self.context) if args or kwargs: msg = msg.format(*args, **kwargs) msg = '[{0}] {1}'.format(prefix, msg) self.logger._log(level, msg, (), exc_info=exc)
def test_throw_goes_to_original_parent2(self): main = fibers.current() def f1(): try: main.switch("f1 ready to catch") except IndexError: return "caught" else: return "normal exit" def f2(): main.switch("from f2") g1 = Fiber(f1) g2 = Fiber(target=f2, parent=g1) res = g1.switch() self.assertEqual(res, "f1 ready to catch") res = g2.throw(IndexError) self.assertEqual(res, "caught") self.assertFalse(g2.is_alive()) self.assertFalse(g1.is_alive())
def __setattr__(self, key, value): current = fibers.current() self._keys.setdefault(current, {})[key] = value
def __delattr__(self, key): current = fibers.current() try: del self._keys[current][key] except KeyError: raise AttributeError(key)
def __init__(self): self._is_started = -1 self.param = None self.fiber = fibers.current() self.sleeping = True
def __getattr__(self, key): current = fibers.current() try: return self._keys[current][key] except KeyError: raise AttributeError(key)
def f(): lst.append(1) current().parent.switch() lst.extend([1, 1])
def switch(val): return fibers.current().parent.switch(val)
def f(): lst.append(1) current().parent.switch()
def greenlet_body(): current().object = object_with_finalizer() try: parent.switch() finally: del current().object