def do_it(ix): if create_hub: gevent.get_hub() try: for i in range(count): if not run: break sem.acquire(*acquire_args) sem.release() results[ix] = i if not create_hub: # We don't artificially create the hub. self.assertIsNone(get_hub_if_exists(), (get_hub_if_exists(), ix, i)) if create_hub and i % 10 == 0: gevent.sleep(timing.SMALLEST_RELIABLE_DELAY) elif i % 100 == 0: native_sleep(timing.SMALLEST_RELIABLE_DELAY) except Exception as ex: # pylint:disable=broad-except import traceback traceback.print_exc() results[ix] = str(ex) ex = None finally: hub = get_hub_if_exists() if hub is not None: hub.join() hub.destroy(destroy_loop=True)
def acquire(self, blocking=True, timeout=-1): # This is the Python 3 signature. # On Python 2, Lock.acquire has the signature `Lock.acquire([wait])` # where `wait` is a boolean that cannot be passed by name, only position. # so we're fine to use the Python 3 signature. # Transform the default -1 argument into the None that our # semaphore implementation expects, and raise the same error # the stdlib implementation does. if timeout == -1: timeout = None if not blocking and timeout is not None: raise ValueError("can't specify a timeout for a non-blocking call") if timeout is not None: if timeout < 0: # in C: if(timeout < 0 && timeout != -1) raise ValueError("timeout value must be strictly positive") if timeout > self._TIMEOUT_MAX: raise OverflowError('timeout value is too large') acquired = BoundedSemaphore.acquire(self, blocking, timeout) if not acquired and not blocking and getcurrent( ) is not get_hub_if_exists(): # Run other callbacks. This makes spin locks works. # We can't do this if we're in the hub, which we could easily be: # printing the repr of a thread checks its tstate_lock, and sometimes we # print reprs in the hub. # See https://github.com/gevent/gevent/issues/1464 # By using sleep() instead of self.wait(0), we don't force a trip # around the event loop *unless* we've been running callbacks for # longer than our switch interval. sleep() return acquired
def _reset_hub(self): from gevent._hub_local import set_hub from gevent._hub_local import set_loop from gevent._hub_local import get_hub_if_exists hub = get_hub_if_exists() if hub is not None: hub.destroy(destroy_loop=True) set_hub(None) set_loop(None)
def wrapper(self, *args, **kwargs): try: return method(self, *args, **kwargs) finally: # Remove any customized handle_error, if set on the # instance. try: del get_hub_if_exists().handle_error except AttributeError: pass if self.peek_error()[0] is not None: getcurrent().throw(*self.peek_error()[1:])
def setUp(self): super(TestCase, self).setUp() # Especially if we're running in leakcheck mode, where # the same test gets executed repeatedly, we need to update the # current time. Tests don't always go through the full event loop, # so that doesn't always happen. test__pool.py:TestPoolYYY.test_async # tends to show timeouts that are too short if we don't. # XXX: Should some core part of the loop call this? hub = get_hub_if_exists() if hub and hub.loop: hub.loop.update_now() self.close_on_teardown = [] self.addCleanup(self._tearDownCloseOnTearDown)
def thread_main(): thread_running.set() try: acquired.append(sem.acquire(**thread_acquire_kwargs)) except: exc_info[:] = sys.exc_info() raise # Print finally: hub = get_hub_if_exists() if hub is not None: hub.join() hub.destroy(destroy_loop=True) thread_acquired.set()
def _capture_hub(self, create): # Subclasses should call this as the first action from any # public method that could, in theory, block and switch # to the hub. This may release the GIL. if self.hub is None: # This next line might release the GIL. current_hub = get_hub() if create else get_hub_if_exists() if current_hub is None: return # We have the GIL again. Did anything change? If so, # we lost the race. if self.hub is None: self.hub = current_hub
def _notify_link_list(self, links): # The core of the _notify_links method to notify # links in order. Lets the ``links`` list be mutated, # and only notifies up to the last item in the list, in case # objects are added to it. if not links: # HMM. How did we get here? Running two threads at once? # Seen once on Py27/Win/Appveyor # https://ci.appveyor.com/project/jamadden/gevent/builds/36875645/job/9wahj9ft4h4qa170 return [] only_while_ready = not self._notify_all final_link = links[-1] done = set() # of ids hub = self.hub if self.hub is not None else get_hub_if_exists() unswitched = [] while links: # remember this can be mutated if only_while_ready and not self.ready(): break link = links.pop(0) # Cython optimizes using list internals id_link = id(link) if id_link not in done: # XXX: JAM: What was I thinking? This doesn't make much sense, # there's a good chance `link` will be deallocated, and its id() will # be free to be reused. This also makes looping difficult, you have to # create new functions inside a loop rather than just once outside the loop. done.add(id_link) try: self._drop_lock_for_switch_out() try: link(self) except greenlet_error: # couldn't switch to a greenlet, we must be # running in a different thread. back on the list it goes for next time. unswitched.append(link) finally: self._acquire_lock_for_switch_in() except: # pylint:disable=bare-except # We're running in the hub, errors must not escape. if hub is not None: hub.handle_error((link, self), *sys.exc_info()) else: import traceback traceback.print_exc() if link is final_link: break return unswitched
def _capture_hub(self, create): # Subclasses should call this as the first action from any # public method that could, in theory, block and switch # to the hub. This may release the GIL. It may # raise InvalidThreadUseError if the result would # First, detect a dead hub and drop it. while 1: my_hub = self.hub if my_hub is None: break if my_hub.dead: # dead is a property, could release GIL # back, holding GIL if self.hub is my_hub: self.hub = None my_hub = None break else: break if self.hub is None: # This next line might release the GIL. current_hub = get_hub() if create else get_hub_if_exists() # We have the GIL again. Did anything change? If so, # we lost the race. if self.hub is None: self.hub = current_hub if self.hub is not None and self.hub.thread_ident != _get_thread_ident( ): raise InvalidThreadUseError( self.hub, get_hub_if_exists(), getcurrent() # pylint:disable=undefined-variable ) return self.hub
def __acquire_using_other_hub(self, owning_hub, timeout): assert owning_hub is not get_hub_if_exists() thread_lock = self._allocate_lock() thread_lock.acquire() results = [] owning_hub.loop.run_callback_threadsafe( spawn_raw, self.__acquire_from_other_thread_cb, results, 1, # blocking, timeout, # timeout, thread_lock) # We MUST use a blocking acquire here, or at least be sure we keep going # until we acquire it. If we timed out waiting here, # just before the callback runs, then we would be out of sync. self.__spin_on_native_lock(thread_lock, None) return results[0]
def __init__(self, hub=None): # Before this implementation, AsyncResult and Semaphore # maintained the order of notifications, but Event did not. # In gevent 1.3, before Semaphore extended this class, that # was changed to not maintain the order. It was done because # Event guaranteed to only call callbacks once (a set) but # AsyncResult had no such guarantees. When Semaphore was # changed to extend this class, it lost its ordering # guarantees. Unfortunately, that made it unfair. There are # rare cases that this can starve a greenlet # (https://github.com/gevent/gevent/issues/1487) and maybe # even lead to deadlock (not tested). # So in gevent 1.5 we go back to maintaining order. But it's # still important not to make duplicate calls, and it's also # important to avoid O(n^2) behaviour that can result from # naive use of a simple list due to the need to handle removed # links in the _notify_links loop. Cython has special support for # built-in sets, lists, and dicts, but not ordereddict. Rather than # use two data structures, or a dict({link: order}), we simply use a # list and remove objects as we go, keeping track of them so as not to # have duplicates called. This makes `unlink` O(n), but we can avoid # calling it in the common case in _wait_core (even so, the number of # waiters should usually be pretty small) self._links = [] self._notifier = None # This is conceptually a class attribute, defined here for ease of access in # cython. If it's true, when notifiers fire, all existing callbacks are called. # If its false, we only call callbacks as long as ready() returns true. self._notify_all = True # we don't want to do get_hub() here to allow defining module-level objects # without initializing the hub. However, for multiple-thread safety, as soon # as a waiting method is entered, even if it won't have to wait, we # need to grab the hub and assign ownership. For that reason, if the hub # is present, we'll go ahead and take it. self.hub = hub if hub is not None else get_hub_if_exists()
def acquire(self, blocking=True, timeout=None): """ acquire(blocking=True, timeout=None) -> bool Acquire the semaphore. .. note:: If this semaphore was initialized with a *value* of 0, this method will block forever (unless a timeout is given or blocking is set to false). :keyword bool blocking: If True (the default), this function will block until the semaphore is acquired. :keyword float timeout: If given, and *blocking* is true, specifies the maximum amount of seconds this method will block. :return: A `bool` indicating whether the semaphore was acquired. If ``blocking`` is True and ``timeout`` is None (the default), then (so long as this semaphore was initialized with a size greater than 0) this will always return True. If a timeout was given, and it expired before the semaphore was acquired, False will be returned. (Note that this can still raise a ``Timeout`` exception, if some other caller had already started a timer.) """ # pylint:disable=too-many-return-statements,too-many-branches # Sadly, the body of this method is rather complicated. if self._multithreaded is _UNSET: self._multithreaded = self._get_thread_ident() elif self._multithreaded != self._get_thread_ident(): self._multithreaded = _MULTI # We conceptually now belong to the hub of the thread that # called this, whether or not we have to block. Note that we # cannot force it to be created yet, because Semaphore is used # by importlib.ModuleLock which is used when importing the hub # itself! This also checks for cross-thread issues. invalid_thread_use = None try: self._capture_hub(False) except InvalidThreadUseError as e: # My hub belongs to some other thread. We didn't release the GIL/object lock # by raising the exception, so we know this is still true. invalid_thread_use = e.args e = None if not self.counter and blocking: # We would need to block. So coordinate with the main hub. return self.__acquire_from_other_thread( invalid_thread_use, blocking, timeout) if self.counter > 0: self.counter -= 1 return True if not blocking: return False if self._multithreaded is not _MULTI and self.hub is None: # pylint:disable=access-member-before-definition self.hub = get_hub() # pylint:disable=attribute-defined-outside-init if self.hub is None and not invalid_thread_use: # Someone else is holding us. There's not a hub here, # nor is there a hub in that thread. We'll need to use regular locks. # This will be unfair to yet a third thread that tries to use us with greenlets. return self.__acquire_from_other_thread( (None, None, self._getcurrent(), "NoHubs"), blocking, timeout) # self._wait may drop both the GIL and the _lock_lock. # By the time we regain control, both have been reacquired. try: success = self._wait(timeout) except LoopExit as ex: args = ex.args ex = None if self.counter: success = True else: # Avoid using ex.hub property to keep holding the GIL if len(args) == 3 and args[1].main_hub: # The main hub, meaning the main thread. We probably can do nothing with this. raise return self.__acquire_from_other_thread( (self.hub, get_hub_if_exists(), self._getcurrent(), "LoopExit"), blocking, timeout) if not success: assert timeout is not None # Our timer expired. return False # Neither our timer or another one expired, so we blocked until # awoke. Therefore, the counter is ours assert self.counter > 0, ( self.counter, blocking, timeout, success, ) self.counter -= 1 return True