def do_it(ix):
if create_hub:
gevent.get_hub()
try:
for i in range(count):
if not run:
break
sem.acquire(*acquire_args)
sem.release()
results[ix] = i
if not create_hub:
# We don't artificially create the hub.
self.assertIsNone(get_hub_if_exists(),
(get_hub_if_exists(), ix, i))
if create_hub and i % 10 == 0:
gevent.sleep(timing.SMALLEST_RELIABLE_DELAY)
elif i % 100 == 0:
native_sleep(timing.SMALLEST_RELIABLE_DELAY)
except Exception as ex: # pylint:disable=broad-except
import traceback
traceback.print_exc()
results[ix] = str(ex)
ex = None
finally:
hub = get_hub_if_exists()
if hub is not None:
hub.join()
hub.destroy(destroy_loop=True)
def acquire(self, blocking=True, timeout=-1):
# This is the Python 3 signature.
# On Python 2, Lock.acquire has the signature `Lock.acquire([wait])`
# where `wait` is a boolean that cannot be passed by name, only position.
# so we're fine to use the Python 3 signature.
# Transform the default -1 argument into the None that our
# semaphore implementation expects, and raise the same error
# the stdlib implementation does.
if timeout == -1:
timeout = None
if not blocking and timeout is not None:
raise ValueError("can't specify a timeout for a non-blocking call")
if timeout is not None:
if timeout < 0:
# in C: if(timeout < 0 && timeout != -1)
raise ValueError("timeout value must be strictly positive")
if timeout > self._TIMEOUT_MAX:
raise OverflowError('timeout value is too large')
acquired = BoundedSemaphore.acquire(self, blocking, timeout)
if not acquired and not blocking and getcurrent(
) is not get_hub_if_exists():
# Run other callbacks. This makes spin locks works.
# We can't do this if we're in the hub, which we could easily be:
# printing the repr of a thread checks its tstate_lock, and sometimes we
# print reprs in the hub.
# See https://github.com/gevent/gevent/issues/1464
# By using sleep() instead of self.wait(0), we don't force a trip
# around the event loop *unless* we've been running callbacks for
# longer than our switch interval.
sleep()
return acquired
def _reset_hub(self):
from gevent._hub_local import set_hub
from gevent._hub_local import set_loop
from gevent._hub_local import get_hub_if_exists
hub = get_hub_if_exists()
if hub is not None:
hub.destroy(destroy_loop=True)
set_hub(None)
set_loop(None)
def wrapper(self, *args, **kwargs):
try:
return method(self, *args, **kwargs)
finally:
# Remove any customized handle_error, if set on the
# instance.
try:
del get_hub_if_exists().handle_error
except AttributeError:
pass
if self.peek_error()[0] is not None:
getcurrent().throw(*self.peek_error()[1:])
def setUp(self):
super(TestCase, self).setUp()
# Especially if we're running in leakcheck mode, where
# the same test gets executed repeatedly, we need to update the
# current time. Tests don't always go through the full event loop,
# so that doesn't always happen. test__pool.py:TestPoolYYY.test_async
# tends to show timeouts that are too short if we don't.
# XXX: Should some core part of the loop call this?
hub = get_hub_if_exists()
if hub and hub.loop:
hub.loop.update_now()
self.close_on_teardown = []
self.addCleanup(self._tearDownCloseOnTearDown)
def thread_main():
thread_running.set()
try:
acquired.append(sem.acquire(**thread_acquire_kwargs))
except:
exc_info[:] = sys.exc_info()
raise # Print
finally:
hub = get_hub_if_exists()
if hub is not None:
hub.join()
hub.destroy(destroy_loop=True)
thread_acquired.set()
def _capture_hub(self, create):
# Subclasses should call this as the first action from any
# public method that could, in theory, block and switch
# to the hub. This may release the GIL.
if self.hub is None:
# This next line might release the GIL.
current_hub = get_hub() if create else get_hub_if_exists()
if current_hub is None:
return
# We have the GIL again. Did anything change? If so,
# we lost the race.
if self.hub is None:
self.hub = current_hub
def _notify_link_list(self, links):
# The core of the _notify_links method to notify
# links in order. Lets the ``links`` list be mutated,
# and only notifies up to the last item in the list, in case
# objects are added to it.
if not links:
# HMM. How did we get here? Running two threads at once?
# Seen once on Py27/Win/Appveyor
# https://ci.appveyor.com/project/jamadden/gevent/builds/36875645/job/9wahj9ft4h4qa170
return []
only_while_ready = not self._notify_all
final_link = links[-1]
done = set() # of ids
hub = self.hub if self.hub is not None else get_hub_if_exists()
unswitched = []
while links: # remember this can be mutated
if only_while_ready and not self.ready():
break
link = links.pop(0) # Cython optimizes using list internals
id_link = id(link)
if id_link not in done:
# XXX: JAM: What was I thinking? This doesn't make much sense,
# there's a good chance `link` will be deallocated, and its id() will
# be free to be reused. This also makes looping difficult, you have to
# create new functions inside a loop rather than just once outside the loop.
done.add(id_link)
try:
self._drop_lock_for_switch_out()
try:
link(self)
except greenlet_error:
# couldn't switch to a greenlet, we must be
# running in a different thread. back on the list it goes for next time.
unswitched.append(link)
finally:
self._acquire_lock_for_switch_in()
except: # pylint:disable=bare-except
# We're running in the hub, errors must not escape.
if hub is not None:
hub.handle_error((link, self), *sys.exc_info())
else:
import traceback
traceback.print_exc()
if link is final_link:
break
return unswitched
def _capture_hub(self, create):
# Subclasses should call this as the first action from any
# public method that could, in theory, block and switch
# to the hub. This may release the GIL. It may
# raise InvalidThreadUseError if the result would
# First, detect a dead hub and drop it.
while 1:
my_hub = self.hub
if my_hub is None:
break
if my_hub.dead: # dead is a property, could release GIL
# back, holding GIL
if self.hub is my_hub:
self.hub = None
my_hub = None
break
else:
break
if self.hub is None:
# This next line might release the GIL.
current_hub = get_hub() if create else get_hub_if_exists()
# We have the GIL again. Did anything change? If so,
# we lost the race.
if self.hub is None:
self.hub = current_hub
if self.hub is not None and self.hub.thread_ident != _get_thread_ident(
):
raise InvalidThreadUseError(
self.hub,
get_hub_if_exists(),
getcurrent() # pylint:disable=undefined-variable
)
return self.hub
def __acquire_using_other_hub(self, owning_hub, timeout):
assert owning_hub is not get_hub_if_exists()
thread_lock = self._allocate_lock()
thread_lock.acquire()
results = []
owning_hub.loop.run_callback_threadsafe(
spawn_raw,
self.__acquire_from_other_thread_cb,
results,
1, # blocking,
timeout, # timeout,
thread_lock)
# We MUST use a blocking acquire here, or at least be sure we keep going
# until we acquire it. If we timed out waiting here,
# just before the callback runs, then we would be out of sync.
self.__spin_on_native_lock(thread_lock, None)
return results[0]
def __init__(self, hub=None):
# Before this implementation, AsyncResult and Semaphore
# maintained the order of notifications, but Event did not.
# In gevent 1.3, before Semaphore extended this class, that
# was changed to not maintain the order. It was done because
# Event guaranteed to only call callbacks once (a set) but
# AsyncResult had no such guarantees. When Semaphore was
# changed to extend this class, it lost its ordering
# guarantees. Unfortunately, that made it unfair. There are
# rare cases that this can starve a greenlet
# (https://github.com/gevent/gevent/issues/1487) and maybe
# even lead to deadlock (not tested).
# So in gevent 1.5 we go back to maintaining order. But it's
# still important not to make duplicate calls, and it's also
# important to avoid O(n^2) behaviour that can result from
# naive use of a simple list due to the need to handle removed
# links in the _notify_links loop. Cython has special support for
# built-in sets, lists, and dicts, but not ordereddict. Rather than
# use two data structures, or a dict({link: order}), we simply use a
# list and remove objects as we go, keeping track of them so as not to
# have duplicates called. This makes `unlink` O(n), but we can avoid
# calling it in the common case in _wait_core (even so, the number of
# waiters should usually be pretty small)
self._links = []
self._notifier = None
# This is conceptually a class attribute, defined here for ease of access in
# cython. If it's true, when notifiers fire, all existing callbacks are called.
# If its false, we only call callbacks as long as ready() returns true.
self._notify_all = True
# we don't want to do get_hub() here to allow defining module-level objects
# without initializing the hub. However, for multiple-thread safety, as soon
# as a waiting method is entered, even if it won't have to wait, we
# need to grab the hub and assign ownership. For that reason, if the hub
# is present, we'll go ahead and take it.
self.hub = hub if hub is not None else get_hub_if_exists()
def acquire(self, blocking=True, timeout=None):
"""
acquire(blocking=True, timeout=None) -> bool
Acquire the semaphore.
.. note:: If this semaphore was initialized with a *value* of 0,
this method will block forever (unless a timeout is given or blocking is
set to false).
:keyword bool blocking: If True (the default), this function will block
until the semaphore is acquired.
:keyword float timeout: If given, and *blocking* is true,
specifies the maximum amount of seconds
this method will block.
:return: A `bool` indicating whether the semaphore was acquired.
If ``blocking`` is True and ``timeout`` is None (the default), then
(so long as this semaphore was initialized with a size greater than 0)
this will always return True. If a timeout was given, and it expired before
the semaphore was acquired, False will be returned. (Note that this can still
raise a ``Timeout`` exception, if some other caller had already started a timer.)
"""
# pylint:disable=too-many-return-statements,too-many-branches
# Sadly, the body of this method is rather complicated.
if self._multithreaded is _UNSET:
self._multithreaded = self._get_thread_ident()
elif self._multithreaded != self._get_thread_ident():
self._multithreaded = _MULTI
# We conceptually now belong to the hub of the thread that
# called this, whether or not we have to block. Note that we
# cannot force it to be created yet, because Semaphore is used
# by importlib.ModuleLock which is used when importing the hub
# itself! This also checks for cross-thread issues.
invalid_thread_use = None
try:
self._capture_hub(False)
except InvalidThreadUseError as e:
# My hub belongs to some other thread. We didn't release the GIL/object lock
# by raising the exception, so we know this is still true.
invalid_thread_use = e.args
e = None
if not self.counter and blocking:
# We would need to block. So coordinate with the main hub.
return self.__acquire_from_other_thread(
invalid_thread_use, blocking, timeout)
if self.counter > 0:
self.counter -= 1
return True
if not blocking:
return False
if self._multithreaded is not _MULTI and self.hub is None: # pylint:disable=access-member-before-definition
self.hub = get_hub() # pylint:disable=attribute-defined-outside-init
if self.hub is None and not invalid_thread_use:
# Someone else is holding us. There's not a hub here,
# nor is there a hub in that thread. We'll need to use regular locks.
# This will be unfair to yet a third thread that tries to use us with greenlets.
return self.__acquire_from_other_thread(
(None, None, self._getcurrent(), "NoHubs"), blocking, timeout)
# self._wait may drop both the GIL and the _lock_lock.
# By the time we regain control, both have been reacquired.
try:
success = self._wait(timeout)
except LoopExit as ex:
args = ex.args
ex = None
if self.counter:
success = True
else:
# Avoid using ex.hub property to keep holding the GIL
if len(args) == 3 and args[1].main_hub:
# The main hub, meaning the main thread. We probably can do nothing with this.
raise
return self.__acquire_from_other_thread(
(self.hub, get_hub_if_exists(), self._getcurrent(),
"LoopExit"), blocking, timeout)
if not success:
assert timeout is not None
# Our timer expired.
return False
# Neither our timer or another one expired, so we blocked until
# awoke. Therefore, the counter is ours
assert self.counter > 0, (
self.counter,
blocking,
timeout,
success,
)
self.counter -= 1
return True
