def test_semaphore_wait(self):
from _multiprocessing import SemLock
kind = self.SEMAPHORE
value = 1
maxvalue = 1
sem = SemLock(kind, value, maxvalue)
assert sem.acquire()
assert not sem.acquire(timeout=0.1)
def test_semaphore_contextmanager(self):
from _multiprocessing import SemLock
kind = self.SEMAPHORE
value = 1
maxvalue = 1
sem = SemLock(kind, value, maxvalue, "5", unlink=True)
with sem:
assert sem._count() == 1
assert sem._count() == 0
def test_semaphore_contextmanager(self):
from _multiprocessing import SemLock
kind = self.SEMAPHORE
value = 1
maxvalue = 1
sem = SemLock(kind, value, maxvalue)
with sem:
assert sem._count() == 1
assert sem._count() == 0
def test_semaphore_wait(self):
from _multiprocessing import SemLock
kind = self.SEMAPHORE
value = 1
maxvalue = 1
sem = SemLock(kind, value, maxvalue)
res = sem.acquire()
assert res == True
res = sem.acquire(timeout=0.1)
assert res == False
def test_notify_all():
"""A low-level variation on test_notify_all() in lib-python's
test_multiprocessing.py
"""
N_THREADS = 1000
lock = SemLock(0, 1, 1)
results = []
def f(n):
if lock.acquire(timeout=5.):
results.append(n)
lock.release()
else:
print("lock acquire timed out!")
threads = [Thread(target=f, args=(i, )) for i in range(N_THREADS)]
n_started = N_THREADS
with lock:
for t in threads:
try:
t.start()
except thread.error:
# too many threads for this system
t.started = False
n_started -= 1
else:
t.started = True
time.sleep(0.1)
print("started %d threads" % n_started)
for t in threads:
if t.started:
t.join()
assert len(results) == n_started
def __init__(self, kind, value, maxvalue):
name = 'loky'
unlink_now = sys.platform == 'win32' or name == 'fork'
for i in range(100):
try:
self._semlock = SemLockC(kind, value, maxvalue,
SemLock._make_name(), unlink_now)
except FileExistsError:
pass
else:
break
else:
raise FileExistsError('cannot find name for semaphore')
self._make_methods()
if sys.platform != 'win32':
def _after_fork(obj):
obj._semlock._after_fork()
util.register_after_fork(self, _after_fork)
if self._semlock.name is not None:
# We only get here if we are on Unix with forking
# disabled. When the object is garbage collected or the
# process shuts down we unlink the semaphore name
from .semaphore_tracker import register
if sys.version_info < (3, 4):
register(self._semlock.name)
util.Finalize(self,
SemLock._cleanup, (self._semlock.name, ),
exitpriority=0)
def test_SemLock_raises_on_non_string_name():
from _multiprocessing import SemLock
try :
SemLock(kind=1, value=1, name={1:2}, maxvalue=1, unlink=1)
except TypeError:
raised = True
assert raised
def test_semaphore_rebuild(self):
from _multiprocessing import SemLock
kind = self.SEMAPHORE
value = 1
maxvalue = 1
sem = SemLock(kind, value, maxvalue)
sem2 = SemLock._rebuild(sem.handle, kind, value)
assert sem.handle == sem2.handle
def test_semaphore_rebuild(self):
import sys
if sys.platform == 'win32':
from _multiprocessing import SemLock
def sem_unlink(*args):
pass
else:
from _multiprocessing import SemLock, sem_unlink
kind = self.SEMAPHORE
value = 1
maxvalue = 1
sem = SemLock(kind, value, maxvalue, "4.2", unlink=False)
try:
sem2 = SemLock._rebuild(-1, kind, value, "4.2")
#assert sem.handle != sem2.handle---even though they come
# from different calls to sem_open(), on Linux at least,
# they are the same pointer
sem2 = SemLock._rebuild(sem.handle, kind, value, None)
assert sem.handle == sem2.handle
finally:
sem_unlink("4.2")
def test_in_threads(self):
from _multiprocessing import SemLock
from threading import Thread
from time import sleep
l = SemLock(0, 1, 1)
if self.runappdirect:
def f(id):
for i in range(10000):
pass
else:
def f(id):
for i in range(1000):
# reduce the probability of thread switching
# at exactly the wrong time in semlock_acquire
for j in range(10):
pass
threads = [Thread(None, f, args=(i,)) for i in range(2)]
[t.start() for t in threads]
# if the RLock calls to sem_wait and sem_post do not match,
# one of the threads will block and the call to join will fail
[t.join() for t in threads]
def __init__(self):
SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1)
def __init__(self, value=1, *, ctx):
SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx)
def __init__(self, *, ctx):
SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx)
def __init__(self, value=1, *, ctx):
SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx)
def __init__(self, *, ctx):
SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx)
def __init__(self):
SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1)
def __init__(self, value=1):
SemLock.__init__(self, SEMAPHORE, value, value)
def test_semaphore_maxvalue(self):
from _multiprocessing import SemLock
import sys
kind = self.SEMAPHORE
value = SemLock.SEM_VALUE_MAX
maxvalue = SemLock.SEM_VALUE_MAX
sem = SemLock(kind, value, maxvalue)
for i in range(10):
res = sem.acquire()
assert res == True
assert sem._count() == i + 1
if sys.platform != 'darwin':
assert sem._get_value() == maxvalue - (i + 1)
value = 0
maxvalue = SemLock.SEM_VALUE_MAX
sem = SemLock(kind, value, maxvalue)
for i in range(10):
sem.release()
assert sem._count() == -(i + 1)
if sys.platform != 'darwin':
assert sem._get_value() == i + 1
class SemLock(object):
_rand = tempfile._RandomNameSequence()
def __init__(self, kind, value, maxvalue):
name = 'loky'
unlink_now = sys.platform == 'win32' or name == 'fork'
for i in range(100):
try:
self._semlock = SemLockC(kind, value, maxvalue,
SemLock._make_name(), unlink_now)
except FileExistsError:
pass
else:
break
else:
raise FileExistsError('cannot find name for semaphore')
self._make_methods()
if sys.platform != 'win32':
def _after_fork(obj):
obj._semlock._after_fork()
util.register_after_fork(self, _after_fork)
if self._semlock.name is not None:
# We only get here if we are on Unix with forking
# disabled. When the object is garbage collected or the
# process shuts down we unlink the semaphore name
from .semaphore_tracker import register
if sys.version_info < (3, 4):
register(self._semlock.name)
util.Finalize(self,
SemLock._cleanup, (self._semlock.name, ),
exitpriority=0)
@staticmethod
def _cleanup(name):
from .semaphore_tracker import unregister
sem_unlink(name)
unregister(name)
def _make_methods(self):
self.acquire = self._semlock.acquire
self.release = self._semlock.release
def __enter__(self):
return self._semlock.acquire()
def __exit__(self, *args):
return self._semlock.release(*args)
def __getstate__(self):
assert is_spawning()
sl = self._semlock
if sys.platform == 'win32':
h = get_spawning_popen().duplicate_for_child(sl.handle)
else:
h = sl.handle
return (h, sl.kind, sl.maxvalue, sl.name)
def __setstate__(self, state):
if sys.version_info < (3, 4):
h, kind, maxvalue, name = state
self._semlock = SemLockC(h, kind, maxvalue, name=name)
else:
self._semlock = SemLockC._rebuild(*state)
util.debug('recreated blocker with handle %r and name %s' %
(state[0], state[3]))
self._make_methods()
@staticmethod
def _make_name():
if sys.version_info < (3, 4):
return None
return '%s-%s' % (process.current_process()._config['semprefix'],
next(SemLock._rand))
# 2nd stage: validate that locking is available on the system and
# issue a warning if not
if mp is not None:
try:
# try to create a named semaphore using SemLock to make sure they are
# available on this platform. We use the low level object
# _multiprocessing.SemLock to avoid spawning a resource tracker on
# Unix system or changing the default backend.
import tempfile
from _multiprocessing import SemLock
_rand = tempfile._RandomNameSequence()
for i in range(100):
try:
name = '/joblib-{}-{}'.format(os.getpid(), next(_rand))
_sem = SemLock(0, 0, 1, name=name, unlink=True)
del _sem # cleanup
break
except FileExistsError: # pragma: no cover
if i >= 99:
raise FileExistsError('cannot find name for semaphore')
except (FileExistsError, AttributeError, ImportError, OSError) as e:
mp = None
warnings.warn('%s. joblib will operate in serial mode' % (e, ))
# 3rd stage: backward compat for the assert_spawning helper
if mp is not None:
from multiprocessing.context import assert_spawning
else:
assert_spawning = None
def test_recursive(self):
from _multiprocessing import SemLock
kind = self.RECURSIVE
value = 1
maxvalue = 1
# the following line gets OSError: [Errno 38] Function not implemented
# if /dev/shm is not mounted on Linux
sem = SemLock(kind, value, maxvalue)
sem.acquire()
sem.release()
assert sem._count() == 0
sem.acquire()
sem.release()
# now recursively
sem.acquire()
sem.acquire()
assert sem._count() == 2
sem.release()
sem.release()
def test_semaphore(self):
from _multiprocessing import SemLock
import sys
assert SemLock.SEM_VALUE_MAX > 10
kind = self.SEMAPHORE
value = 1
maxvalue = 1
# the following line gets OSError: [Errno 38] Function not implemented
# if /dev/shm is not mounted on Linux
sem = SemLock(kind, value, maxvalue)
assert sem.kind == kind
assert sem.maxvalue == maxvalue
assert isinstance(sem.handle, (int, long))
assert sem._count() == 0
if sys.platform == "darwin":
raises(NotImplementedError, "sem._get_value()")
else:
assert sem._get_value() == 1
assert sem._is_zero() == False
sem.acquire()
assert sem._is_mine()
assert sem._count() == 1
if sys.platform == "darwin":
raises(NotImplementedError, "sem._get_value()")
else:
assert sem._get_value() == 0
assert sem._is_zero() == True
sem.release()
assert sem._count() == 0
sem.acquire()
sem._after_fork()
assert sem._count() == 0
def __init__(self, value=1):
SemLock.__init__(self, SEMAPHORE, value, value)
def __setstate__(self, state):
self._semlock = _SemLock._rebuild(*state)
util.debug('recreated blocker with handle %r and name "%s"'
% (state[0], state[3]))
self._make_methods()
def __setstate__(self, state):
self._semlock = _SemLock._rebuild(*state)
util.debug('recreated blocker with handle %r and name "%s"'
% (state[0], state[3]))
self._make_methods()
def test_unlink(self):
from _multiprocessing import SemLock
sem = SemLock(self.SEMAPHORE, 1, 1, '/mp-123', unlink=True)
assert sem._count() == 0
def __init__(self, value=1):
SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX)
def test_semaphore(self):
from _multiprocessing import SemLock
import sys
assert SemLock.SEM_VALUE_MAX > 10
kind = self.SEMAPHORE
value = 1
maxvalue = 1
# the following line gets OSError: [Errno 38] Function not implemented
# if /dev/shm is not mounted on Linux
sem = SemLock(kind, value, maxvalue, "1", unlink=True)
assert sem.kind == kind
assert sem.maxvalue == maxvalue
assert isinstance(sem.handle, int)
assert sem.name is None
assert sem._count() == 0
if sys.platform == 'darwin':
raises(NotImplementedError, 'sem._get_value()')
else:
assert sem._get_value() == 1
assert sem._is_zero() == False
sem.acquire()
assert sem._is_mine()
assert sem._count() == 1
if sys.platform == 'darwin':
raises(NotImplementedError, 'sem._get_value()')
else:
assert sem._get_value() == 0
assert sem._is_zero() == True
sem.release()
assert sem._count() == 0
sem.acquire()
sem._after_fork()
assert sem._count() == 0
def __init__(self):
SemLock.__init__(self, SEMAPHORE, 1, 1)
def test_recursive(self):
from _multiprocessing import SemLock
kind = self.RECURSIVE
value = 1
maxvalue = 1
# the following line gets OSError: [Errno 38] Function not implemented
# if /dev/shm is not mounted on Linux
sem = SemLock(kind, value, maxvalue, "2", unlink=True)
sem.acquire()
sem.release()
assert sem._count() == 0
sem.acquire()
sem.release()
# now recursively
sem.acquire()
sem.acquire()
assert sem._count() == 2
sem.release()
sem.release()
def __init__(self, value=1):
SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX)
def test_semaphore(self):
from _multiprocessing import SemLock
import sys
assert SemLock.SEM_VALUE_MAX > 10
kind = self.SEMAPHORE
value = 1
maxvalue = 1
sem = SemLock(kind, value, maxvalue)
assert sem.kind == kind
assert sem.maxvalue == maxvalue
assert isinstance(sem.handle, (int, long))
assert sem._count() == 0
if sys.platform == 'darwin':
raises(NotImplementedError, 'sem._get_value()')
else:
assert sem._get_value() == 1
assert sem._is_zero() == False
sem.acquire()
assert sem._is_mine()
assert sem._count() == 1
if sys.platform == 'darwin':
raises(NotImplementedError, 'sem._get_value()')
else:
assert sem._get_value() == 0
assert sem._is_zero() == True
sem.release()
assert sem._count() == 0
sem.acquire()
sem._after_fork()
assert sem._count() == 0
def __init__(self):
SemLock.__init__(self, SEMAPHORE, 1, 1)
def test_recursive(self):
from _multiprocessing import SemLock
kind = self.RECURSIVE
value = 1
maxvalue = 1
sem = SemLock(kind, value, maxvalue)
sem.acquire()
sem.release()
assert sem._count() == 0
sem.acquire()
sem.release()
# now recursively
sem.acquire()
sem.acquire()
assert sem._count() == 2
sem.release()
sem.release()
def __init__(self, value=1, *, ctx):
SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx)
def __setstate__(self, state):
self._semlock = _SemLock._rebuild(*state)
util.debug(
f'recreated blocker with handle {state[0]!r} and name "{state[3]}"'
)
self._make_methods()
def __init__(self, *, ctx):
SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1, ctx=ctx)
def test_semaphore(self):
from _multiprocessing import SemLock
import sys
assert SemLock.SEM_VALUE_MAX > 10
kind = self.SEMAPHORE
value = 1
maxvalue = 1
sem = SemLock(kind, value, maxvalue)
assert sem.kind == kind
assert sem.maxvalue == maxvalue
assert isinstance(sem.handle, (int, long))
assert sem._count() == 0
if sys.platform == 'darwin':
raises(NotImplementedError, 'sem._get_value()')
else:
assert sem._get_value() == 1
assert sem._is_zero() == False
sem.acquire()
assert sem._is_mine()
assert sem._count() == 1
if sys.platform == 'darwin':
raises(NotImplementedError, 'sem._get_value()')
else:
assert sem._get_value() == 0
assert sem._is_zero() == True
sem.release()
assert sem._count() == 0
sem.acquire()
sem._after_fork()
assert sem._count() == 0
def __init__(self, value=1, *, ctx):
SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx)
def test_recursive(self):
from _multiprocessing import SemLock
kind = self.RECURSIVE
value = 1
maxvalue = 1
sem = SemLock(kind, value, maxvalue)
sem.acquire()
sem.release()
assert sem._count() == 0
sem.acquire()
sem.release()
# now recursively
sem.acquire()
sem.acquire()
assert sem._count() == 2
sem.release()
sem.release()
def __init__(self, *, ctx):
SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1, ctx=ctx)
def __setstate__(self, state):
reader, writer = state
self._reader = SemLock._rebuild(*reader)
self._writer = SemLock._rebuild(*writer)