Esempio n. 1
0
    def test_semaphore_wait(self):
        from _multiprocessing import SemLock
        kind = self.SEMAPHORE
        value = 1
        maxvalue = 1
        sem = SemLock(kind, value, maxvalue)

        assert sem.acquire()
        assert not sem.acquire(timeout=0.1)
    def test_semaphore_contextmanager(self):
        from _multiprocessing import SemLock
        kind = self.SEMAPHORE
        value = 1
        maxvalue = 1
        sem = SemLock(kind, value, maxvalue, "5", unlink=True)

        with sem:
            assert sem._count() == 1
        assert sem._count() == 0
Esempio n. 3
0
    def test_semaphore_contextmanager(self):
        from _multiprocessing import SemLock
        kind = self.SEMAPHORE
        value = 1
        maxvalue = 1
        sem = SemLock(kind, value, maxvalue)

        with sem:
            assert sem._count() == 1
        assert sem._count() == 0
Esempio n. 4
0
    def test_semaphore_wait(self):
        from _multiprocessing import SemLock
        kind = self.SEMAPHORE
        value = 1
        maxvalue = 1
        sem = SemLock(kind, value, maxvalue)

        res = sem.acquire()
        assert res == True
        res = sem.acquire(timeout=0.1)
        assert res == False
Esempio n. 5
0
def test_notify_all():
    """A low-level variation on test_notify_all() in lib-python's
    test_multiprocessing.py
    """
    N_THREADS = 1000
    lock = SemLock(0, 1, 1)
    results = []

    def f(n):
        if lock.acquire(timeout=5.):
            results.append(n)
            lock.release()
        else:
            print("lock acquire timed out!")

    threads = [Thread(target=f, args=(i, )) for i in range(N_THREADS)]
    n_started = N_THREADS
    with lock:
        for t in threads:
            try:
                t.start()
            except thread.error:
                # too many threads for this system
                t.started = False
                n_started -= 1
            else:
                t.started = True
        time.sleep(0.1)
        print("started %d threads" % n_started)
    for t in threads:
        if t.started:
            t.join()
    assert len(results) == n_started
Esempio n. 6
0
    def __init__(self, kind, value, maxvalue):
        name = 'loky'
        unlink_now = sys.platform == 'win32' or name == 'fork'
        for i in range(100):
            try:
                self._semlock = SemLockC(kind, value, maxvalue,
                                         SemLock._make_name(), unlink_now)
            except FileExistsError:
                pass
            else:
                break
        else:
            raise FileExistsError('cannot find name for semaphore')

        self._make_methods()

        if sys.platform != 'win32':

            def _after_fork(obj):
                obj._semlock._after_fork()

            util.register_after_fork(self, _after_fork)

        if self._semlock.name is not None:
            # We only get here if we are on Unix with forking
            # disabled.  When the object is garbage collected or the
            # process shuts down we unlink the semaphore name
            from .semaphore_tracker import register
            if sys.version_info < (3, 4):
                register(self._semlock.name)
            util.Finalize(self,
                          SemLock._cleanup, (self._semlock.name, ),
                          exitpriority=0)
Esempio n. 7
0
def test_SemLock_raises_on_non_string_name():    
    from _multiprocessing import SemLock
    try :
        SemLock(kind=1, value=1, name={1:2}, maxvalue=1, unlink=1)
    except TypeError:
        raised = True
    assert raised
Esempio n. 8
0
    def test_semaphore_rebuild(self):
        from _multiprocessing import SemLock
        kind = self.SEMAPHORE
        value = 1
        maxvalue = 1
        sem = SemLock(kind, value, maxvalue)

        sem2 = SemLock._rebuild(sem.handle, kind, value)
        assert sem.handle == sem2.handle
Esempio n. 9
0
 def test_semaphore_rebuild(self):
     import sys
     if sys.platform == 'win32':
         from _multiprocessing import SemLock
         def sem_unlink(*args):
             pass
     else:
         from _multiprocessing import SemLock, sem_unlink
     kind = self.SEMAPHORE
     value = 1
     maxvalue = 1
     sem = SemLock(kind, value, maxvalue, "4.2", unlink=False)
     try:
         sem2 = SemLock._rebuild(-1, kind, value, "4.2")
         #assert sem.handle != sem2.handle---even though they come
         # from different calls to sem_open(), on Linux at least,
         # they are the same pointer
         sem2 = SemLock._rebuild(sem.handle, kind, value, None)
         assert sem.handle == sem2.handle
     finally:
         sem_unlink("4.2")
Esempio n. 10
0
 def test_in_threads(self):
     from _multiprocessing import SemLock
     from threading import Thread
     from time import sleep
     l = SemLock(0, 1, 1)
     if self.runappdirect:
         def f(id):
             for i in range(10000):
                 pass
     else:
         def f(id):
             for i in range(1000):
                 # reduce the probability of thread switching
                 # at exactly the wrong time in semlock_acquire
                 for j in range(10):
                     pass
     threads = [Thread(None, f, args=(i,)) for i in range(2)]
     [t.start() for t in threads]
     # if the RLock calls to sem_wait and sem_post do not match,
     # one of the threads will block and the call to join will fail
     [t.join() for t in threads]
Esempio n. 11
0
 def __init__(self):
     SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1)
Esempio n. 12
0
 def __init__(self, value=1, *, ctx):
     SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx)
Esempio n. 13
0
 def __init__(self, *, ctx):
     SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx)
Esempio n. 14
0
 def __init__(self, value=1, *, ctx):
     SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx)
Esempio n. 15
0
 def __init__(self, *, ctx):
     SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx)
Esempio n. 16
0
 def __init__(self):
     SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1)
Esempio n. 17
0
 def __init__(self, value=1):
     SemLock.__init__(self, SEMAPHORE, value, value)
Esempio n. 18
0
    def test_semaphore_maxvalue(self):
        from _multiprocessing import SemLock
        import sys
        kind = self.SEMAPHORE
        value = SemLock.SEM_VALUE_MAX
        maxvalue = SemLock.SEM_VALUE_MAX
        sem = SemLock(kind, value, maxvalue)

        for i in range(10):
            res = sem.acquire()
            assert res == True
            assert sem._count() == i + 1
            if sys.platform != 'darwin':
                assert sem._get_value() == maxvalue - (i + 1)

        value = 0
        maxvalue = SemLock.SEM_VALUE_MAX
        sem = SemLock(kind, value, maxvalue)

        for i in range(10):
            sem.release()
            assert sem._count() == -(i + 1)
            if sys.platform != 'darwin':
                assert sem._get_value() == i + 1
Esempio n. 19
0
class SemLock(object):

    _rand = tempfile._RandomNameSequence()

    def __init__(self, kind, value, maxvalue):
        name = 'loky'
        unlink_now = sys.platform == 'win32' or name == 'fork'
        for i in range(100):
            try:
                self._semlock = SemLockC(kind, value, maxvalue,
                                         SemLock._make_name(), unlink_now)
            except FileExistsError:
                pass
            else:
                break
        else:
            raise FileExistsError('cannot find name for semaphore')

        self._make_methods()

        if sys.platform != 'win32':

            def _after_fork(obj):
                obj._semlock._after_fork()

            util.register_after_fork(self, _after_fork)

        if self._semlock.name is not None:
            # We only get here if we are on Unix with forking
            # disabled.  When the object is garbage collected or the
            # process shuts down we unlink the semaphore name
            from .semaphore_tracker import register
            if sys.version_info < (3, 4):
                register(self._semlock.name)
            util.Finalize(self,
                          SemLock._cleanup, (self._semlock.name, ),
                          exitpriority=0)

    @staticmethod
    def _cleanup(name):
        from .semaphore_tracker import unregister
        sem_unlink(name)
        unregister(name)

    def _make_methods(self):
        self.acquire = self._semlock.acquire
        self.release = self._semlock.release

    def __enter__(self):
        return self._semlock.acquire()

    def __exit__(self, *args):
        return self._semlock.release(*args)

    def __getstate__(self):
        assert is_spawning()
        sl = self._semlock
        if sys.platform == 'win32':
            h = get_spawning_popen().duplicate_for_child(sl.handle)
        else:
            h = sl.handle
        return (h, sl.kind, sl.maxvalue, sl.name)

    def __setstate__(self, state):
        if sys.version_info < (3, 4):
            h, kind, maxvalue, name = state
            self._semlock = SemLockC(h, kind, maxvalue, name=name)
        else:
            self._semlock = SemLockC._rebuild(*state)
        util.debug('recreated blocker with handle %r and name %s' %
                   (state[0], state[3]))
        self._make_methods()

    @staticmethod
    def _make_name():
        if sys.version_info < (3, 4):
            return None
        return '%s-%s' % (process.current_process()._config['semprefix'],
                          next(SemLock._rand))
Esempio n. 20
0
# 2nd stage: validate that locking is available on the system and
#            issue a warning if not
if mp is not None:
    try:
        # try to create a named semaphore using SemLock to make sure they are
        # available on this platform. We use the low level object
        # _multiprocessing.SemLock to avoid spawning a resource tracker on
        # Unix system or changing the default backend.
        import tempfile
        from _multiprocessing import SemLock

        _rand = tempfile._RandomNameSequence()
        for i in range(100):
            try:
                name = '/joblib-{}-{}'.format(os.getpid(), next(_rand))
                _sem = SemLock(0, 0, 1, name=name, unlink=True)
                del _sem  # cleanup
                break
            except FileExistsError:  # pragma: no cover
                if i >= 99:
                    raise FileExistsError('cannot find name for semaphore')
    except (FileExistsError, AttributeError, ImportError, OSError) as e:
        mp = None
        warnings.warn('%s.  joblib will operate in serial mode' % (e, ))

# 3rd stage: backward compat for the assert_spawning helper
if mp is not None:
    from multiprocessing.context import assert_spawning
else:
    assert_spawning = None
Esempio n. 21
0
    def test_recursive(self):
        from _multiprocessing import SemLock

        kind = self.RECURSIVE
        value = 1
        maxvalue = 1
        # the following line gets OSError: [Errno 38] Function not implemented
        # if /dev/shm is not mounted on Linux
        sem = SemLock(kind, value, maxvalue)

        sem.acquire()
        sem.release()
        assert sem._count() == 0
        sem.acquire()
        sem.release()

        # now recursively
        sem.acquire()
        sem.acquire()
        assert sem._count() == 2
        sem.release()
        sem.release()
Esempio n. 22
0
    def test_semaphore(self):
        from _multiprocessing import SemLock
        import sys

        assert SemLock.SEM_VALUE_MAX > 10

        kind = self.SEMAPHORE
        value = 1
        maxvalue = 1
        # the following line gets OSError: [Errno 38] Function not implemented
        # if /dev/shm is not mounted on Linux
        sem = SemLock(kind, value, maxvalue)
        assert sem.kind == kind
        assert sem.maxvalue == maxvalue
        assert isinstance(sem.handle, (int, long))

        assert sem._count() == 0
        if sys.platform == "darwin":
            raises(NotImplementedError, "sem._get_value()")
        else:
            assert sem._get_value() == 1
        assert sem._is_zero() == False
        sem.acquire()
        assert sem._is_mine()
        assert sem._count() == 1
        if sys.platform == "darwin":
            raises(NotImplementedError, "sem._get_value()")
        else:
            assert sem._get_value() == 0
        assert sem._is_zero() == True
        sem.release()
        assert sem._count() == 0

        sem.acquire()
        sem._after_fork()
        assert sem._count() == 0
Esempio n. 23
0
 def __init__(self, value=1):
     SemLock.__init__(self, SEMAPHORE, value, value)
Esempio n. 24
0
 def __setstate__(self, state):
     self._semlock = _SemLock._rebuild(*state)
     util.debug('recreated blocker with handle %r and name "%s"'
                % (state[0], state[3]))
     self._make_methods()
Esempio n. 25
0
 def __setstate__(self, state):
     self._semlock = _SemLock._rebuild(*state)
     util.debug('recreated blocker with handle %r and name "%s"'
                % (state[0], state[3]))
     self._make_methods()
Esempio n. 26
0
 def test_unlink(self):
     from _multiprocessing import SemLock
     sem = SemLock(self.SEMAPHORE, 1, 1, '/mp-123', unlink=True)
     assert sem._count() == 0
Esempio n. 27
0
 def __init__(self, value=1):
     SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX)
Esempio n. 28
0
    def test_semaphore(self):
        from _multiprocessing import SemLock
        import sys
        assert SemLock.SEM_VALUE_MAX > 10

        kind = self.SEMAPHORE
        value = 1
        maxvalue = 1
        # the following line gets OSError: [Errno 38] Function not implemented
        # if /dev/shm is not mounted on Linux
        sem = SemLock(kind, value, maxvalue, "1", unlink=True)
        assert sem.kind == kind
        assert sem.maxvalue == maxvalue
        assert isinstance(sem.handle, int)
        assert sem.name is None

        assert sem._count() == 0
        if sys.platform == 'darwin':
            raises(NotImplementedError, 'sem._get_value()')
        else:
            assert sem._get_value() == 1
        assert sem._is_zero() == False
        sem.acquire()
        assert sem._is_mine()
        assert sem._count() == 1
        if sys.platform == 'darwin':
            raises(NotImplementedError, 'sem._get_value()')
        else:
            assert sem._get_value() == 0
        assert sem._is_zero() == True
        sem.release()
        assert sem._count() == 0

        sem.acquire()
        sem._after_fork()
        assert sem._count() == 0
Esempio n. 29
0
 def __init__(self):
     SemLock.__init__(self, SEMAPHORE, 1, 1)
Esempio n. 30
0
    def test_recursive(self):
        from _multiprocessing import SemLock
        kind = self.RECURSIVE
        value = 1
        maxvalue = 1
        # the following line gets OSError: [Errno 38] Function not implemented
        # if /dev/shm is not mounted on Linux
        sem = SemLock(kind, value, maxvalue, "2", unlink=True)

        sem.acquire()
        sem.release()
        assert sem._count() == 0
        sem.acquire()
        sem.release()

        # now recursively
        sem.acquire()
        sem.acquire()
        assert sem._count() == 2
        sem.release()
        sem.release()
Esempio n. 31
0
 def __init__(self, value=1):
     SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX)
Esempio n. 32
0
    def test_semaphore(self):
        from _multiprocessing import SemLock
        import sys
        assert SemLock.SEM_VALUE_MAX > 10

        kind = self.SEMAPHORE
        value = 1
        maxvalue = 1
        sem = SemLock(kind, value, maxvalue)
        assert sem.kind == kind
        assert sem.maxvalue == maxvalue
        assert isinstance(sem.handle, (int, long))

        assert sem._count() == 0
        if sys.platform == 'darwin':
            raises(NotImplementedError, 'sem._get_value()')
        else:
            assert sem._get_value() == 1
        assert sem._is_zero() == False
        sem.acquire()
        assert sem._is_mine()
        assert sem._count() == 1
        if sys.platform == 'darwin':
            raises(NotImplementedError, 'sem._get_value()')
        else:
            assert sem._get_value() == 0
        assert sem._is_zero() == True
        sem.release()
        assert sem._count() == 0

        sem.acquire()
        sem._after_fork()
        assert sem._count() == 0
Esempio n. 33
0
 def __init__(self):
     SemLock.__init__(self, SEMAPHORE, 1, 1)
Esempio n. 34
0
    def test_recursive(self):
        from _multiprocessing import SemLock
        kind = self.RECURSIVE
        value = 1
        maxvalue = 1
        sem = SemLock(kind, value, maxvalue)

        sem.acquire()
        sem.release()
        assert sem._count() == 0
        sem.acquire()
        sem.release()

        # now recursively
        sem.acquire()
        sem.acquire()
        assert sem._count() == 2
        sem.release()
        sem.release()
Esempio n. 35
0
 def __init__(self, value=1, *, ctx):
     SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx)
Esempio n. 36
0
 def __setstate__(self, state):
     self._semlock = _SemLock._rebuild(*state)
     util.debug(
         f'recreated blocker with handle {state[0]!r} and name "{state[3]}"'
     )
     self._make_methods()
Esempio n. 37
0
 def __init__(self, *, ctx):
     SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1, ctx=ctx)
Esempio n. 38
0
    def test_semaphore(self):
        from _multiprocessing import SemLock
        import sys
        assert SemLock.SEM_VALUE_MAX > 10

        kind = self.SEMAPHORE
        value = 1
        maxvalue = 1
        sem = SemLock(kind, value, maxvalue)
        assert sem.kind == kind
        assert sem.maxvalue == maxvalue
        assert isinstance(sem.handle, (int, long))

        assert sem._count() == 0
        if sys.platform == 'darwin':
            raises(NotImplementedError, 'sem._get_value()')
        else:
            assert sem._get_value() == 1
        assert sem._is_zero() == False
        sem.acquire()
        assert sem._is_mine()
        assert sem._count() == 1
        if sys.platform == 'darwin':
            raises(NotImplementedError, 'sem._get_value()')
        else:
            assert sem._get_value() == 0
        assert sem._is_zero() == True
        sem.release()
        assert sem._count() == 0

        sem.acquire()
        sem._after_fork()
        assert sem._count() == 0
Esempio n. 39
0
 def __init__(self, value=1, *, ctx):
     SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx)
Esempio n. 40
0
    def test_recursive(self):
        from _multiprocessing import SemLock
        kind = self.RECURSIVE
        value = 1
        maxvalue = 1
        sem = SemLock(kind, value, maxvalue)

        sem.acquire()
        sem.release()
        assert sem._count() == 0
        sem.acquire()
        sem.release()

        # now recursively
        sem.acquire()
        sem.acquire()
        assert sem._count() == 2
        sem.release()
        sem.release()
Esempio n. 41
0
 def __init__(self, *, ctx):
     SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1, ctx=ctx)
Esempio n. 42
0
 def __setstate__(self, state):
     reader, writer = state
     self._reader = SemLock._rebuild(*reader)
     self._writer = SemLock._rebuild(*writer)