Example #1
0
    def acquire (self, blocking = True):
        """Acquire a semaphore.

        When invoked without arguments: if the internal counter is larger than
        zero on entry, decrement it by one and return immediately. If it is zero
        on entry, block, waiting until some other thread has called release() to
        make it larger than zero. This is done with proper interlocking so that
        if multiple acquire() calls are blocked, release() will wake exactly one
        of them up. The implementation may pick one at random, so the order in
        which blocked threads are awakened should not be relied on. There is no
        return value in this case.

        When invoked with blocking set to true, do the same thing as when called
        without arguments, and return true.

        When invoked with blocking set to false, do not block. If a call without
        an argument would block, return false immediately; otherwise, do the
        same thing as when called without arguments, and return true."""
        if not blocking and self.locked():
            return False
        if self.counter <= 0:
            self._waiters.add(greenthread.getcurrent())
            try:
                while self.counter <= 0:
                    hubs.get_hub().switch()
            finally:
                self._waiters.discard(greenthread.getcurrent())
        self.counter -= 1
        return True
Example #2
0
    def spawn (self, function, *args, **kwargs):
        """
        Run the *function* with its arguments in its own green thread.
        Returns the :class:`GreenThread <evy.greenthread.GreenThread>`
        object that is running the function, which can be used to retrieve the
        results.

        If the pool is currently at capacity, ``spawn`` will block until one of
        the running greenthreads completes its task and frees up a slot.

        This function is reentrant; *function* can call ``spawn`` on the same
        pool without risk of deadlocking the whole thing.
        """
        # if reentering an empty pool, don't try to wait on a coroutine freeing
        # itself -- instead, just execute in the current coroutine
        current = greenthread.getcurrent()
        if self.sem.locked() and current in self.coroutines_running:
            # a bit hacky to use the GT without switching to it
            gt = greenthread.GreenThread(current)
            gt.main(function, args, kwargs)
            return gt
        else:
            self.sem.acquire()
            gt = greenthread.spawn(function, *args, **kwargs)
            if not self.coroutines_running:
                self.no_coros_running = event.Event()
            self.coroutines_running.add(gt)
            gt.link(self._spawn_done)
        return gt
Example #3
0
 def waitall (self):
     """
     Waits until all greenthreads in the pool are finished working.
     """
     assert greenthread.getcurrent() not in self.coroutines_running,\
     "Calling waitall() from within one of the GreenPool's greenthreads will never terminate."
     if self.running():
         self.no_coros_running.wait()
Example #4
0
File: queue.py Project: inercia/evy
 def wait (self):
     """Wait until switch() or throw() is called.
     """
     assert self.greenlet is None, 'This Waiter is already used by %r' % (self.greenlet, )
     self.greenlet = getcurrent()
     try:
         return get_hub().switch()
     finally:
         self.greenlet = None
Example #5
0
 def wait(self):
     """Wait until switch() or throw() is called.
     """
     assert self.greenlet is None, 'This Waiter is already used by %r' % (
         self.greenlet, )
     self.greenlet = getcurrent()
     try:
         return get_hub().switch()
     finally:
         self.greenlet = None
Example #6
0
File: queue.py Project: inercia/evy
 def throw (self, *throw_args):
     """Make greenlet calling wait() wake up (if there is a wait()).
     Can only be called from Hub's greenlet.
     """
     assert getcurrent() is get_hub().greenlet, "Can only use Waiter.switch method from the mainloop"
     if self.greenlet is not None:
         try:
             self.greenlet.throw(*throw_args)
         except:
             traceback.print_exc()
Example #7
0
File: queue.py Project: inercia/evy
 def switch (self, value = None):
     """Wake up the greenlet that is calling wait() currently (if there is one).
     Can only be called from Hub's greenlet.
     """
     assert getcurrent() is get_hub().greenlet, "Can only use Waiter.switch method from the mainloop"
     if self.greenlet is not None:
         try:
             self.greenlet.switch(value)
         except:
             traceback.print_exc()
Example #8
0
def select(read_list, write_list, error_list, timeout=None):
    # error checking like this is required by the stdlib unit tests
    if timeout is not None:
        try:
            timeout = float(timeout)
        except ValueError:
            raise TypeError("Expected number for timeout")
    hub = get_hub()
    t = None
    current = getcurrent()
    assert hub.greenlet is not current, 'do not call blocking functions from the mainloop'
    ds = {}

    for r in read_list:
        ds[get_fileno(r)] = {'read': r}

    for w in write_list:
        ds.setdefault(get_fileno(w), {})['write'] = w

    for e in error_list:
        ds.setdefault(get_fileno(e), {})['error'] = e

    listeners = []

    def on_read(d):
        original = ds[get_fileno(d)]['read']
        current.switch(([original], [], []))

    def on_write(d):
        original = ds[get_fileno(d)]['write']
        current.switch(([], [original], []))

    def on_error(d, _err=None):
        original = ds[get_fileno(d)]['error']
        current.switch(([], [], [original]))

    def on_timeout():
        current.switch(([], [], []))

    if timeout is not None:
        t = hub.schedule_call_global(timeout, on_timeout)
    try:
        for k, v in ds.iteritems():
            if v.get('read'):
                listeners.append(hub.add(hub.READ, k, on_read))
            if v.get('write'):
                listeners.append(hub.add(hub.WRITE, k, on_write))
        try:
            return hub.switch()
        finally:
            for l in listeners:
                hub.remove(l)
    finally:
        if t is not None:
            t.cancel()
Example #9
0
def select (read_list, write_list, error_list, timeout = None):
    # error checking like this is required by the stdlib unit tests
    if timeout is not None:
        try:
            timeout = float(timeout)
        except ValueError:
            raise TypeError("Expected number for timeout")
    hub = get_hub()
    t = None
    current = getcurrent()
    assert hub.greenlet is not current, 'do not call blocking functions from the mainloop'
    ds = {}

    for r in read_list:
        ds[get_fileno(r)] = {'read': r}

    for w in write_list:
        ds.setdefault(get_fileno(w), {})['write'] = w

    for e in error_list:
        ds.setdefault(get_fileno(e), {})['error'] = e

    listeners = []

    def on_read (d):
        original = ds[get_fileno(d)]['read']
        current.switch(([original], [], []))

    def on_write (d):
        original = ds[get_fileno(d)]['write']
        current.switch(([], [original], []))

    def on_error (d, _err = None):
        original = ds[get_fileno(d)]['error']
        current.switch(([], [], [original]))

    def on_timeout ():
        current.switch(([], [], []))

    if timeout is not None:
        t = hub.schedule_call_global(timeout, on_timeout)
    try:
        for k, v in ds.iteritems():
            if v.get('read'):
                listeners.append(hub.add(hub.READ, k, on_read))
            if v.get('write'):
                listeners.append(hub.add(hub.WRITE, k, on_write))
        try:
            return hub.switch()
        finally:
            for l in listeners:
                hub.remove(l)
    finally:
        if t is not None:
            t.cancel()
Example #10
0
 def switch(self, value=None):
     """Wake up the greenlet that is calling wait() currently (if there is one).
     Can only be called from Hub's greenlet.
     """
     assert getcurrent() is get_hub(
     ).greenlet, "Can only use Waiter.switch method from the mainloop"
     if self.greenlet is not None:
         try:
             self.greenlet.switch(value)
         except:
             traceback.print_exc()
Example #11
0
 def throw(self, *throw_args):
     """Make greenlet calling wait() wake up (if there is a wait()).
     Can only be called from Hub's greenlet.
     """
     assert getcurrent() is get_hub(
     ).greenlet, "Can only use Waiter.switch method from the mainloop"
     if self.greenlet is not None:
         try:
             self.greenlet.throw(*throw_args)
         except:
             traceback.print_exc()
Example #12
0
def _patch(thrl):
    greens = object.__getattribute__(thrl, '_local__greens')

    # until we can store the localdict on greenlets themselves,
    # we store it in _local__greens on the local object
    cur = greenthread.getcurrent()
    if cur not in greens:
        # must be the first time we've seen this greenlet, call __init__
        greens[cur] = {}
        cls = type(thrl)
        if cls.__init__ is not object.__init__:
            args, kw = object.__getattribute__(thrl, '_local__args')
            thrl.__init__(*args, **kw)
    object.__setattr__(thrl, '__dict__', greens[cur])
Example #13
0
 def _spawn_n_impl (self, func, args, kwargs, coro):
     try:
         try:
             func(*args, **kwargs)
         except (KeyboardInterrupt, SystemExit, greenlet.GreenletExit):
             raise
         except:
             if DEBUG:
                 traceback.print_exc()
     finally:
         if coro is None:
             return
         else:
             coro = greenthread.getcurrent()
             self._spawn_done(coro)
Example #14
0
 def spawn_n (self, function, *args, **kwargs):
     """Create a greenthread to run the *function*, the same as
     :meth:`spawn`.  The difference is that :meth:`spawn_n` returns
     None; the results of *function* are not retrievable.
     """
     # if reentering an empty pool, don't try to wait on a coroutine freeing
     # itself -- instead, just execute in the current coroutine
     current = greenthread.getcurrent()
     if self.sem.locked() and current in self.coroutines_running:
         self._spawn_n_impl(function, args, kwargs, None)
     else:
         self.sem.acquire()
         g = greenthread.spawn_n(self._spawn_n_impl,
                                 function, args, kwargs, True)
         if not self.coroutines_running:
             self.no_coros_running = event.Event()
         self.coroutines_running.add(g)
Example #15
0
    def put(self, item, block=True, timeout=None):
        """
        Put an item into the queue.

        If optional arg *block* is true and *timeout* is ``None`` (the default),
        block if necessary until a free slot is available. If *timeout* is
        a positive number, it blocks at most *timeout* seconds and raises
        the :class:`Full` exception if no free slot was available within that time.
        Otherwise (*block* is false), put an item on the queue if a free slot
        is immediately available, else raise the :class:`Full` exception (*timeout*
        is ignored in that case).
        """
        if self.maxsize is None or self.qsize() < self.maxsize:
            # there's a free slot, put an item right away
            self._put(item)
            if self.getters:
                self._schedule_unlock()
        elif not block and get_hub().greenlet is getcurrent():
            # we're in the mainloop, so we cannot wait; we can switch() to other greenlets though
            # find a getter and deliver an item to it
            while self.getters:
                getter = self.getters.pop()
                if getter:
                    self._put(item)
                    item = self._get()
                    getter.switch(item)
                    return
            raise Full
        elif block:
            waiter = ItemWaiter(item)
            self.putters.add(waiter)
            timeout = Timeout(timeout, Full)
            try:
                if self.getters:
                    self._schedule_unlock()
                result = waiter.wait()
                assert result is waiter, "Invalid switch into Queue.put: %r" % (
                    result, )
                if waiter.item is not _NONE:
                    self._put(item)
            finally:
                timeout.cancel()
                self.putters.discard(waiter)
        else:
            raise Full
Example #16
0
    def test_calls_init (self):
        init_args = []

        class Init(corolocal.local):
            def __init__ (self, *args):
                init_args.append((args, getcurrent()))

        my_local = Init(1, 2, 3)
        self.assertEqual(init_args[0][0], (1, 2, 3))
        self.assertEqual(init_args[0][1], getcurrent())

        def do_something ():
            my_local.foo = 'bar'
            self.assertEqual(len(init_args), 2, init_args)
            self.assertEqual(init_args[1][0], (1, 2, 3))
            self.assertEqual(init_args[1][1], getcurrent())

        spawn(do_something).wait()
Example #17
0
    def test_calls_init(self):
        init_args = []

        class Init(corolocal.local):
            def __init__(self, *args):
                init_args.append((args, getcurrent()))

        my_local = Init(1, 2, 3)
        self.assertEqual(init_args[0][0], (1, 2, 3))
        self.assertEqual(init_args[0][1], getcurrent())

        def do_something():
            my_local.foo = 'bar'
            self.assertEqual(len(init_args), 2, init_args)
            self.assertEqual(init_args[1][0], (1, 2, 3))
            self.assertEqual(init_args[1][1], getcurrent())

        spawn(do_something).wait()
Example #18
0
File: queue.py Project: inercia/evy
    def put (self, item, block = True, timeout = None):
        """
        Put an item into the queue.

        If optional arg *block* is true and *timeout* is ``None`` (the default),
        block if necessary until a free slot is available. If *timeout* is
        a positive number, it blocks at most *timeout* seconds and raises
        the :class:`Full` exception if no free slot was available within that time.
        Otherwise (*block* is false), put an item on the queue if a free slot
        is immediately available, else raise the :class:`Full` exception (*timeout*
        is ignored in that case).
        """
        if self.maxsize is None or self.qsize() < self.maxsize:
            # there's a free slot, put an item right away
            self._put(item)
            if self.getters:
                self._schedule_unlock()
        elif not block and get_hub().greenlet is getcurrent():
            # we're in the mainloop, so we cannot wait; we can switch() to other greenlets though
            # find a getter and deliver an item to it
            while self.getters:
                getter = self.getters.pop()
                if getter:
                    self._put(item)
                    item = self._get()
                    getter.switch(item)
                    return
            raise Full
        elif block:
            waiter = ItemWaiter(item)
            self.putters.add(waiter)
            timeout = Timeout(timeout, Full)
            try:
                if self.getters:
                    self._schedule_unlock()
                result = waiter.wait()
                assert result is waiter, "Invalid switch into Queue.put: %r" % (result, )
                if waiter.item is not _NONE:
                    self._put(item)
            finally:
                timeout.cancel()
                self.putters.discard(waiter)
        else:
            raise Full
Example #19
0
def execute(meth, *args, **kwargs):
    """
    Execute *meth* in a Python thread, blocking the current coroutine/
    greenthread until the method completes.

    The primary use case for this is to wrap an object or module that is not
    amenable to monkeypatching or any of the other tricks that Eventlet uses
    to achieve cooperative yielding.  With tpool, you can force such objects to
    cooperate with green threads by sticking them in native threads, at the cost
    of some overhead.
    """
    setup()
    # if already in tpool, don't recurse into the tpool
    # also, call functions directly if we're inside an import lock, because
    # if meth does any importing (sadly common), it will hang
    my_thread = threading.currentThread()
    if my_thread in _threads or imp.lock_held() or _nthreads == 0:
        return meth(*args, **kwargs)

    cur = greenthread.getcurrent()
    # a mini mixing function to make up for the fact that hash(greenlet) doesn't
    # have much variability in the lower bits
    k = hash(cur)
    k = k + 0x2c865fd + (k >> 5)
    k = k ^ 0xc84d1b7 ^ (k >> 7)
    thread_index = k % _nthreads

    reqq, _thread = _threads[thread_index]
    e = event.Event()
    reqq.put((e, meth, args, kwargs))

    rv = e.wait()
    if isinstance(rv, tuple)\
       and len(rv) == 3\
    and isinstance(rv[1], EXC_CLASSES):
        import traceback

        (c, e, tb) = rv
        if not QUIET:
            traceback.print_exception(c, e, tb)
            traceback.print_stack()
        raise c, e, tb
    return rv
Example #20
0
def serve(sock, handle, concurrency=1000):
    """
    Runs a server on the supplied socket.  Calls the function *handle* in a
    separate greenthread for every incoming client connection.  *handle* takes
    two arguments: the client socket object, and the client address::
        
        def myhandle(client_sock, client_addr):
            print "client connected", client_addr
        
        evy.serve(evy.listen(('127.0.0.1', 9999)), myhandle)
        
    Returning from *handle* closes the client socket.
     
    :func:`serve` blocks the calling greenthread; it won't return until 
    the server completes.  If you desire an immediate return,
    spawn a new greenthread for :func:`serve`.
      
    Any uncaught exceptions raised in *handle* are raised as exceptions 
    from :func:`serve`, terminating the server, so be sure to be aware of the 
    exceptions your application can raise.  The return value of *handle* is 
    ignored.      
      
    Raise a :class:`~evy.StopServe` exception to gracefully terminate the 
    server -- that's the only way to get the server() function to return rather 
    than raise.

    The value in *concurrency* controls the maximum number of
    greenthreads that will be open at any time handling requests.  When
    the server hits the concurrency limit, it stops accepting new
    connections until the existing ones complete.
    """
    pool = GreenPool(concurrency)
    server_gt = getcurrent()

    while True:
        try:
            conn, addr = sock.accept()
            gt = pool.spawn(handle, conn, addr)
            gt.link(_stop_checker, server_gt, conn)
            conn, addr, gt = None, None, None
        except StopServe:
            return
Example #21
0
def serve (sock, handle, concurrency = 1000):
    """
    Runs a server on the supplied socket.  Calls the function *handle* in a
    separate greenthread for every incoming client connection.  *handle* takes
    two arguments: the client socket object, and the client address::
        
        def myhandle(client_sock, client_addr):
            print "client connected", client_addr
        
        evy.serve(evy.listen(('127.0.0.1', 9999)), myhandle)
        
    Returning from *handle* closes the client socket.
     
    :func:`serve` blocks the calling greenthread; it won't return until 
    the server completes.  If you desire an immediate return,
    spawn a new greenthread for :func:`serve`.
      
    Any uncaught exceptions raised in *handle* are raised as exceptions 
    from :func:`serve`, terminating the server, so be sure to be aware of the 
    exceptions your application can raise.  The return value of *handle* is 
    ignored.      
      
    Raise a :class:`~evy.StopServe` exception to gracefully terminate the 
    server -- that's the only way to get the server() function to return rather 
    than raise.

    The value in *concurrency* controls the maximum number of
    greenthreads that will be open at any time handling requests.  When
    the server hits the concurrency limit, it stops accepting new
    connections until the existing ones complete.
    """
    pool = GreenPool(concurrency)
    server_gt = getcurrent()

    while True:
        try:
            conn, addr = sock.accept()
            gt = pool.spawn(handle, conn, addr)
            gt.link(_stop_checker, server_gt, conn)
            conn, addr, gt = None, None, None
        except StopServe:
            return
Example #22
0
File: tpool.py Project: inercia/evy
def execute(meth, *args, **kwargs):
    """
    Execute *meth* in a Python thread, blocking the current coroutine/
    greenthread until the method completes.

    The primary use case for this is to wrap an object or module that is not
    amenable to monkeypatching or any of the other tricks that Eventlet uses
    to achieve cooperative yielding.  With tpool, you can force such objects to
    cooperate with green threads by sticking them in native threads, at the cost
    of some overhead.
    """
    setup()
    # if already in tpool, don't recurse into the tpool
    # also, call functions directly if we're inside an import lock, because
    # if meth does any importing (sadly common), it will hang
    my_thread = threading.currentThread()
    if my_thread in _threads or imp.lock_held() or _nthreads == 0:
        return meth(*args, **kwargs)

    cur = greenthread.getcurrent()
    # a mini mixing function to make up for the fact that hash(greenlet) doesn't
    # have much variability in the lower bits
    k = hash(cur)
    k = k + 0x2C865FD + (k >> 5)
    k = k ^ 0xC84D1B7 ^ (k >> 7)
    thread_index = k % _nthreads

    reqq, _thread = _threads[thread_index]
    e = event.Event()
    reqq.put((e, meth, args, kwargs))

    rv = e.wait()
    if isinstance(rv, tuple) and len(rv) == 3 and isinstance(rv[1], EXC_CLASSES):
        import traceback

        (c, e, tb) = rv
        if not QUIET:
            traceback.print_exception(c, e, tb)
            traceback.print_stack()
        raise c, e, tb
    return rv
Example #23
0
File: queue.py Project: inercia/evy
    def get (self, block = True, timeout = None):
        """
        Remove and return an item from the queue.

        If optional args *block* is true and *timeout* is ``None`` (the default),
        block if necessary until an item is available. If *timeout* is a positive number,
        it blocks at most *timeout* seconds and raises the :class:`Empty` exception
        if no item was available within that time. Otherwise (*block* is false), return
        an item if one is immediately available, else raise the :class:`Empty` exception
        (*timeout* is ignored in that case).
        """
        if self.qsize():
            if self.putters:
                self._schedule_unlock()
            return self._get()
        elif not block and get_hub().greenlet is getcurrent():
            # special case to make get_nowait() runnable in the mainloop greenlet
            # there are no items in the queue; try to fix the situation by unlocking putters
            while self.putters:
                putter = self.putters.pop()
                if putter:
                    putter.switch(putter)
                    if self.qsize():
                        return self._get()
            raise Empty
        elif block:
            waiter = Waiter()
            timeout = Timeout(timeout, Empty)
            try:
                self.getters.add(waiter)
                if self.putters:
                    self._schedule_unlock()
                return waiter.wait()
            finally:
                self.getters.discard(waiter)
                timeout.cancel()
        else:
            raise Empty
Example #24
0
    def get(self, block=True, timeout=None):
        """
        Remove and return an item from the queue.

        If optional args *block* is true and *timeout* is ``None`` (the default),
        block if necessary until an item is available. If *timeout* is a positive number,
        it blocks at most *timeout* seconds and raises the :class:`Empty` exception
        if no item was available within that time. Otherwise (*block* is false), return
        an item if one is immediately available, else raise the :class:`Empty` exception
        (*timeout* is ignored in that case).
        """
        if self.qsize():
            if self.putters:
                self._schedule_unlock()
            return self._get()
        elif not block and get_hub().greenlet is getcurrent():
            # special case to make get_nowait() runnable in the mainloop greenlet
            # there are no items in the queue; try to fix the situation by unlocking putters
            while self.putters:
                putter = self.putters.pop()
                if putter:
                    putter.switch(putter)
                    if self.qsize():
                        return self._get()
            raise Empty
        elif block:
            waiter = Waiter()
            timeout = Timeout(timeout, Empty)
            try:
                self.getters.add(waiter)
                if self.putters:
                    self._schedule_unlock()
                return waiter.wait()
            finally:
                self.getters.discard(waiter)
                timeout.cancel()
        else:
            raise Empty
Example #25
0
 def do_something ():
     my_local.foo = 'bar'
     self.assertEqual(len(init_args), 2, init_args)
     self.assertEqual(init_args[1][0], (1, 2, 3))
     self.assertEqual(init_args[1][1], getcurrent())
Example #26
0
 def ContextWrapper (self, arg, t):
     current = greenthread.getcurrent()
     if current != self.current_tasklet:
         self.SwitchTasklet(self.current_tasklet, current, t)
         t = 0.0 #the time was billed to the previous tasklet
     return f(self, arg, t)
Example #27
0
 def _setup(self):
     self.cur, self.timings, self.current_tasklet = None, {}, greenthread.getcurrent(
     )
     self.thread_id = thread.get_ident()
     self.simulate_call("profiler")
Example #28
0
 def __init__ (self, timer = None, bias = None):
     self.current_tasklet = greenthread.getcurrent()
     self.thread_id = thread.get_ident()
     self.base.__init__(self, timer, bias)
     self.sleeping = {}
Example #29
0
 def ContextWrapper(self, arg, t):
     current = greenthread.getcurrent()
     if current != self.current_tasklet:
         self.SwitchTasklet(self.current_tasklet, current, t)
         t = 0.0  #the time was billed to the previous tasklet
     return f(self, arg, t)
Example #30
0
 def __init__(self, timer=None, bias=None):
     self.current_tasklet = greenthread.getcurrent()
     self.thread_id = thread.get_ident()
     self.base.__init__(self, timer, bias)
     self.sleeping = {}
Example #31
0
 def _setup (self):
     self.cur, self.timings, self.current_tasklet = None, {}, greenthread.getcurrent()
     self.thread_id = thread.get_ident()
     self.simulate_call("profiler")
Example #32
0
def get_ident():
    """ Returns ``id()`` of current greenlet.  Useful for debugging."""
    return id(greenthread.getcurrent())
Example #33
0
 def __init__ (self, *args):
     init_args.append((args, getcurrent()))
Example #34
0
 def __init__(self, *args):
     init_args.append((args, getcurrent()))
Example #35
0
 def do_something():
     my_local.foo = 'bar'
     self.assertEqual(len(init_args), 2, init_args)
     self.assertEqual(init_args[1][0], (1, 2, 3))
     self.assertEqual(init_args[1][1], getcurrent())