Beispiel #1
0
def dont_exit_pytest():
    """ Raiden will quit on any unhandled exception.

    This allows the test suite to finish in case an exception is unhandled.
    """
    gevent.get_hub().SYSTEM_ERROR = BaseException
    gevent.get_hub().NOT_ERROR = (gevent.GreenletExit, SystemExit)
Beispiel #2
0
    def test_pool_full(self):
        self.init_server()
        with self.makefile() as long_request:
            with self.makefile() as short_request:
                self.send_request_to_fd(short_request, '/short')
                self.send_request_to_fd(long_request, '/long')

                # keep long_request in scope, otherwise the connection will be closed
                gevent.get_hub().loop.update_now()
                gevent.sleep(_DEFAULT_SOCKET_TIMEOUT / 10.0)
                self.assertPoolFull()
                self.assertPoolFull()
                # XXX Not entirely clear why this fails (timeout) on appveyor;
                # underlying socket timeout causing the long_request to close?
                self.assertPoolFull()

        # gevent.http and gevent.wsgi cannot detect socket close, so sleep a little
        # to let /short request finish
        gevent.sleep(_DEFAULT_SOCKET_TIMEOUT)
        # XXX: This tends to timeout. Which is weird, because what would have
        # been the third call to assertPoolFull() DID NOT timeout, hence why it
        # was removed.
        try:
            self.assertRequestSucceeded()
        except socket.timeout:
            greentest.reraiseFlakyTestTimeout()
Beispiel #3
0
 def wrapped(self, *args, **kwargs):
     SYSTEM_ERROR = gevent.get_hub().SYSTEM_ERROR
     gevent.get_hub().SYSTEM_ERROR = object
     try:
         return method(self, *args, **kwargs)
     finally:
         gevent.get_hub().SYSTEM_ERROR = SYSTEM_ERROR
    def test_destroy_gc(self):
        # Issue 1098: destroying the default loop
        # while using the C extension could crash
        # the interpreter when it exits

        # Create the hub greenlet. This creates one loop
        # object pointing to the default loop.
        gevent.get_hub()

        # Get a new loop object, but using the default
        # C loop
        loop = gevent.config.loop(default=True)
        self.assertTrue(loop.default)
        # Destroy it
        loop.destroy()
        # It no longer claims to be the default
        self.assertFalse(loop.default)

        # Delete it
        del loop
        # Delete the hub. This prompts garbage
        # collection of it and its loop object.
        # (making this test more repeatable; the exit
        # crash only happened when that greenlet object
        # was collected at exit time, which was most common
        # in CPython 3.5)
        from gevent._hub_local import set_hub
        set_hub(None)
def _run_resolver_in_greenlet(d, resolver):
    try:
        result = resolver()
        get_hub().loop.run_callback(d.callback, result)
    except:
        e = DeferredException()
        get_hub().loop.run_callback(d.errback, e)
    def get(self):
        if "wsgi.websocket" in request.environ:
            async = gevent.get_hub().loop.async()
            async.start(lambda: None)

            guest_asyncs.add(async)

            ws = request.environ["wsgi.websocket"]

            try:
                ws.send(self.dump_guests())

                while True:
                    gevent.get_hub().wait(async)
                    ws.send(self.dump_guests())
            except WebSocketError:
                pass
            finally:
                guest_asyncs.remove(async)

                if not ws.closed:
                    ws.close()

            return Response()
        else:
            return Response(self.dump_guests(), mimetype="application/json")
Beispiel #7
0
 def _rollback(self, conn):
     try:
         conn.rollback()
     except:
         gevent.get_hub().handle_error(conn, *sys.exc_info())
         return
     return conn
Beispiel #8
0
    def query(self, qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN,
              tcp=False, source=None, raise_on_no_answer=True,
              _hosts_rdtypes=(dns.rdatatype.A, dns.rdatatype.AAAA, dns.rdatatype.PTR)):
        # Query the resolver, using /etc/hosts

        # Behavior:
        # 1. if hosts is enabled and contains answer, return it now
        # 2. query nameservers for qname
        if qname is None:
            qname = '0.0.0.0'

        if not isinstance(qname, string_types):
            if isinstance(qname, bytes):
                qname = qname.decode("idna")

        if isinstance(qname, string_types):
            qname = dns.name.from_text(qname, None)

        if isinstance(rdtype, string_types):
            rdtype = dns.rdatatype.from_text(rdtype)

        if rdclass == dns.rdataclass.IN and rdtype in _hosts_rdtypes:
            try:
                answer = self.hosts_resolver.query(qname, rdtype, raise_on_no_answer=False)
            except Exception: # pylint: disable=broad-except
                from gevent import get_hub
                get_hub().handle_error(self, *sys.exc_info())
            else:
                if answer.rrset:
                    return answer

        return self.network_resolver.query(qname, rdtype, rdclass,
                                           tcp, source, raise_on_no_answer=raise_on_no_answer)
Beispiel #9
0
 def getline(self, filename, lineno):
     if filename not in self.cache:
         gevent.get_hub().threadpool.apply(self.update, (filename,))
     lines = self.cache.get(filename, [])
     try:
         return lines[lineno]
     except IndexError:
         return ''
Beispiel #10
0
 def wrapped(self, *args, **kwargs):
     old = gevent.get_hub().handle_error
     try:
         return method(self, *args, **kwargs)
     finally:
         gevent.get_hub().handle_error = old
     if self.peek_error()[0] is not None:
         gevent.getcurrent().throw(*self.peek_error()[1:])
Beispiel #11
0
 def _notify_links(self):
     links = self._links
     for link in links:
         try:
             link(self)
         except:
             get_hub().handle_error((link, self), *sys.exc_info())
     del self._links[:]
Beispiel #12
0
 def wrapped(self, *args, **kwargs):
     # XXX should also be able to do gevent.SYSTEM_ERROR = object
     # which is a global default to all hubs
     SYSTEM_ERROR = gevent.get_hub().SYSTEM_ERROR
     gevent.get_hub().SYSTEM_ERROR = object
     try:
         return method(self, *args, **kwargs)
     finally:
         gevent.get_hub().SYSTEM_ERROR = SYSTEM_ERROR
Beispiel #13
0
    def load(self): # pylint:disable=too-many-locals
        # Load hosts file

        # This will (re)load the data from the hosts
        # file if it has changed.

        try:
            load_time = os.stat(self.fname).st_mtime
            needs_load = load_time > self._last_load
        except (IOError, OSError):
            from gevent import get_hub
            get_hub().handle_error(self, *sys.exc_info())
            needs_load = False

        if not needs_load:
            return

        v4 = {}
        v6 = {}
        aliases = {}
        reverse = {}

        for line in self._readlines():
            parts = line.split()
            if len(parts) < 2:
                continue
            ip = parts.pop(0)
            if _is_ipv4_addr(ip):
                ipmap = v4
            elif _is_ipv6_addr(ip):
                if ip.startswith('fe80'):
                    # Do not use link-local addresses, OSX stores these here
                    continue
                ipmap = v6
            else:
                continue
            cname = parts.pop(0).lower()
            ipmap[cname] = ip
            for alias in parts:
                alias = alias.lower()
                ipmap[alias] = ip
                aliases[alias] = cname

            # XXX: This is wrong for ipv6
            if ipmap is v4:
                ptr = '.'.join(reversed(ip.split('.'))) + '.in-addr.arpa'
            else:
                ptr = ip + '.ip6.arpa.'
            if ptr not in reverse:
                reverse[ptr] = cname

        self._last_load = load_time
        self.v4 = v4
        self.v6 = v6
        self.aliases = aliases
        self.reverse = reverse
Beispiel #14
0
def call_switch_directly():
    import gevent

    def talk(msg):
        print msg
        gevent.sleep(0)
        print msg

    g = gevent.spawn(talk, 'bar')
    gevent.get_hub().switch()
 def open(self):
     super(GeventDaemonContext, self).open()
     # always reinit even when not forked when registering signals
     self._apply_monkey_patch()
     import gevent
     if self.gevent_hub is not None:
         # gevent 1.0 only
         gevent.get_hub(self.gevent_hub)
     gevent.reinit()
     self._setup_gevent_signals()
Beispiel #16
0
 def setUp(self):
     super(TestCase, self).setUp()
     # Especially if we're running in leakcheck mode, where
     # the same test gets executed repeatedly, we need to update the
     # current time. Tests don't always go through the full event loop,
     # so that doesn't always happen. test__pool.py:TestPoolYYY.test_async
     # tends to show timeouts that are too short if we don't.
     # XXX: Should some core part of the loop call this?
     gevent.get_hub().loop.update_now()
     self.close_on_teardown = []
Beispiel #17
0
def signal(signalnum, handler):
    """
    Exactly the same as :func:`signal.signal` except where
    :const:`signal.SIGCHLD` is concerned.

    .. note::

       A :const:`signal.SIGCHLD` handler installed with this function
       will only be triggered for children that are forked using
       :func:`gevent.os.fork` (:func:`gevent.os.fork_and_watch`);
       children forked before monkey patching, or otherwise by the raw
       :func:`os.fork`, will not trigger the handler installed by this
       function. (It's unlikely that a SIGCHLD handler installed with
       the builtin :func:`signal.signal` would be triggered either;
       libev typically overwrites such a handler at the C level. At
       the very least, it's full of race conditions.)

    .. note::

        Use of ``SIG_IGN`` and ``SIG_DFL`` may also have race conditions
        with libev child watchers and the :mod:`gevent.subprocess` module.

    .. versionchanged:: 1.2a1
         If ``SIG_IGN`` or ``SIG_DFL`` are used to ignore ``SIGCHLD``, a
         future use of ``gevent.subprocess`` and libev child watchers
         will once again work. However, on Python 2, use of ``os.popen``
         will fail.

    .. versionchanged:: 1.1rc2
         Allow using ``SIG_IGN`` and ``SIG_DFL`` to reset and ignore ``SIGCHLD``.
         However, this allows the possibility of a race condition if ``gevent.subprocess``
         had already been used.
    """
    if signalnum != _signal.SIGCHLD:
        return _signal_signal(signalnum, handler)

    # TODO: raise value error if not called from the main
    # greenlet, just like threads

    if handler != _signal.SIG_IGN and handler != _signal.SIG_DFL and not callable(handler):
        # exact same error message raised by the stdlib
        raise TypeError("signal handler must be signal.SIG_IGN, signal.SIG_DFL, or a callable object")

    old_handler = getsignal(signalnum)
    global _child_handler
    _child_handler = handler
    if handler == _signal.SIG_IGN or handler == _signal.SIG_DFL:
        # Allow resetting/ignoring this signal at the process level.
        # Note that this conflicts with gevent.subprocess and other users
        # of child watchers, until the next time gevent.subprocess/loop.install_sigchld()
        # is called.
        from gevent import get_hub # Are we always safe to import here?
        _signal_signal(signalnum, handler)
        get_hub().loop.reset_sigchld()
    return old_handler
    def _work(self, burst=False, logging_level="INFO"):
        """Starts the work loop.

        Pops and performs all jobs on the current list of queues.  When all
        queues are empty, block and wait for new jobs to arrive on any of the
        queues, unless `burst` mode is enabled.

        The return value indicates whether any jobs were processed.
        """
        setup_loghandlers(logging_level)
        self._install_signal_handlers()

        self.did_perform_work = False
        self.register_birth()
        self.log.info("RQ GEVENT worker (Greenlet pool size={0}) {1!r} started, version {2}".
                      format(self.gevent_pool.size, self.key, VERSION))
        self.set_state(WorkerStatus.STARTED)

        try:
            while True:
                try:
                    self.check_for_suspension(burst)

                    if self.should_run_maintenance_tasks:
                        self.clean_registries()

                    if self._stop_requested:
                        self.log.info('Stopping on request.')
                        break

                    timeout = None if burst else max(1, self.default_worker_ttl - 60)

                    result = self.dequeue_job_and_maintain_ttl(timeout)
                    if result is None and burst:
                        self.log.info("RQ worker {0!r} done, quitting".format(self.key))

                        try:
                            # Make sure dependented jobs are enqueued.
                            get_hub().switch()
                        except LoopExit:
                            pass
                        result = self.dequeue_job_and_maintain_ttl(timeout)

                    if result is None:
                        break
                except StopRequested:
                    break

                job, queue = result
                self.execute_job(job, queue)

        finally:
            if not self.is_horse:
                self.register_death()
        return self.did_perform_work
Beispiel #19
0
 def _store_error(self, where, type, value, tb):
     del tb
     if self._error != self._none:
         gevent.get_hub().parent.throw(type, value)
     else:
         self._error = (where, type, value)
         if sys.version_info[0] >= 3:
             try:
                 value.__traceback__ = None
             except AttributeError:
                 pass
 def start(self):
    # Start grabbing SIGCHLD within libev event loop.
    gevent.get_hub().loop.install_sigchld()
    # Run new process (based on `fork()` on POSIX-compliant systems).
    super(_GProcess, self).start()
    # The occurrence of SIGCHLD is recorded asynchronously in libev.
    # This guarantees proper behavior even if the child watcher is
    # started after the child exits. Start child watcher now.
    self._sigchld_watcher = gevent.get_hub().loop.child(self.pid)
    self._returnevent = gevent.event.Event()
    self._sigchld_watcher.start(self._on_sigchld, self._sigchld_watcher)
Beispiel #21
0
 def _client_exception_handler(self, greenlet):
     self._running = False
     try:
         greenlet.get()
     except MatrixError as ex:
         gevent.get_hub().handle_system_error(
             TransportError,
             TransportError(
                 f'Unexpected error while communicating with Matrix homeserver: {ex}',
             ),
         )
Beispiel #22
0
    def setup(self):
        if not batchy_gevent:
            raise SkipTest()

        use_gevent_local()

        # Quiet gevent's internal exception printing.
        self.old_print_exception = gevent.get_hub().print_exception
        gevent.get_hub().print_exception = lambda context, type, value, tb: None

        global CALL_COUNT
        CALL_COUNT = 0
Beispiel #23
0
    def _notify_waiter(self, service_name, action):
        """
        通知指定service_name下的所有waiter
        :param service_name:
        :param action: 当前变更动作,定义与constatn.SERVICE_ACTION
        :type service_name: str
        :type action: str
        :return:
        """

        for waiter in self._waiter_dict[service_name]:
            gevent.get_hub().loop.run_callback(lambda: waiter.switch(action))
Beispiel #24
0
 def _test_wait_read_invalid_switch(self, sleep):
     sock1, sock2 = socket.socketpair()
     try:
         p = gevent.spawn(util.wrap_errors(AssertionError, socket.wait_read), sock1.fileno())
         gevent.get_hub().loop.run_callback(switch_None, p)
         if sleep is not None:
             gevent.sleep(sleep)
         result = p.get()
         assert isinstance(result, AssertionError), result
         assert 'Invalid switch' in str(result), repr(str(result))
     finally:
         sock1.close()
         sock2.close()
Beispiel #25
0
    def test(self):
        # hub.join() guarantees that loop has exited cleanly
        res = gevent.get_hub().join()
        self.assertTrue(res)

        res = gevent.get_hub().join()
        self.assertTrue(res)

        # but it is still possible to use gevent afterwards
        gevent.sleep(0.01)

        res = gevent.get_hub().join()
        self.assertTrue(res)
def _child(target, args, kwargs):
   """Wrapper function that runs in child process. Resets gevent/libev state
   and executes user-given function.
   """
   _tryGevent()
   _reset_signal_handlers()
   gevent.reinit()
   hub = gevent.get_hub()
   del hub.threadpool
   hub._threadpool = None
   hub.destroy(destroy_loop=True)
   h = gevent.get_hub(default=True)
   assert h.loop.default, 'Could not create libev default event loop.'
   target(*args, **kwargs)
Beispiel #27
0
 def __init__(self):
     # {int -> flags}
     # We can't keep watcher objects in here because people commonly
     # just drop the poll object when they're done, without calling
     # unregister(). dnspython does this.
     self.fds = {}
     self.loop = get_hub().loop
Beispiel #28
0
def run(ctx, dev):
    """Start the client"""
    # create app
    app = EthApp(ctx.obj['config'])

    if dev:
        gevent.get_hub().SYSTEM_ERROR = BaseException
        try:
            ctx.obj['config']['client_version'] += '/' + os.getlogin()
        except:
            log.warn("can't get and add login name to client_version")
            pass

    # register services
    for service in services:
        assert issubclass(service, BaseService)
        if service.name not in app.config['deactivated_services']:
            assert service.name not in app.services
            service.register_with_app(app)
            assert hasattr(app.services, service.name)

    # start app
    app.start()

    # wait for interupt
    evt = Event()
    gevent.signal(signal.SIGQUIT, evt.set)
    gevent.signal(signal.SIGTERM, evt.set)
    gevent.signal(signal.SIGINT, evt.set)
    evt.wait()

    # finally stop
    app.stop()
Beispiel #29
0
def _start_app(config, account):
    # create app
    app = HPCApp(config)

    # development mode
    if False:
        gevent.get_hub().SYSTEM_ERROR = BaseException

    # dump config
    pyethapp_app.dump_config(config)
    # init accounts first, as we need (and set by copy) the coinbase early FIXME
    if AccountsService in services:
        AccountsService.register_with_app(app)
    # add account
    app.services.accounts.add_account(account, store=False)

    assert app.services.accounts.coinbase in config['hdc']['validators']
    # register services
    for service in services:
        assert issubclass(service, BaseService)
        if service.name not in app.config['deactivated_services'] + [AccountsService.name]:
            assert service.name not in app.services
            service.register_with_app(app)
            assert hasattr(app.services, service.name)

    # start app
    log.info('starting')
    app.start()
    if config['post_app_start_callback'] is not None:
        config['post_app_start_callback'](app)
    return app
Beispiel #30
0
 def test2(self):
     timer = gevent.get_hub().loop.timer(0)
     timer.start(hello2)
     gevent.sleep(0.1)
     assert sys.exc_info() == (None, None, None), sys.exc_info()
     if six.PY3:
         expected_error.__traceback__ = None
Beispiel #31
0
 def __init__(self, delay, fn, *args, **kwargs):
     self._timer = gevent.get_hub().loop.timer(delay, 0)
     self._fn = fn
     self._delay = delay
     self._args = args
     self._kwargs = kwargs
Beispiel #32
0
 def gevent_error_handler(context, exc_info):
     e = exc_info[1]
     if isinstance(e, ssl.SSLError) and e.reason == 'HTTP_REQUEST':
         return
     gevent.get_hub().handle_system_error(exc_info[0], exc_info[1])
Beispiel #33
0
import gevent
import greenlet


def callback(event, args):
    print
    event, args[0], '===:>>>>', args[1]


def foo():
    print('Running in foo')
    gevent.sleep(0)
    print('Explicit context switch to foo again')


def bar():
    print('Explicit context to bar')
    gevent.sleep(0)
    print('Implicit context switch back to bar')


print('main greenlet info: ', greenlet.greenlet.getcurrent())
print('hub info', gevent.get_hub())
oldtrace = greenlet.settrace(callback)

gevent.joinall([
    gevent.spawn(foo),
    gevent.spawn(bar),
])
greenlet.settrace(oldtrace)
Beispiel #34
0
from ssh2.sftp import LIBSSH2_FXF_READ, LIBSSH2_FXF_CREAT, LIBSSH2_FXF_WRITE, \
    LIBSSH2_FXF_TRUNC, LIBSSH2_SFTP_S_IRUSR, LIBSSH2_SFTP_S_IRGRP, \
    LIBSSH2_SFTP_S_IWUSR, LIBSSH2_SFTP_S_IXUSR, LIBSSH2_SFTP_S_IROTH, \
    LIBSSH2_SFTP_S_IXGRP, LIBSSH2_SFTP_S_IXOTH

from ...exceptions import UnknownHostException, AuthenticationException, \
     ConnectionErrorException, SessionError, SFTPError, SFTPIOError, Timeout, \
     SCPError
from ...constants import DEFAULT_RETRIES, RETRY_DELAY
from ...native._ssh2 import wait_select, eagain_write, _read_output
from .common import _validate_pkey_path

Hub.NOT_ERROR = (Exception, )
host_logger = logging.getLogger('pssh.host_logger')
logger = logging.getLogger(__name__)
THREAD_POOL = get_hub().threadpool


class SSHClient(object):
    """ssh2-python (libssh2) based non-blocking SSH client."""

    IDENTITIES = (
        os.path.expanduser('~/.ssh/id_rsa'),
        os.path.expanduser('~/.ssh/id_dsa'),
        os.path.expanduser('~/.ssh/identity'),
        os.path.expanduser('~/.ssh/id_ecdsa'),
    )

    def __init__(self,
                 host,
                 user=None,
Beispiel #35
0
def _gevent_pool_spawn(func, args=()):
    import gevent
    return gevent.get_hub().threadpool.spawn(func, *args)
import signal
import subprocess

def _waitpid(pid):
    try:
        _, stat = os.waitpid(pid, 0)
    except OSError:
        # Interrupted system call
        _, stat = os.waitpid(pid, 0)
    assert stat == 0, stat

if hasattr(signal, 'SIGCHLD'):
    # Do what subprocess does and make sure we have the watcher
    # in the parent
    get_hub().loop.install_sigchld()


    pid = os.fork()

    if pid: # parent
        _waitpid(pid)
    else:
        # Child resets.
        signal.signal(signal.SIGCHLD, signal.SIG_DFL)

        # Go through subprocess because we expect it to automatically
        # set up the waiting for us.
        popen = subprocess.Popen([sys.executable, '-c', 'import sys'],
                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        popen.stderr.read()
Beispiel #37
0
def reset_gevent():
    gevent.reinit()
    gevent.get_hub().destroy(destroy_loop=True)  # Forget previous callbacks.
    gevent.get_hub(default=True)  # Here is necessary.
Beispiel #38
0
from gevent import monkey

import os
import re
import gevent.testing as greentest
import unittest
import socket
from time import time
import traceback
import gevent.socket as gevent_socket
from gevent.testing.util import log
from gevent.testing import six
from gevent.testing.six import xrange
from gevent.testing import flaky

resolver = gevent.get_hub().resolver
log('Resolver: %s', resolver)

if getattr(resolver, 'pool', None) is not None:
    resolver.pool.size = 1

from gevent.testing.sysinfo import RESOLVER_NOT_SYSTEM
from gevent.testing.sysinfo import RESOLVER_DNSPYTHON
from gevent.testing.sysinfo import PY2
import gevent.testing.timing

assert gevent_socket.gaierror is socket.gaierror
assert gevent_socket.error is socket.error

DEBUG = os.getenv('GEVENT_DEBUG', '') == 'trace'
Beispiel #39
0
from gevent.queue import Queue
from mock import patch
from testil import Config, tempdir

from corehq.apps.tzmigration.timezonemigration import FormJsonDiff
from corehq.form_processor.parsers.ledgers.helpers import UniqueLedgerReference

from .. import casediff as mod
from ..statedb import StateDB, init_state_db

log = logging.getLogger(__name__)


@patch.object(mod.CaseDiffQueue, "BATCH_SIZE", 2)
@patch.object(mod.BatchProcessor, "MAX_RETRIES", 0)
@patch.object(gevent.get_hub(), "SYSTEM_ERROR", BaseException)
class TestCaseDiffQueue(SimpleTestCase):
    def setUp(self):
        super(TestCaseDiffQueue, self).setUp()
        self.patches = [
            patch.object(mod, "diff_cases", self.diff_cases),
            patch(
                "corehq.form_processor.backends.couch.dbaccessors.CaseAccessorCouch.get_cases",
                self.get_cases,
            ),
            patch.object(mod, "get_stock_forms_by_case_id",
                         self.get_stock_forms),
        ]

        for patcher in self.patches:
            patcher.start()
Beispiel #40
0
# -*- coding: utf-8 -*-
# pylint: disable=wrong-import-position,redefined-outer-name,unused-wildcard-import,wildcard-import
import gevent
from gevent import monkey
monkey.patch_all()

import pytest
from ethereum import slogging
from ethereum.keys import PBKDF2_CONSTANTS
from ethereum import processblock
from ethereum import tester

from raiden.network.rpc.client import GAS_LIMIT
from raiden.tests.fixtures import *  # noqa: F401,F403

gevent.get_hub().SYSTEM_ERROR = BaseException
PBKDF2_CONSTANTS['c'] = 100

CATCH_LOG_HANDLER_NAME = 'catch_log_handler'


def pytest_addoption(parser):
    parser.addoption(
        '--blockchain-type',
        choices=['geth', 'tester'],
        default='geth',
    )

    parser.addoption(
        '--blockchain-cache',
        action='store_true',
Beispiel #41
0
 def gevent_error_handler(context, exc_info):
     e = exc_info[1]
     # recover if HTTP request is done to a HTTPS endpoint
     if isinstance(e, ssl.SSLError) and e.reason == 'HTTP_REQUEST':
         return
     gevent.get_hub().handle_system_error(exc_info[0], exc_info[1])
Beispiel #42
0
 def expect_one_error(self):
     self.assertEqual(self._error, self._none)
     gevent.get_hub().handle_error = self._store_error
Beispiel #43
0
    importlib.reload(gevent)

def handleGreenletError(context, type, value, tb):
    if context.__class__ is tuple and context[0].__class__.__name__ == "ThreadPool":
        # Exceptions in ThreadPool will be handled in the main Thread
        return None

    if isinstance(value, str):
        # Cython can raise errors where the value is a plain string
        # e.g., AttributeError, "_semaphore.Semaphore has no attr", <traceback>
        value = type(value)

    if not issubclass(type, gevent.get_hub().NOT_ERROR):
        sys.excepthook(type, value, tb)

gevent.get_hub().handle_error = handleGreenletError

try:
    signal.signal(signal.SIGTERM, lambda signum, stack_frame: shutdown("SIGTERM"))
except Exception as err:
    logging.debug("Error setting up SIGTERM watcher: %s" % err)


if __name__ == "__main__":
    import time
    from gevent import monkey
    monkey.patch_all(thread=False, ssl=False)
    from . import Debug

    def sleeper(num):
        print("started", num)
Beispiel #44
0
                    # exit_subprocess() will skip the enclosing with block.
                    with closing(done_w):
                        try:
                            run_test()
                        except:
                            pickle.dump(traceback.format_exc(), done_w)
                        else:
                            pickle.dump(None, done_w)
                except:
                    print('Uh oh, raised from pickle.')
                    traceback.print_exc()
                finally:
                    os._exit(0)

            done_w.close()
            # Block for up to MAX_DURATION seconds for the test to finish.
            r, w, x = select.select([done_r], [], [], MAX_DURATION)
            if done_r in r:
                tb = pickle.load(done_r)
                assert not tb, tb
            else:
                os.kill(child, 9)
                assert False, 'Test deadlocked after %d seconds.' % MAX_DURATION


if __name__ == "__main__":
    print gevent.get_hub()
    while True:
        test_main()
        sys.stderr.write('.')
Beispiel #45
0
 def __init__(self, socket):
     self.fileno = socket.fileno()
     self.family = socket.family
     self.proto = socket.proto
     self._socket = socket
     self._hub_id = id(gevent.get_hub())
Beispiel #46
0
 def setUp(self):
     super(TestCase, self).setUp()
     if not self.verbose_dns:
         # Silence the default reporting of errors from the ThreadPool,
         # we handle those here.
         gevent.get_hub().exception_stream = None
Beispiel #47
0
from __future__ import print_function
import gevent
# Loop of initial Hub is default loop.
hub = gevent.get_hub()
assert hub.loop.default, hub

# Destroy hub. Does not destroy default loop if not explicitly told to.
hub.destroy()
hub = gevent.get_hub()
assert hub.loop.default, hub

saved_loop = hub.loop
# Destroy hub including default loop.
hub.destroy(destroy_loop=True)
assert saved_loop.fileno() is None, saved_loop
print(hub, saved_loop)

# Create new hub and explicitly request creation of a new default loop.
hub = gevent.get_hub(default=True)
assert hub.loop.default, hub

# Destroy hub including default loop.
hub.destroy(destroy_loop=True)

# Create new non-default loop in new hub.
hub = gevent.get_hub()
assert not hub.loop.default, hub
hub.destroy()
Beispiel #48
0
from __future__ import print_function
import gevent.monkey; gevent.monkey.patch_all()
import gevent
import os

hub = gevent.get_hub()
pid = os.getpid()
newpid = None


def on_fork():
    global newpid
    newpid = os.getpid()

fork_watcher = hub.loop.fork(ref=False)
fork_watcher.start(on_fork)

# fork watchers weren't firing in multi-threading processes.


def run():
    # libev only calls fork callbacks at the beginning of
    # the loop; we use callbacks extensively so it takes *two*
    # calls to sleep (with a timer) to actually get wrapped
    # around to the beginning of the loop.
    gevent.sleep(0.01)
    gevent.sleep(0.01)
    q.put(newpid)


import multiprocessing
Beispiel #49
0
address = ('127.0.0.10', 53)
listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
    listener.bind(address)
except socket.error as ex:
    if 'permission denied' in str(ex).lower():
        sys.stderr.write(
            'This test binds on port 53 and thus must be run as root.\n')
        sys.exit(0)
    raise


def reader():
    while True:
        print listener.recvfrom(10000)


gevent.spawn(reader)

r = gevent.get_hub().resolver = Resolver(servers=['127.0.0.10'],
                                         timeout=0.001,
                                         tries=1)
try:
    result = r.gethostbyname('www.google.com')
except socket.gaierror as ex:
    if 'ARES_ETIMEOUT' not in str(ex):
        raise
else:
    raise AssertionError('Expected timeout, got %r' % (result, ))
Beispiel #50
0
 def expect_one_error(self):
     assert self._error == self._none, self._error
     self._old_handle_error = gevent.get_hub().handle_error
     gevent.get_hub().handle_error = self._store_error
Beispiel #51
0
def dont_exit_pytest():
    """Raiden will quit on any unhandled exception.

    This allows the test suite to finish in case an exception is unhandled.
    """
    gevent.get_hub().NOT_ERROR = (gevent.GreenletExit, SystemExit)
Beispiel #52
0
 def _store_error(self, where, type, value, tb):
     del tb
     if self._error != self._none:
         gevent.get_hub().parent.throw(type, value)
     else:
         self._error = (where, type, value)
Beispiel #53
0
    yield
    elapsed = time() - start
    assert expected - fuzzy <= elapsed <= expected + fuzzy, 'Expected: %r; elapsed: %r' % (expected, elapsed)


def no_time(fuzzy=0.001):
    return expected_time(0, fuzzy=fuzzy)


for _a in xrange(2):

    # exiting because the spawned greenlet finished execution (spawn (=callback) variant)
    for _ in xrange(2):
        x = gevent.spawn(lambda: 5)
        with no_time(SMALL):
            result = gevent.get_hub().join(timeout=10)
        assert result is True, repr(result)
        assert x.dead, x
        assert x.value == 5, x

    # exiting because the spawned greenlet finished execution (spawn_later (=timer) variant)
    for _ in xrange(2):
        x = gevent.spawn_later(SMALL, lambda: 5)
        with expected_time(SMALL):
            result = gevent.get_hub().join(timeout=10)
        assert result is True, repr(result)
        assert x.dead, x

    # exiting because of timeout (the spawned greenlet still runs)
    for _ in xrange(2):
        x = gevent.spawn_later(10, lambda: 5)
Beispiel #54
0
 def _schedule(self):
     timer = gevent.get_hub().loop.timer(self.wait)
     timer.start(self.__call__)
Beispiel #55
0
    for _ in xrange(2):
        x = gevent.spawn_later(10, lambda: 5)
        event = Event()
        event_set = gevent.spawn_later(SMALL, event.set)
        with expected_time(SMALL):
            result = gevent.wait([event])
        assert result == [event], repr(result)
        assert not x.dead, x
        assert event_set.dead
        assert event.is_set()
        x.kill()
        with no_time():
            result = gevent.wait()
        assert result is True

    # checking "ref=False" argument
    for _ in xrange(2):
        gevent.get_hub().loop.timer(10, ref=False).start(lambda: None)
        with no_time():
            result = gevent.wait()
        assert result is True

    # checking "ref=False" attribute
    for _d in xrange(2):
        w = gevent.get_hub().loop.timer(10)
        w.start(lambda: None)
        w.ref = False
        with no_time():
            result = gevent.wait()
        assert result is True
Beispiel #56
0
def main(
    ctx,
    scenario_file,
    keystore_file,
    password,
    chains,
    data_path,
    auth,
    mailgun_api_key,
):
    gevent.get_hub().exception_stream = DummyStream()

    is_subcommand = ctx.invoked_subcommand is not None
    if not is_subcommand and scenario_file is None:
        ctx.fail('No scenario definition file provided')

    if is_subcommand:
        log_file_name = (
            f'scenario-player-{ctx.invoked_subcommand}_{datetime.now():%Y-%m-%dT%H:%M:%S}.log'
        )
    else:
        scenario_basename = basename(scenario_file.name)
        log_file_name = (
            f'scenario-player_{scenario_basename}_{datetime.now():%Y-%m-%dT%H:%M:%S}.log'
        )
    click.secho(f'Writing log to {log_file_name}', fg='yellow')
    configure_logging(
        {
            '': 'INFO',
            'raiden': 'DEBUG',
            'scenario_player': 'DEBUG'
        },
        debug_log_file_name=log_file_name,
        _first_party_packages=_FIRST_PARTY_PACKAGES
        | frozenset(['scenario_player']),
        _debug_log_file_additional_level_filters={'scenario_player': 'DEBUG'},
    )

    log_buffer = None
    if sys.stdout.isatty() and not is_subcommand:
        log_buffer = UrwidLogWalker([])
        for handler in logging.getLogger('').handlers:
            if isinstance(handler, logging.StreamHandler):
                handler.terminator = ConcatenableNone()
                handler.formatter = NonStringifyingProcessorFormatter(
                    UrwidLogRenderer(),
                    foreign_pre_chain=LOGGING_PROCESSORS,
                )
                handler.stream = log_buffer
                break

    chain_rpc_urls = defaultdict(list)
    for chain_name, chain_rpc_url in chains:
        chain_rpc_urls[chain_name].append(chain_rpc_url)

    with open(keystore_file, 'r') as keystore:
        account = Account(json.load(keystore), password, keystore_file)
        log.info("Using account", account=to_checksum_address(account.address))

    if is_subcommand:
        ctx.obj = dict(
            account=account,
            chain_rpc_urls=chain_rpc_urls,
            data_path=data_path,
        )
        return

    # Collect tasks
    collect_tasks(tasks)

    runner = ScenarioRunner(account, chain_rpc_urls, auth, Path(data_path),
                            scenario_file)
    ui = ScenarioUI(runner, log_buffer, log_file_name)
    ui_greenlet = ui.run()
    success = False
    try:
        try:
            runner.run_scenario()
            success = True
            log.info('Run finished', result='success')
            send_notification_mail(
                runner.notification_email,
                f'Scenario successful {scenario_file.name}',
                'Success',
                mailgun_api_key,
            )
        except ScenarioAssertionError as ex:
            log.error('Run finished', result='assertion errors')
            send_notification_mail(
                runner.notification_email,
                f'Assertion mismatch in {scenario_file.name}',
                str(ex),
                mailgun_api_key,
            )
        except ScenarioError:
            log.exception('Run finished', result='scenario error')
            send_notification_mail(
                runner.notification_email,
                f'Invalid scenario {scenario_file.name}',
                traceback.format_exc(),
                mailgun_api_key,
            )
    except Exception:
        log.exception('Exception while running scenario')
        send_notification_mail(
            runner.notification_email,
            f'Error running scenario {scenario_file.name}',
            traceback.format_exc(),
            mailgun_api_key,
        )
    finally:
        try:
            if sys.stdout.isatty():
                ui.set_success(success)
                log.warning('Press q to exit')
                while not ui_greenlet.dead:
                    gevent.sleep(1)
        finally:
            if runner.is_managed:
                runner.node_controller.stop()
            if not ui_greenlet.dead:
                ui_greenlet.kill(ExitMainLoop)
                ui_greenlet.join()
Beispiel #57
0
def _child(target, args, kwargs):
    """Wrapper function that runs in child process. Resets gevent/libev state
    and executes user-given function.

    After fork on POSIX-compliant systems, gevent's state is inherited by the
    child which may lead to undesired behavior, such as greenlets running in
    both, the parent and the child. Therefore, if not on Windows, gevent's and
    libev's state is reset before running the user-given function.
    """
    log.debug("_child start. target: `%s`", target)
    childhandles = list(_filter_handles(chain(args, kwargs.values())))
    if not WINDOWS:
        # Restore default signal handlers (SIG_DFL). Orphaned libev signal
        # watchers may not become properly deactivated otherwise. Note: here, we
        # could even reset sigprocmask (Python 2.x does not have API for it, but
        # it could be done via ctypes).
        _reset_signal_handlers()
        # `gevent.reinit` calls `libev.ev_loop_fork()`, which reinitialises
        # the kernel state for backends that have one. Must be called in the
        # child before using further libev API.
        gevent.reinit()
        log.debug("Delete current hub's threadpool.")
        hub = gevent.get_hub()
        # Delete threadpool before hub destruction, otherwise `hub.destroy()`
        # might block forever upon `ThreadPool.kill()` as of gevent 1.0rc2.
        del hub.threadpool
        hub._threadpool = None
        # Destroy default event loop via `libev.ev_loop_destroy()` and delete
        # hub. This orphans all registered events and greenlets that have been
        # duplicated from the parent via fork().
        log.debug("Destroy hub and default loop.")
        hub.destroy(destroy_loop=True)
        # Create a new hub and a new default event loop via
        # `libev.gevent_ev_default_loop`.
        h = gevent.get_hub(default=True)
        log.debug("Created new hub and default event loop.")
        assert h.loop.default, 'Could not create libev default event loop.'
        # On Unix, file descriptors are inherited by default. Also, the global
        # `_all_handles` is inherited from the parent. Close dispensable gipc-
        # related file descriptors in child.
        for h in _all_handles[:]:
            if h not in childhandles:
                log.debug("Invalidate %s in child.", h)
                h._set_legit_process()
                # At duplication time the handle might have been locked. Unlock.
                h._lock.counter = 1
                h.close()
    else:
        # On Windows, the state of module globals is not transferred to
        # children. Set `_all_handles`.
        _set_all_handles(childhandles)
    # `_all_handles` now must contain only those handles that have been
    # transferred to the child on purpose.
    for h in _all_handles:
        assert h in childhandles
    # Register transferred handles for current process.
    for h in childhandles:
        h._set_legit_process()
        if WINDOWS:
            h._winapi_childhandle_after_createprocess_child()
        log.debug("Handle `%s` is now valid in child.", h)
    # Invoke user-given function.
    target(*args, **kwargs)
    # Close file descriptors before exiting process. Usually needless (OS
    # should take care of this), but being expressive about this is clean.
    for h in childhandles:
        try:
            # The user might already have closed it.
            h.close()
        except GIPCClosed:
            pass
Beispiel #58
0
#!/usr/bin/env python
from __future__ import print_function
import gevent
gevent.get_hub(
    'select')  # this is just to make sure we don't pass any fds to children
from gevent import monkey
monkey.patch_all()
import six
import sys
import os
import glob
import traceback
from time import time

from gevent.pool import Pool
import util

TIMEOUT = 180
NWORKERS = int(os.environ.get('NWORKERS') or 8)
pool = None


def spawn(*args, **kwargs):
    g = pool.spawn(*args, **kwargs)
    g.link_exception(lambda *args: sys.exit(
        'Internal error in testrunner.py: %s %s' % (g, g.exception)))
    return g


def run_many(tests, expected=None, failfast=False):
    global NWORKERS, pool
Beispiel #59
0
def main():
    # config
    import yaml
    import io
    import sys
    import signal
    import gevent
    from .peermanager import PeerManager
    from .discovery import NodeDiscovery
    from devp2p import slogging
    log = slogging.get_logger('app')

    # read config
    sample_config = b"""
discovery:
    bootstrap_nodes:
        # local bootstrap
        - enode://6ed2fecb28ff17dec8647f08aa4368b57790000e0e9b33a7b91f32c41b6ca9ba21600e9a8c44248ce63a71544388c6745fa291f88f8b81e109ba3da11f7b41b9@127.0.0.1:30303
        # go_bootstrap
        #- enode://6cdd090303f394a1cac34ecc9f7cda18127eafa2a3a06de39f6d920b0e583e062a7362097c7c65ee490a758b442acd5c80c6fce4b148c6a391e946b45131365b@54.169.166.226:30303
        # cpp_bootstrap
        #- enode://4a44599974518ea5b0f14c31c4463692ac0329cb84851f3435e6d1b18ee4eae4aa495f846a0fa1219bd58035671881d44423876e57db2abd57254d0197da0ebe@5.1.83.226:30303

p2p:
    num_peers: 10

    listen_host: 0.0.0.0
    listen_port: 30303
node:
    privkey_hex: 65462b0520ef7d3df61b9992ed3bea0c56ead753be7c8b3614e0ce01e4cac41b
    """
    if len(sys.argv) == 1:
        config = yaml.load(io.BytesIO(sample_config))
        pubkey = crypto.privtopub(decode_hex(config['node']['privkey_hex']))
        config['node']['id'] = crypto.sha3(pubkey)
    else:
        fn = sys.argv[1]
        log.info('loading config from', fn=fn)
        config = yaml.load(open(fn))

    # stop on every unhandled exception!
    gevent.get_hub(
    ).SYSTEM_ERROR = BaseException  # (KeyboardInterrupt, SystemExit, SystemError)

    print(config)
    # create app
    app = BaseApp(config)

    # register services
    NodeDiscovery.register_with_app(app)
    PeerManager.register_with_app(app)

    # start app
    app.start()

    # wait for interupt
    evt = gevent.event.Event()
    # gevent.signal(signal.SIGQUIT, gevent.kill) ## killall pattern
    gevent.signal(signal.SIGQUIT, evt.set)
    gevent.signal(signal.SIGTERM, evt.set)
    gevent.signal(signal.SIGINT, evt.set)
    evt.wait()

    # finally stop
    app.stop()
Beispiel #60
0
# pylint: disable=wrong-import-position,redefined-outer-name,unused-wildcard-import,wildcard-import
import re

import gevent
import py
import sys
from gevent import monkey
monkey.patch_all()

import pytest

from raiden.exceptions import RaidenShuttingDown
from raiden.tests.fixtures.variables import *  # noqa: F401,F403
from raiden.log_config import configure_logging

gevent.get_hub().SYSTEM_ERROR = BaseException
gevent.get_hub().NOT_ERROR = (gevent.GreenletExit, SystemExit,
                              RaidenShuttingDown)

CATCH_LOG_HANDLER_NAME = 'catch_log_handler'


def pytest_addoption(parser):
    parser.addoption(
        '--blockchain-type',
        choices=['geth'],
        default='geth',
    )

    parser.addoption(
        '--initial-port',