def dont_exit_pytest():
""" Raiden will quit on any unhandled exception.
This allows the test suite to finish in case an exception is unhandled.
"""
gevent.get_hub().SYSTEM_ERROR = BaseException
gevent.get_hub().NOT_ERROR = (gevent.GreenletExit, SystemExit)
def test_pool_full(self):
self.init_server()
with self.makefile() as long_request:
with self.makefile() as short_request:
self.send_request_to_fd(short_request, '/short')
self.send_request_to_fd(long_request, '/long')
# keep long_request in scope, otherwise the connection will be closed
gevent.get_hub().loop.update_now()
gevent.sleep(_DEFAULT_SOCKET_TIMEOUT / 10.0)
self.assertPoolFull()
self.assertPoolFull()
# XXX Not entirely clear why this fails (timeout) on appveyor;
# underlying socket timeout causing the long_request to close?
self.assertPoolFull()
# gevent.http and gevent.wsgi cannot detect socket close, so sleep a little
# to let /short request finish
gevent.sleep(_DEFAULT_SOCKET_TIMEOUT)
# XXX: This tends to timeout. Which is weird, because what would have
# been the third call to assertPoolFull() DID NOT timeout, hence why it
# was removed.
try:
self.assertRequestSucceeded()
except socket.timeout:
greentest.reraiseFlakyTestTimeout()
def wrapped(self, *args, **kwargs):
SYSTEM_ERROR = gevent.get_hub().SYSTEM_ERROR
gevent.get_hub().SYSTEM_ERROR = object
try:
return method(self, *args, **kwargs)
finally:
gevent.get_hub().SYSTEM_ERROR = SYSTEM_ERROR
def test_destroy_gc(self):
# Issue 1098: destroying the default loop
# while using the C extension could crash
# the interpreter when it exits
# Create the hub greenlet. This creates one loop
# object pointing to the default loop.
gevent.get_hub()
# Get a new loop object, but using the default
# C loop
loop = gevent.config.loop(default=True)
self.assertTrue(loop.default)
# Destroy it
loop.destroy()
# It no longer claims to be the default
self.assertFalse(loop.default)
# Delete it
del loop
# Delete the hub. This prompts garbage
# collection of it and its loop object.
# (making this test more repeatable; the exit
# crash only happened when that greenlet object
# was collected at exit time, which was most common
# in CPython 3.5)
from gevent._hub_local import set_hub
set_hub(None)
def _run_resolver_in_greenlet(d, resolver):
try:
result = resolver()
get_hub().loop.run_callback(d.callback, result)
except:
e = DeferredException()
get_hub().loop.run_callback(d.errback, e)
def get(self):
if "wsgi.websocket" in request.environ:
async = gevent.get_hub().loop.async()
async.start(lambda: None)
guest_asyncs.add(async)
ws = request.environ["wsgi.websocket"]
try:
ws.send(self.dump_guests())
while True:
gevent.get_hub().wait(async)
ws.send(self.dump_guests())
except WebSocketError:
pass
finally:
guest_asyncs.remove(async)
if not ws.closed:
ws.close()
return Response()
else:
return Response(self.dump_guests(), mimetype="application/json")
def _rollback(self, conn):
try:
conn.rollback()
except:
gevent.get_hub().handle_error(conn, *sys.exc_info())
return
return conn
def query(self, qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN,
tcp=False, source=None, raise_on_no_answer=True,
_hosts_rdtypes=(dns.rdatatype.A, dns.rdatatype.AAAA, dns.rdatatype.PTR)):
# Query the resolver, using /etc/hosts
# Behavior:
# 1. if hosts is enabled and contains answer, return it now
# 2. query nameservers for qname
if qname is None:
qname = '0.0.0.0'
if not isinstance(qname, string_types):
if isinstance(qname, bytes):
qname = qname.decode("idna")
if isinstance(qname, string_types):
qname = dns.name.from_text(qname, None)
if isinstance(rdtype, string_types):
rdtype = dns.rdatatype.from_text(rdtype)
if rdclass == dns.rdataclass.IN and rdtype in _hosts_rdtypes:
try:
answer = self.hosts_resolver.query(qname, rdtype, raise_on_no_answer=False)
except Exception: # pylint: disable=broad-except
from gevent import get_hub
get_hub().handle_error(self, *sys.exc_info())
else:
if answer.rrset:
return answer
return self.network_resolver.query(qname, rdtype, rdclass,
tcp, source, raise_on_no_answer=raise_on_no_answer)
def getline(self, filename, lineno):
if filename not in self.cache:
gevent.get_hub().threadpool.apply(self.update, (filename,))
lines = self.cache.get(filename, [])
try:
return lines[lineno]
except IndexError:
return ''
def wrapped(self, *args, **kwargs):
old = gevent.get_hub().handle_error
try:
return method(self, *args, **kwargs)
finally:
gevent.get_hub().handle_error = old
if self.peek_error()[0] is not None:
gevent.getcurrent().throw(*self.peek_error()[1:])
def _notify_links(self):
links = self._links
for link in links:
try:
link(self)
except:
get_hub().handle_error((link, self), *sys.exc_info())
del self._links[:]
def wrapped(self, *args, **kwargs):
# XXX should also be able to do gevent.SYSTEM_ERROR = object
# which is a global default to all hubs
SYSTEM_ERROR = gevent.get_hub().SYSTEM_ERROR
gevent.get_hub().SYSTEM_ERROR = object
try:
return method(self, *args, **kwargs)
finally:
gevent.get_hub().SYSTEM_ERROR = SYSTEM_ERROR
def load(self): # pylint:disable=too-many-locals
# Load hosts file
# This will (re)load the data from the hosts
# file if it has changed.
try:
load_time = os.stat(self.fname).st_mtime
needs_load = load_time > self._last_load
except (IOError, OSError):
from gevent import get_hub
get_hub().handle_error(self, *sys.exc_info())
needs_load = False
if not needs_load:
return
v4 = {}
v6 = {}
aliases = {}
reverse = {}
for line in self._readlines():
parts = line.split()
if len(parts) < 2:
continue
ip = parts.pop(0)
if _is_ipv4_addr(ip):
ipmap = v4
elif _is_ipv6_addr(ip):
if ip.startswith('fe80'):
# Do not use link-local addresses, OSX stores these here
continue
ipmap = v6
else:
continue
cname = parts.pop(0).lower()
ipmap[cname] = ip
for alias in parts:
alias = alias.lower()
ipmap[alias] = ip
aliases[alias] = cname
# XXX: This is wrong for ipv6
if ipmap is v4:
ptr = '.'.join(reversed(ip.split('.'))) + '.in-addr.arpa'
else:
ptr = ip + '.ip6.arpa.'
if ptr not in reverse:
reverse[ptr] = cname
self._last_load = load_time
self.v4 = v4
self.v6 = v6
self.aliases = aliases
self.reverse = reverse
def call_switch_directly():
import gevent
def talk(msg):
print msg
gevent.sleep(0)
print msg
g = gevent.spawn(talk, 'bar')
gevent.get_hub().switch()
def open(self):
super(GeventDaemonContext, self).open()
# always reinit even when not forked when registering signals
self._apply_monkey_patch()
import gevent
if self.gevent_hub is not None:
# gevent 1.0 only
gevent.get_hub(self.gevent_hub)
gevent.reinit()
self._setup_gevent_signals()
def setUp(self):
super(TestCase, self).setUp()
# Especially if we're running in leakcheck mode, where
# the same test gets executed repeatedly, we need to update the
# current time. Tests don't always go through the full event loop,
# so that doesn't always happen. test__pool.py:TestPoolYYY.test_async
# tends to show timeouts that are too short if we don't.
# XXX: Should some core part of the loop call this?
gevent.get_hub().loop.update_now()
self.close_on_teardown = []
def signal(signalnum, handler):
"""
Exactly the same as :func:`signal.signal` except where
:const:`signal.SIGCHLD` is concerned.
.. note::
A :const:`signal.SIGCHLD` handler installed with this function
will only be triggered for children that are forked using
:func:`gevent.os.fork` (:func:`gevent.os.fork_and_watch`);
children forked before monkey patching, or otherwise by the raw
:func:`os.fork`, will not trigger the handler installed by this
function. (It's unlikely that a SIGCHLD handler installed with
the builtin :func:`signal.signal` would be triggered either;
libev typically overwrites such a handler at the C level. At
the very least, it's full of race conditions.)
.. note::
Use of ``SIG_IGN`` and ``SIG_DFL`` may also have race conditions
with libev child watchers and the :mod:`gevent.subprocess` module.
.. versionchanged:: 1.2a1
If ``SIG_IGN`` or ``SIG_DFL`` are used to ignore ``SIGCHLD``, a
future use of ``gevent.subprocess`` and libev child watchers
will once again work. However, on Python 2, use of ``os.popen``
will fail.
.. versionchanged:: 1.1rc2
Allow using ``SIG_IGN`` and ``SIG_DFL`` to reset and ignore ``SIGCHLD``.
However, this allows the possibility of a race condition if ``gevent.subprocess``
had already been used.
"""
if signalnum != _signal.SIGCHLD:
return _signal_signal(signalnum, handler)
# TODO: raise value error if not called from the main
# greenlet, just like threads
if handler != _signal.SIG_IGN and handler != _signal.SIG_DFL and not callable(handler):
# exact same error message raised by the stdlib
raise TypeError("signal handler must be signal.SIG_IGN, signal.SIG_DFL, or a callable object")
old_handler = getsignal(signalnum)
global _child_handler
_child_handler = handler
if handler == _signal.SIG_IGN or handler == _signal.SIG_DFL:
# Allow resetting/ignoring this signal at the process level.
# Note that this conflicts with gevent.subprocess and other users
# of child watchers, until the next time gevent.subprocess/loop.install_sigchld()
# is called.
from gevent import get_hub # Are we always safe to import here?
_signal_signal(signalnum, handler)
get_hub().loop.reset_sigchld()
return old_handler
def _work(self, burst=False, logging_level="INFO"):
"""Starts the work loop.
Pops and performs all jobs on the current list of queues. When all
queues are empty, block and wait for new jobs to arrive on any of the
queues, unless `burst` mode is enabled.
The return value indicates whether any jobs were processed.
"""
setup_loghandlers(logging_level)
self._install_signal_handlers()
self.did_perform_work = False
self.register_birth()
self.log.info("RQ GEVENT worker (Greenlet pool size={0}) {1!r} started, version {2}".
format(self.gevent_pool.size, self.key, VERSION))
self.set_state(WorkerStatus.STARTED)
try:
while True:
try:
self.check_for_suspension(burst)
if self.should_run_maintenance_tasks:
self.clean_registries()
if self._stop_requested:
self.log.info('Stopping on request.')
break
timeout = None if burst else max(1, self.default_worker_ttl - 60)
result = self.dequeue_job_and_maintain_ttl(timeout)
if result is None and burst:
self.log.info("RQ worker {0!r} done, quitting".format(self.key))
try:
# Make sure dependented jobs are enqueued.
get_hub().switch()
except LoopExit:
pass
result = self.dequeue_job_and_maintain_ttl(timeout)
if result is None:
break
except StopRequested:
break
job, queue = result
self.execute_job(job, queue)
finally:
if not self.is_horse:
self.register_death()
return self.did_perform_work
def _store_error(self, where, type, value, tb):
del tb
if self._error != self._none:
gevent.get_hub().parent.throw(type, value)
else:
self._error = (where, type, value)
if sys.version_info[0] >= 3:
try:
value.__traceback__ = None
except AttributeError:
pass
def start(self):
# Start grabbing SIGCHLD within libev event loop.
gevent.get_hub().loop.install_sigchld()
# Run new process (based on `fork()` on POSIX-compliant systems).
super(_GProcess, self).start()
# The occurrence of SIGCHLD is recorded asynchronously in libev.
# This guarantees proper behavior even if the child watcher is
# started after the child exits. Start child watcher now.
self._sigchld_watcher = gevent.get_hub().loop.child(self.pid)
self._returnevent = gevent.event.Event()
self._sigchld_watcher.start(self._on_sigchld, self._sigchld_watcher)
def _client_exception_handler(self, greenlet):
self._running = False
try:
greenlet.get()
except MatrixError as ex:
gevent.get_hub().handle_system_error(
TransportError,
TransportError(
f'Unexpected error while communicating with Matrix homeserver: {ex}',
),
)
def setup(self):
if not batchy_gevent:
raise SkipTest()
use_gevent_local()
# Quiet gevent's internal exception printing.
self.old_print_exception = gevent.get_hub().print_exception
gevent.get_hub().print_exception = lambda context, type, value, tb: None
global CALL_COUNT
CALL_COUNT = 0
def _notify_waiter(self, service_name, action):
"""
通知指定service_name下的所有waiter
:param service_name:
:param action: 当前变更动作，定义与constatn.SERVICE_ACTION
:type service_name: str
:type action: str
:return:
"""
for waiter in self._waiter_dict[service_name]:
gevent.get_hub().loop.run_callback(lambda: waiter.switch(action))
def _test_wait_read_invalid_switch(self, sleep):
sock1, sock2 = socket.socketpair()
try:
p = gevent.spawn(util.wrap_errors(AssertionError, socket.wait_read), sock1.fileno())
gevent.get_hub().loop.run_callback(switch_None, p)
if sleep is not None:
gevent.sleep(sleep)
result = p.get()
assert isinstance(result, AssertionError), result
assert 'Invalid switch' in str(result), repr(str(result))
finally:
sock1.close()
sock2.close()
def test(self):
# hub.join() guarantees that loop has exited cleanly
res = gevent.get_hub().join()
self.assertTrue(res)
res = gevent.get_hub().join()
self.assertTrue(res)
# but it is still possible to use gevent afterwards
gevent.sleep(0.01)
res = gevent.get_hub().join()
self.assertTrue(res)
def _child(target, args, kwargs): """Wrapper function that runs in child process. Resets gevent/libev state and executes user-given function. """ _tryGevent() _reset_signal_handlers() gevent.reinit() hub = gevent.get_hub() del hub.threadpool hub._threadpool = None hub.destroy(destroy_loop=True) h = gevent.get_hub(default=True) assert h.loop.default, 'Could not create libev default event loop.' target(*args, **kwargs)
def __init__(self):
# {int -> flags}
# We can't keep watcher objects in here because people commonly
# just drop the poll object when they're done, without calling
# unregister(). dnspython does this.
self.fds = {}
self.loop = get_hub().loop
def run(ctx, dev):
"""Start the client"""
# create app
app = EthApp(ctx.obj['config'])
if dev:
gevent.get_hub().SYSTEM_ERROR = BaseException
try:
ctx.obj['config']['client_version'] += '/' + os.getlogin()
except:
log.warn("can't get and add login name to client_version")
pass
# register services
for service in services:
assert issubclass(service, BaseService)
if service.name not in app.config['deactivated_services']:
assert service.name not in app.services
service.register_with_app(app)
assert hasattr(app.services, service.name)
# start app
app.start()
# wait for interupt
evt = Event()
gevent.signal(signal.SIGQUIT, evt.set)
gevent.signal(signal.SIGTERM, evt.set)
gevent.signal(signal.SIGINT, evt.set)
evt.wait()
# finally stop
app.stop()
def _start_app(config, account):
# create app
app = HPCApp(config)
# development mode
if False:
gevent.get_hub().SYSTEM_ERROR = BaseException
# dump config
pyethapp_app.dump_config(config)
# init accounts first, as we need (and set by copy) the coinbase early FIXME
if AccountsService in services:
AccountsService.register_with_app(app)
# add account
app.services.accounts.add_account(account, store=False)
assert app.services.accounts.coinbase in config['hdc']['validators']
# register services
for service in services:
assert issubclass(service, BaseService)
if service.name not in app.config['deactivated_services'] + [AccountsService.name]:
assert service.name not in app.services
service.register_with_app(app)
assert hasattr(app.services, service.name)
# start app
log.info('starting')
app.start()
if config['post_app_start_callback'] is not None:
config['post_app_start_callback'](app)
return app
def test2(self):
timer = gevent.get_hub().loop.timer(0)
timer.start(hello2)
gevent.sleep(0.1)
assert sys.exc_info() == (None, None, None), sys.exc_info()
if six.PY3:
expected_error.__traceback__ = None
def __init__(self, delay, fn, *args, **kwargs):
self._timer = gevent.get_hub().loop.timer(delay, 0)
self._fn = fn
self._delay = delay
self._args = args
self._kwargs = kwargs
def gevent_error_handler(context, exc_info):
e = exc_info[1]
if isinstance(e, ssl.SSLError) and e.reason == 'HTTP_REQUEST':
return
gevent.get_hub().handle_system_error(exc_info[0], exc_info[1])
import gevent
import greenlet
def callback(event, args):
print
event, args[0], '===:>>>>', args[1]
def foo():
print('Running in foo')
gevent.sleep(0)
print('Explicit context switch to foo again')
def bar():
print('Explicit context to bar')
gevent.sleep(0)
print('Implicit context switch back to bar')
print('main greenlet info: ', greenlet.greenlet.getcurrent())
print('hub info', gevent.get_hub())
oldtrace = greenlet.settrace(callback)
gevent.joinall([
gevent.spawn(foo),
gevent.spawn(bar),
])
greenlet.settrace(oldtrace)
from ssh2.sftp import LIBSSH2_FXF_READ, LIBSSH2_FXF_CREAT, LIBSSH2_FXF_WRITE, \
LIBSSH2_FXF_TRUNC, LIBSSH2_SFTP_S_IRUSR, LIBSSH2_SFTP_S_IRGRP, \
LIBSSH2_SFTP_S_IWUSR, LIBSSH2_SFTP_S_IXUSR, LIBSSH2_SFTP_S_IROTH, \
LIBSSH2_SFTP_S_IXGRP, LIBSSH2_SFTP_S_IXOTH
from ...exceptions import UnknownHostException, AuthenticationException, \
ConnectionErrorException, SessionError, SFTPError, SFTPIOError, Timeout, \
SCPError
from ...constants import DEFAULT_RETRIES, RETRY_DELAY
from ...native._ssh2 import wait_select, eagain_write, _read_output
from .common import _validate_pkey_path
Hub.NOT_ERROR = (Exception, )
host_logger = logging.getLogger('pssh.host_logger')
logger = logging.getLogger(__name__)
THREAD_POOL = get_hub().threadpool
class SSHClient(object):
"""ssh2-python (libssh2) based non-blocking SSH client."""
IDENTITIES = (
os.path.expanduser('~/.ssh/id_rsa'),
os.path.expanduser('~/.ssh/id_dsa'),
os.path.expanduser('~/.ssh/identity'),
os.path.expanduser('~/.ssh/id_ecdsa'),
)
def __init__(self,
host,
user=None,
def _gevent_pool_spawn(func, args=()):
import gevent
return gevent.get_hub().threadpool.spawn(func, *args)
import signal
import subprocess
def _waitpid(pid):
try:
_, stat = os.waitpid(pid, 0)
except OSError:
# Interrupted system call
_, stat = os.waitpid(pid, 0)
assert stat == 0, stat
if hasattr(signal, 'SIGCHLD'):
# Do what subprocess does and make sure we have the watcher
# in the parent
get_hub().loop.install_sigchld()
pid = os.fork()
if pid: # parent
_waitpid(pid)
else:
# Child resets.
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
# Go through subprocess because we expect it to automatically
# set up the waiting for us.
popen = subprocess.Popen([sys.executable, '-c', 'import sys'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
popen.stderr.read()
def reset_gevent():
gevent.reinit()
gevent.get_hub().destroy(destroy_loop=True) # Forget previous callbacks.
gevent.get_hub(default=True) # Here is necessary.
from gevent import monkey
import os
import re
import gevent.testing as greentest
import unittest
import socket
from time import time
import traceback
import gevent.socket as gevent_socket
from gevent.testing.util import log
from gevent.testing import six
from gevent.testing.six import xrange
from gevent.testing import flaky
resolver = gevent.get_hub().resolver
log('Resolver: %s', resolver)
if getattr(resolver, 'pool', None) is not None:
resolver.pool.size = 1
from gevent.testing.sysinfo import RESOLVER_NOT_SYSTEM
from gevent.testing.sysinfo import RESOLVER_DNSPYTHON
from gevent.testing.sysinfo import PY2
import gevent.testing.timing
assert gevent_socket.gaierror is socket.gaierror
assert gevent_socket.error is socket.error
DEBUG = os.getenv('GEVENT_DEBUG', '') == 'trace'
from gevent.queue import Queue
from mock import patch
from testil import Config, tempdir
from corehq.apps.tzmigration.timezonemigration import FormJsonDiff
from corehq.form_processor.parsers.ledgers.helpers import UniqueLedgerReference
from .. import casediff as mod
from ..statedb import StateDB, init_state_db
log = logging.getLogger(__name__)
@patch.object(mod.CaseDiffQueue, "BATCH_SIZE", 2)
@patch.object(mod.BatchProcessor, "MAX_RETRIES", 0)
@patch.object(gevent.get_hub(), "SYSTEM_ERROR", BaseException)
class TestCaseDiffQueue(SimpleTestCase):
def setUp(self):
super(TestCaseDiffQueue, self).setUp()
self.patches = [
patch.object(mod, "diff_cases", self.diff_cases),
patch(
"corehq.form_processor.backends.couch.dbaccessors.CaseAccessorCouch.get_cases",
self.get_cases,
),
patch.object(mod, "get_stock_forms_by_case_id",
self.get_stock_forms),
]
for patcher in self.patches:
patcher.start()
# -*- coding: utf-8 -*-
# pylint: disable=wrong-import-position,redefined-outer-name,unused-wildcard-import,wildcard-import
import gevent
from gevent import monkey
monkey.patch_all()
import pytest
from ethereum import slogging
from ethereum.keys import PBKDF2_CONSTANTS
from ethereum import processblock
from ethereum import tester
from raiden.network.rpc.client import GAS_LIMIT
from raiden.tests.fixtures import * # noqa: F401,F403
gevent.get_hub().SYSTEM_ERROR = BaseException
PBKDF2_CONSTANTS['c'] = 100
CATCH_LOG_HANDLER_NAME = 'catch_log_handler'
def pytest_addoption(parser):
parser.addoption(
'--blockchain-type',
choices=['geth', 'tester'],
default='geth',
)
parser.addoption(
'--blockchain-cache',
action='store_true',
def gevent_error_handler(context, exc_info):
e = exc_info[1]
# recover if HTTP request is done to a HTTPS endpoint
if isinstance(e, ssl.SSLError) and e.reason == 'HTTP_REQUEST':
return
gevent.get_hub().handle_system_error(exc_info[0], exc_info[1])
def expect_one_error(self):
self.assertEqual(self._error, self._none)
gevent.get_hub().handle_error = self._store_error
importlib.reload(gevent)
def handleGreenletError(context, type, value, tb):
if context.__class__ is tuple and context[0].__class__.__name__ == "ThreadPool":
# Exceptions in ThreadPool will be handled in the main Thread
return None
if isinstance(value, str):
# Cython can raise errors where the value is a plain string
# e.g., AttributeError, "_semaphore.Semaphore has no attr", <traceback>
value = type(value)
if not issubclass(type, gevent.get_hub().NOT_ERROR):
sys.excepthook(type, value, tb)
gevent.get_hub().handle_error = handleGreenletError
try:
signal.signal(signal.SIGTERM, lambda signum, stack_frame: shutdown("SIGTERM"))
except Exception as err:
logging.debug("Error setting up SIGTERM watcher: %s" % err)
if __name__ == "__main__":
import time
from gevent import monkey
monkey.patch_all(thread=False, ssl=False)
from . import Debug
def sleeper(num):
print("started", num)
# exit_subprocess() will skip the enclosing with block.
with closing(done_w):
try:
run_test()
except:
pickle.dump(traceback.format_exc(), done_w)
else:
pickle.dump(None, done_w)
except:
print('Uh oh, raised from pickle.')
traceback.print_exc()
finally:
os._exit(0)
done_w.close()
# Block for up to MAX_DURATION seconds for the test to finish.
r, w, x = select.select([done_r], [], [], MAX_DURATION)
if done_r in r:
tb = pickle.load(done_r)
assert not tb, tb
else:
os.kill(child, 9)
assert False, 'Test deadlocked after %d seconds.' % MAX_DURATION
if __name__ == "__main__":
print gevent.get_hub()
while True:
test_main()
sys.stderr.write('.')
def __init__(self, socket):
self.fileno = socket.fileno()
self.family = socket.family
self.proto = socket.proto
self._socket = socket
self._hub_id = id(gevent.get_hub())
def setUp(self):
super(TestCase, self).setUp()
if not self.verbose_dns:
# Silence the default reporting of errors from the ThreadPool,
# we handle those here.
gevent.get_hub().exception_stream = None
from __future__ import print_function import gevent # Loop of initial Hub is default loop. hub = gevent.get_hub() assert hub.loop.default, hub # Destroy hub. Does not destroy default loop if not explicitly told to. hub.destroy() hub = gevent.get_hub() assert hub.loop.default, hub saved_loop = hub.loop # Destroy hub including default loop. hub.destroy(destroy_loop=True) assert saved_loop.fileno() is None, saved_loop print(hub, saved_loop) # Create new hub and explicitly request creation of a new default loop. hub = gevent.get_hub(default=True) assert hub.loop.default, hub # Destroy hub including default loop. hub.destroy(destroy_loop=True) # Create new non-default loop in new hub. hub = gevent.get_hub() assert not hub.loop.default, hub hub.destroy()
from __future__ import print_function
import gevent.monkey; gevent.monkey.patch_all()
import gevent
import os
hub = gevent.get_hub()
pid = os.getpid()
newpid = None
def on_fork():
global newpid
newpid = os.getpid()
fork_watcher = hub.loop.fork(ref=False)
fork_watcher.start(on_fork)
# fork watchers weren't firing in multi-threading processes.
def run():
# libev only calls fork callbacks at the beginning of
# the loop; we use callbacks extensively so it takes *two*
# calls to sleep (with a timer) to actually get wrapped
# around to the beginning of the loop.
gevent.sleep(0.01)
gevent.sleep(0.01)
q.put(newpid)
import multiprocessing
address = ('127.0.0.10', 53)
listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
listener.bind(address)
except socket.error as ex:
if 'permission denied' in str(ex).lower():
sys.stderr.write(
'This test binds on port 53 and thus must be run as root.\n')
sys.exit(0)
raise
def reader():
while True:
print listener.recvfrom(10000)
gevent.spawn(reader)
r = gevent.get_hub().resolver = Resolver(servers=['127.0.0.10'],
timeout=0.001,
tries=1)
try:
result = r.gethostbyname('www.google.com')
except socket.gaierror as ex:
if 'ARES_ETIMEOUT' not in str(ex):
raise
else:
raise AssertionError('Expected timeout, got %r' % (result, ))
def expect_one_error(self):
assert self._error == self._none, self._error
self._old_handle_error = gevent.get_hub().handle_error
gevent.get_hub().handle_error = self._store_error
def dont_exit_pytest():
"""Raiden will quit on any unhandled exception.
This allows the test suite to finish in case an exception is unhandled.
"""
gevent.get_hub().NOT_ERROR = (gevent.GreenletExit, SystemExit)
def _store_error(self, where, type, value, tb):
del tb
if self._error != self._none:
gevent.get_hub().parent.throw(type, value)
else:
self._error = (where, type, value)
yield
elapsed = time() - start
assert expected - fuzzy <= elapsed <= expected + fuzzy, 'Expected: %r; elapsed: %r' % (expected, elapsed)
def no_time(fuzzy=0.001):
return expected_time(0, fuzzy=fuzzy)
for _a in xrange(2):
# exiting because the spawned greenlet finished execution (spawn (=callback) variant)
for _ in xrange(2):
x = gevent.spawn(lambda: 5)
with no_time(SMALL):
result = gevent.get_hub().join(timeout=10)
assert result is True, repr(result)
assert x.dead, x
assert x.value == 5, x
# exiting because the spawned greenlet finished execution (spawn_later (=timer) variant)
for _ in xrange(2):
x = gevent.spawn_later(SMALL, lambda: 5)
with expected_time(SMALL):
result = gevent.get_hub().join(timeout=10)
assert result is True, repr(result)
assert x.dead, x
# exiting because of timeout (the spawned greenlet still runs)
for _ in xrange(2):
x = gevent.spawn_later(10, lambda: 5)
def _schedule(self):
timer = gevent.get_hub().loop.timer(self.wait)
timer.start(self.__call__)
for _ in xrange(2):
x = gevent.spawn_later(10, lambda: 5)
event = Event()
event_set = gevent.spawn_later(SMALL, event.set)
with expected_time(SMALL):
result = gevent.wait([event])
assert result == [event], repr(result)
assert not x.dead, x
assert event_set.dead
assert event.is_set()
x.kill()
with no_time():
result = gevent.wait()
assert result is True
# checking "ref=False" argument
for _ in xrange(2):
gevent.get_hub().loop.timer(10, ref=False).start(lambda: None)
with no_time():
result = gevent.wait()
assert result is True
# checking "ref=False" attribute
for _d in xrange(2):
w = gevent.get_hub().loop.timer(10)
w.start(lambda: None)
w.ref = False
with no_time():
result = gevent.wait()
assert result is True
def main(
ctx,
scenario_file,
keystore_file,
password,
chains,
data_path,
auth,
mailgun_api_key,
):
gevent.get_hub().exception_stream = DummyStream()
is_subcommand = ctx.invoked_subcommand is not None
if not is_subcommand and scenario_file is None:
ctx.fail('No scenario definition file provided')
if is_subcommand:
log_file_name = (
f'scenario-player-{ctx.invoked_subcommand}_{datetime.now():%Y-%m-%dT%H:%M:%S}.log'
)
else:
scenario_basename = basename(scenario_file.name)
log_file_name = (
f'scenario-player_{scenario_basename}_{datetime.now():%Y-%m-%dT%H:%M:%S}.log'
)
click.secho(f'Writing log to {log_file_name}', fg='yellow')
configure_logging(
{
'': 'INFO',
'raiden': 'DEBUG',
'scenario_player': 'DEBUG'
},
debug_log_file_name=log_file_name,
_first_party_packages=_FIRST_PARTY_PACKAGES
| frozenset(['scenario_player']),
_debug_log_file_additional_level_filters={'scenario_player': 'DEBUG'},
)
log_buffer = None
if sys.stdout.isatty() and not is_subcommand:
log_buffer = UrwidLogWalker([])
for handler in logging.getLogger('').handlers:
if isinstance(handler, logging.StreamHandler):
handler.terminator = ConcatenableNone()
handler.formatter = NonStringifyingProcessorFormatter(
UrwidLogRenderer(),
foreign_pre_chain=LOGGING_PROCESSORS,
)
handler.stream = log_buffer
break
chain_rpc_urls = defaultdict(list)
for chain_name, chain_rpc_url in chains:
chain_rpc_urls[chain_name].append(chain_rpc_url)
with open(keystore_file, 'r') as keystore:
account = Account(json.load(keystore), password, keystore_file)
log.info("Using account", account=to_checksum_address(account.address))
if is_subcommand:
ctx.obj = dict(
account=account,
chain_rpc_urls=chain_rpc_urls,
data_path=data_path,
)
return
# Collect tasks
collect_tasks(tasks)
runner = ScenarioRunner(account, chain_rpc_urls, auth, Path(data_path),
scenario_file)
ui = ScenarioUI(runner, log_buffer, log_file_name)
ui_greenlet = ui.run()
success = False
try:
try:
runner.run_scenario()
success = True
log.info('Run finished', result='success')
send_notification_mail(
runner.notification_email,
f'Scenario successful {scenario_file.name}',
'Success',
mailgun_api_key,
)
except ScenarioAssertionError as ex:
log.error('Run finished', result='assertion errors')
send_notification_mail(
runner.notification_email,
f'Assertion mismatch in {scenario_file.name}',
str(ex),
mailgun_api_key,
)
except ScenarioError:
log.exception('Run finished', result='scenario error')
send_notification_mail(
runner.notification_email,
f'Invalid scenario {scenario_file.name}',
traceback.format_exc(),
mailgun_api_key,
)
except Exception:
log.exception('Exception while running scenario')
send_notification_mail(
runner.notification_email,
f'Error running scenario {scenario_file.name}',
traceback.format_exc(),
mailgun_api_key,
)
finally:
try:
if sys.stdout.isatty():
ui.set_success(success)
log.warning('Press q to exit')
while not ui_greenlet.dead:
gevent.sleep(1)
finally:
if runner.is_managed:
runner.node_controller.stop()
if not ui_greenlet.dead:
ui_greenlet.kill(ExitMainLoop)
ui_greenlet.join()
def _child(target, args, kwargs):
"""Wrapper function that runs in child process. Resets gevent/libev state
and executes user-given function.
After fork on POSIX-compliant systems, gevent's state is inherited by the
child which may lead to undesired behavior, such as greenlets running in
both, the parent and the child. Therefore, if not on Windows, gevent's and
libev's state is reset before running the user-given function.
"""
log.debug("_child start. target: `%s`", target)
childhandles = list(_filter_handles(chain(args, kwargs.values())))
if not WINDOWS:
# Restore default signal handlers (SIG_DFL). Orphaned libev signal
# watchers may not become properly deactivated otherwise. Note: here, we
# could even reset sigprocmask (Python 2.x does not have API for it, but
# it could be done via ctypes).
_reset_signal_handlers()
# `gevent.reinit` calls `libev.ev_loop_fork()`, which reinitialises
# the kernel state for backends that have one. Must be called in the
# child before using further libev API.
gevent.reinit()
log.debug("Delete current hub's threadpool.")
hub = gevent.get_hub()
# Delete threadpool before hub destruction, otherwise `hub.destroy()`
# might block forever upon `ThreadPool.kill()` as of gevent 1.0rc2.
del hub.threadpool
hub._threadpool = None
# Destroy default event loop via `libev.ev_loop_destroy()` and delete
# hub. This orphans all registered events and greenlets that have been
# duplicated from the parent via fork().
log.debug("Destroy hub and default loop.")
hub.destroy(destroy_loop=True)
# Create a new hub and a new default event loop via
# `libev.gevent_ev_default_loop`.
h = gevent.get_hub(default=True)
log.debug("Created new hub and default event loop.")
assert h.loop.default, 'Could not create libev default event loop.'
# On Unix, file descriptors are inherited by default. Also, the global
# `_all_handles` is inherited from the parent. Close dispensable gipc-
# related file descriptors in child.
for h in _all_handles[:]:
if h not in childhandles:
log.debug("Invalidate %s in child.", h)
h._set_legit_process()
# At duplication time the handle might have been locked. Unlock.
h._lock.counter = 1
h.close()
else:
# On Windows, the state of module globals is not transferred to
# children. Set `_all_handles`.
_set_all_handles(childhandles)
# `_all_handles` now must contain only those handles that have been
# transferred to the child on purpose.
for h in _all_handles:
assert h in childhandles
# Register transferred handles for current process.
for h in childhandles:
h._set_legit_process()
if WINDOWS:
h._winapi_childhandle_after_createprocess_child()
log.debug("Handle `%s` is now valid in child.", h)
# Invoke user-given function.
target(*args, **kwargs)
# Close file descriptors before exiting process. Usually needless (OS
# should take care of this), but being expressive about this is clean.
for h in childhandles:
try:
# The user might already have closed it.
h.close()
except GIPCClosed:
pass
#!/usr/bin/env python
from __future__ import print_function
import gevent
gevent.get_hub(
'select') # this is just to make sure we don't pass any fds to children
from gevent import monkey
monkey.patch_all()
import six
import sys
import os
import glob
import traceback
from time import time
from gevent.pool import Pool
import util
TIMEOUT = 180
NWORKERS = int(os.environ.get('NWORKERS') or 8)
pool = None
def spawn(*args, **kwargs):
g = pool.spawn(*args, **kwargs)
g.link_exception(lambda *args: sys.exit(
'Internal error in testrunner.py: %s %s' % (g, g.exception)))
return g
def run_many(tests, expected=None, failfast=False):
global NWORKERS, pool
def main():
# config
import yaml
import io
import sys
import signal
import gevent
from .peermanager import PeerManager
from .discovery import NodeDiscovery
from devp2p import slogging
log = slogging.get_logger('app')
# read config
sample_config = b"""
discovery:
bootstrap_nodes:
# local bootstrap
- enode://6ed2fecb28ff17dec8647f08aa4368b57790000e0e9b33a7b91f32c41b6ca9ba21600e9a8c44248ce63a71544388c6745fa291f88f8b81e109ba3da11f7b41b9@127.0.0.1:30303
# go_bootstrap
#- enode://6cdd090303f394a1cac34ecc9f7cda18127eafa2a3a06de39f6d920b0e583e062a7362097c7c65ee490a758b442acd5c80c6fce4b148c6a391e946b45131365b@54.169.166.226:30303
# cpp_bootstrap
#- enode://4a44599974518ea5b0f14c31c4463692ac0329cb84851f3435e6d1b18ee4eae4aa495f846a0fa1219bd58035671881d44423876e57db2abd57254d0197da0ebe@5.1.83.226:30303
p2p:
num_peers: 10
listen_host: 0.0.0.0
listen_port: 30303
node:
privkey_hex: 65462b0520ef7d3df61b9992ed3bea0c56ead753be7c8b3614e0ce01e4cac41b
"""
if len(sys.argv) == 1:
config = yaml.load(io.BytesIO(sample_config))
pubkey = crypto.privtopub(decode_hex(config['node']['privkey_hex']))
config['node']['id'] = crypto.sha3(pubkey)
else:
fn = sys.argv[1]
log.info('loading config from', fn=fn)
config = yaml.load(open(fn))
# stop on every unhandled exception!
gevent.get_hub(
).SYSTEM_ERROR = BaseException # (KeyboardInterrupt, SystemExit, SystemError)
print(config)
# create app
app = BaseApp(config)
# register services
NodeDiscovery.register_with_app(app)
PeerManager.register_with_app(app)
# start app
app.start()
# wait for interupt
evt = gevent.event.Event()
# gevent.signal(signal.SIGQUIT, gevent.kill) ## killall pattern
gevent.signal(signal.SIGQUIT, evt.set)
gevent.signal(signal.SIGTERM, evt.set)
gevent.signal(signal.SIGINT, evt.set)
evt.wait()
# finally stop
app.stop()
# pylint: disable=wrong-import-position,redefined-outer-name,unused-wildcard-import,wildcard-import
import re
import gevent
import py
import sys
from gevent import monkey
monkey.patch_all()
import pytest
from raiden.exceptions import RaidenShuttingDown
from raiden.tests.fixtures.variables import * # noqa: F401,F403
from raiden.log_config import configure_logging
gevent.get_hub().SYSTEM_ERROR = BaseException
gevent.get_hub().NOT_ERROR = (gevent.GreenletExit, SystemExit,
RaidenShuttingDown)
CATCH_LOG_HANDLER_NAME = 'catch_log_handler'
def pytest_addoption(parser):
parser.addoption(
'--blockchain-type',
choices=['geth'],
default='geth',
)
parser.addoption(
'--initial-port',