def green_monitor(self):
"""
Start point method for setting up greenlet trace for finding
out blocked green threads.
"""
self._green_hub = gevent.hub.get_hub()
self._green_active = None
self._green_counter = 0
greenlet.settrace(self._green_callback)
monkey.get_original('_thread', 'start_new_thread')(self._green_monitor_thread, ())
self._green_main_threadid = monkey.get_original('_thread', 'get_ident')()
def test_concurrent_real_thread():
from easypy.concurrency import IS_GEVENT
from gevent.monkey import get_original
import logging
sleep = get_original("time", "sleep")
current_thread = get_original("threading", "get_ident")
main_thread = current_thread()
ran = 0
def log_and_sleep():
nonlocal ran
logging.info("test")
sleep(.1)
ran += 1
return current_thread()
if IS_GEVENT:
before = ran
with concurrent(log_and_sleep, real_thread_no_greenlet=True) as c:
pass
result = c.result()
assert ran == before + 1
assert main_thread != result
before = ran
with concurrent(log_and_sleep) as c:
pass
result = c.result()
assert ran == before + 1
assert main_thread == result
else:
before = ran
with concurrent(log_and_sleep, real_thread_no_greenlet=True) as c:
pass
result = c.result()
assert ran == before + 1
assert main_thread != result
before = ran
with concurrent(log_and_sleep) as c:
pass
result = c.result()
assert ran == before + 1
assert main_thread != result
assert ran
def _create_accept(self):
# TODO: raise appropriate errors
# Handle case where we are being used inside gevent
sock = None
try:
from gevent import monkey
if monkey.is_object_patched('socket', 'socket'):
sock = monkey.get_original('socket', 'socket')()
except:
pass
# connect to sam
sock = pysocket.create_connection(self._samAddr)
# say hello
repl = _sam_cmd(sock, _greeting())
if repl.opts["RESULT"] != "OK":
raise pysocket.error(errno.ENOTCONN,
"failed to accept: %s" % repl.opts['RESULT'])
# send command
cmd = 'STREAM ACCEPT ID={} SILENT=false'.format(self.socketname)
repl = _sam_cmd(sock, cmd)
if repl.opts['RESULT'] != 'OK':
raise pysocket.error(errno.ENOTCONN,
"failed to accept: %s" % repl.opts['RESULT'])
sock.setblocking(self._blocking_flag)
return (sock, '')
def _test(self, func_name, *args):
"""
Runs the function *func_name* with *args* and compares gevent and the system.
Returns the gevent result.
"""
gevent_func = getattr(gevent_socket, func_name)
real_func = monkey.get_original('socket', func_name)
tester = getattr(self, '_run_test_' + func_name,
self._run_test_generic)
result = tester(func_name, real_func, gevent_func, args)
_real_result, time_real, gevent_result, time_gevent = result
if self.verbose_dns and time_gevent > time_real + 0.02 and time_gevent > 0.03:
msg = 'gevent:%s%s took %dms versus %dms stdlib' % (
func_name, args, time_gevent * 1000.0, time_real * 1000.0)
if time_gevent > time_real + 1:
word = 'VERY'
else:
word = 'quite'
util.log('\nWARNING: %s slow: %s', word, msg, color='warning')
return gevent_result
def start(command, quiet=False, **kwargs):
timeout = kwargs.pop('timeout', None)
preexec_fn = None
if not os.environ.get('DO_NOT_SETPGRP'):
preexec_fn = getattr(os, 'setpgrp', None)
env = kwargs.pop('env', None)
setenv = kwargs.pop('setenv', None) or {}
name = getname(command, env=env, setenv=setenv)
if preexec_fn is not None:
setenv['DO_NOT_SETPGRP'] = '1'
if setenv:
env = env.copy() if env else os.environ.copy()
env.update(setenv)
if not quiet:
log('+ %s', name)
popen = Popen(command, preexec_fn=preexec_fn, env=env, **kwargs)
popen.name = name
popen.setpgrp_enabled = preexec_fn is not None
popen.was_killed = False
if timeout is not None:
t = get_original('threading', 'Timer')(timeout, kill, args=(popen, ))
popen.timer = t
t.setDaemon(True)
t.start()
popen.timer = t
return popen
def __init__(self, threadpool):
# Construct in the main thread (owner of the threadpool)
# The parent greenlet and thread identifier will be set once the
# new thread begins running.
RawGreenlet.__init__(self)
self._hub = threadpool.hub
# Avoid doing any imports in the background thread if it's not
# necessary (monkey.get_original imports if not patched).
# Background imports can hang Python 2 (gevent's thread resolver runs in the BG,
# and resolving may have to import the idna module, which needs an import lock, so
# resolving at module scope)
if monkey.is_module_patched('sys'):
stderr = monkey.get_original('sys', 'stderr')
else:
stderr = sys.stderr
self._stderr = stderr
# We can capture the task_queue; even though it can change if the threadpool
# is re-innitted, we won't be running in that case
self._task_queue = threadpool.task_queue # type:gevent._threading.Queue
self._task_queue_cookie = self._task_queue.allocate_cookie()
self._unregister_worker = threadpool._unregister_worker
threadpool._register_worker(self)
try:
start_new_thread(self._begin, ())
except:
self._unregister_worker(self)
raise
def _child(self, socket, run, args, kwargs):
"""The body of the child process."""
kill_signo = getattr(self, '_kill_signo', None)
# Reset environments.
reset_signal_handlers(exclude=set([kill_signo] if kill_signo else []))
reset_gevent()
# Reinit the socket because the hub has been destroyed.
socket = fromfd(socket.fileno(), socket.family, socket.proto)
# Make a greenlet but don't start yet.
greenlet = Quietlet(run, *args, **kwargs)
# Register kill signal handler.
if kill_signo:
killed = (lambda signo, frame, socket=socket, greenlet=greenlet:
self._child_killed(socket, greenlet, frame))
signal.signal(kill_signo, killed)
# Notify birth. Use non-gevent socket to avoid gevent conflict.
_fromfd = get_original('socket', 'fromfd')
_socket = _fromfd(socket.fileno(), socket.family, socket.proto)
_socket.sendall(b'\x01')
# Run the greenlet.
greenlet.start()
try:
greenlet.join(0) # Catch exceptions before blocking.
gevent.spawn(self._watch_child_killers, socket, greenlet)
rv = greenlet.get() # Run the function.
except SystemExit as e:
ok, code, rv = False, e.code, e
except BaseException as e:
ok, code, rv = False, 1, e
else:
ok, code = True, 0
# Notify the result.
put(socket, (ok, rv))
os._exit(code)
def test_subprocess_in_native_thread(self):
# gevent.subprocess doesn't work from a background
# native thread. See #688
from gevent import monkey
# must be a native thread; defend against monkey-patching
ex = []
Thread = monkey.get_original('threading', 'Thread')
def fn():
with self.assertRaises(TypeError) as exc:
gevent.subprocess.Popen('echo 123', shell=True)
raise AssertionError(
"Should not be able to construct Popen")
ex.append(exc.exception)
thread = Thread(target=fn)
thread.start()
thread.join()
self.assertEqual(len(ex), 1)
self.assertTrue(isinstance(ex[0], TypeError), ex)
self.assertEqual(
ex[0].args[0],
'child watchers are only available on the default loop')
def test_main_thread(self):
current = threading.current_thread()
self.assertFalse(isinstance(current, threading._DummyThread))
self.assertTrue(isinstance(current, monkey.get_original('threading', 'Thread')))
# in 3.4, if the patch is incorrectly done, getting the repr
# of the thread fails
repr(current)
def test_main_thread(self):
current = threading.current_thread()
self.assertFalse(isinstance(current, threading._DummyThread))
self.assertTrue(
isinstance(current, monkey.get_original('threading', 'Thread')))
# in 3.4, if the patch is incorrectly done, getting the repr
# of the thread fails
repr(current)
def _patch_no_check(module):
if hasattr(module, '__all__'):
attributes = module.__all__
else
attributes = dir(module)
for attrname in attributes:
obj = getattr(module, attrname)
if hasattr(obj, '__module__') and obj.__module__.startswith('gevent.'):
original_obj = monkey.get_original(module.__name__, attrname)
setattr(module, attrname, _proxy(obj, original_obj))
def with_orig():
mod = __import__(mod_name)
now = getattr(mod, func_name)
from gevent.monkey import get_original
orig = get_original(mod_name, func_name)
try:
setattr(mod, func_name, orig)
yield
finally:
setattr(mod, func_name, now)
class _Threadlocal(get_original(thread_mod_name, '_local')):
def __init__(self):
# Use a class with an initializer so that we can test
# for 'is None' instead of catching AttributeError, making
# the code cleaner and possibly solving some corner cases
# (like #687)
super(_Threadlocal, self).__init__()
self.Hub = None
self.loop = None
self.hub = None
def test_patch_in_thread(self):
all_warnings = []
try:
get_ident = threading.get_ident
except AttributeError:
get_ident = threading._get_ident
def process_warnings(warnings):
all_warnings.extend(warnings)
monkey._process_warnings = process_warnings
current = threading.current_thread()
current_id = get_ident()
def target():
tcurrent = threading.current_thread()
monkey.patch_all()
tcurrent2 = threading.current_thread()
self.assertIsNot(tcurrent, current)
# We get a dummy thread now
self.assertIsNot(tcurrent, tcurrent2)
thread = threading.Thread(target=target)
thread.start()
try:
thread.join()
except: # pylint:disable=bare-except
# XXX: This can raise LoopExit in some cases.
greentest.reraiseFlakyTestRaceCondition()
self.assertNotIsInstance(current, threading._DummyThread)
self.assertIsInstance(current,
monkey.get_original('threading', 'Thread'))
# We generated some warnings
if sys.version_info >= (3, 4):
self.assertEqual(all_warnings, [
'Monkey-patching outside the main native thread. Some APIs will not be '
'available. Expect a KeyError to be printed at shutdown.',
'Monkey-patching not on the main thread; threading.main_thread().join() '
'will hang from a greenlet'
])
else:
self.assertEqual(all_warnings, [
'Monkey-patching outside the main native thread. Some APIs will not be '
'available. Expect a KeyError to be printed at shutdown.'
])
# Manual clean up so we don't get a KeyError
del threading._active[current_id]
threading._active[(getattr(threading, 'get_ident', None)
or threading._get_ident)()] = current
def test_keyboard_interrupt_stderr_patched(self):
from gevent import monkey
monkey.patch_sys(stdin=False, stdout=False, stderr=True)
try:
try:
self.start(raise_, KeyboardInterrupt)
while True:
gevent.sleep(0.1)
except KeyboardInterrupt:
pass # expected
finally:
sys.stderr = monkey.get_original('sys', 'stderr')
def test_keyboard_interrupt_stderr_patched(self):
from gevent import monkey
monkey.patch_sys(stdin=False, stdout=False, stderr=True)
try:
try:
self.start(raise_, KeyboardInterrupt)
while True:
gevent.sleep(0.1)
except KeyboardInterrupt:
pass # expected
finally:
sys.stderr = monkey.get_original('sys', 'stderr')
def format_run_info(thread_stacks=True,
greenlet_stacks=True,
limit=_NONE,
current_thread_ident=None):
"""
format_run_info(thread_stacks=True, greenlet_stacks=True, limit=None) -> [str]
Request information about the running threads of the current process.
This is a debugging utility. Its output has no guarantees other than being
intended for human consumption.
:keyword bool thread_stacks: If true, then include the stacks for
running threads.
:keyword bool greenlet_stacks: If true, then include the stacks for
running greenlets. (Spawning stacks will always be printed.)
Setting this to False can reduce the output volume considerably
without reducing the overall information if *thread_stacks* is true
and you can associate a greenlet to a thread (using ``thread_ident``
printed values).
:keyword int limit: If given, passed directly to `traceback.format_stack`.
If not given, this defaults to the whole stack under CPython, and a
smaller stack under PyPy.
:return: A sequence of text lines detailing the stacks of running
threads and greenlets. (One greenlet will duplicate one thread,
the current thread and greenlet. If there are multiple running threads,
the stack for the current greenlet may be incorrectly duplicated in multiple
greenlets.)
Extra information about
:class:`gevent.Greenlet` object will also be returned.
.. versionadded:: 1.3a1
.. versionchanged:: 1.3a2
Renamed from ``dump_stacks`` to reflect the fact that this
prints additional information about greenlets, including their
spawning stack, parent, locals, and any spawn tree locals.
.. versionchanged:: 1.3b1
Added the *thread_stacks*, *greenlet_stacks*, and *limit* params.
"""
if current_thread_ident is None:
from gevent import monkey
current_thread_ident = monkey.get_original(thread_mod_name,
'get_ident')()
lines = []
limit = _STACK_LIMIT if limit is _NONE else limit
_format_thread_info(lines, thread_stacks, limit, current_thread_ident)
_format_greenlet_info(lines, greenlet_stacks, limit)
return lines
def format_run_info(thread_stacks=True,
greenlet_stacks=True,
limit=_NONE,
current_thread_ident=None):
"""
format_run_info(thread_stacks=True, greenlet_stacks=True, limit=None) -> [str]
Request information about the running threads of the current process.
This is a debugging utility. Its output has no guarantees other than being
intended for human consumption.
:keyword bool thread_stacks: If true, then include the stacks for
running threads.
:keyword bool greenlet_stacks: If true, then include the stacks for
running greenlets. (Spawning stacks will always be printed.)
Setting this to False can reduce the output volume considerably
without reducing the overall information if *thread_stacks* is true
and you can associate a greenlet to a thread (using ``thread_ident``
printed values).
:keyword int limit: If given, passed directly to `traceback.format_stack`.
If not given, this defaults to the whole stack under CPython, and a
smaller stack under PyPy.
:return: A sequence of text lines detailing the stacks of running
threads and greenlets. (One greenlet will duplicate one thread,
the current thread and greenlet. If there are multiple running threads,
the stack for the current greenlet may be incorrectly duplicated in multiple
greenlets.)
Extra information about
:class:`gevent.Greenlet` object will also be returned.
.. versionadded:: 1.3a1
.. versionchanged:: 1.3a2
Renamed from ``dump_stacks`` to reflect the fact that this
prints additional information about greenlets, including their
spawning stack, parent, locals, and any spawn tree locals.
.. versionchanged:: 1.3b1
Added the *thread_stacks*, *greenlet_stacks*, and *limit* params.
"""
if current_thread_ident is None:
from gevent import monkey
current_thread_ident = monkey.get_original(thread_mod_name, 'get_ident')()
lines = []
limit = _STACK_LIMIT if limit is _NONE else limit
_format_thread_info(lines, thread_stacks, limit, current_thread_ident)
_format_greenlet_info(lines, greenlet_stacks, limit)
return lines
def test_keyboard_interrupt_stderr_patched(self):
# XXX: This one non-top-level call prevents us from being
# run in a process with other tests.
from gevent import monkey
monkey.patch_sys(stdin=False, stdout=False, stderr=True)
try:
try:
self.start(raise_, KeyboardInterrupt)
while True:
gevent.sleep(0.1)
except KeyboardInterrupt:
pass # expected
finally:
sys.stderr = monkey.get_original('sys', 'stderr')
def test_patch_in_thread(self):
all_warnings = []
try:
get_ident = threading.get_ident
except AttributeError:
get_ident = threading._get_ident
def process_warnings(warnings):
all_warnings.extend(warnings)
monkey._process_warnings = process_warnings
current = threading.current_thread()
current_id = get_ident()
def target():
tcurrent = threading.current_thread()
monkey.patch_all()
tcurrent2 = threading.current_thread()
self.assertIsNot(tcurrent, current)
# We get a dummy thread now
self.assertIsNot(tcurrent, tcurrent2)
thread = threading.Thread(target=target)
thread.start()
try:
thread.join()
except: # pylint:disable=bare-except
# XXX: This can raise LoopExit in some cases.
greentest.reraiseFlakyTestRaceCondition()
self.assertNotIsInstance(current, threading._DummyThread)
self.assertIsInstance(current, monkey.get_original('threading', 'Thread'))
# We generated some warnings
if greentest.PY3:
self.assertEqual(all_warnings,
['Monkey-patching outside the main native thread. Some APIs will not be '
'available. Expect a KeyError to be printed at shutdown.',
'Monkey-patching not on the main thread; threading.main_thread().join() '
'will hang from a greenlet'])
else:
self.assertEqual(all_warnings,
['Monkey-patching outside the main native thread. Some APIs will not be '
'available. Expect a KeyError to be printed at shutdown.'])
# Manual clean up so we don't get a KeyError
del threading._active[current_id]
threading._active[(getattr(threading, 'get_ident', None) or threading._get_ident)()] = current
def test_attributes(self):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
self.assertEqual(socket.AF_INET, s.type)
self.assertEqual(socket.SOCK_DGRAM, s.family)
self.assertEqual(0, s.proto)
if hasattr(socket, 'SOCK_NONBLOCK'):
s.settimeout(1)
self.assertEqual(socket.AF_INET, s.type)
s.setblocking(0)
std_socket = monkey.get_original('socket', 'socket')(socket.AF_INET, socket.SOCK_DGRAM, 0)
std_socket.setblocking(0)
self.assertEqual(std_socket.type, s.type)
def _process(self):
"""runs in a thread, careful"""
self.output.close()
tlog("Thread started, creating RTMPClient")
self.rtmp = RTMPClient(self.url.encode("utf-8"), **self.options)
try:
self.rtmp.connect()
except RTMPError:
raise
event.call_from_thread(self.chunk.file.set_offline,
"cannot connect")
return
path = self.chunk.file.get_download_file()
startat = 0
try:
if os.path.isfile(path) and self.chunk.pos > 0:
startat = self.rtmp.resumefrom(path)
except RTMPResumeError as e:
tlog("no resume, beginning from start " + e.message)
try:
if startat > 0:
output = open(path, "r+b")
output.seek(self.rtmp.tell())
self.last_index = self.rtmp.tell()
else:
output = open(path, "wb+")
except (OSError, IOError) as e:
tlog("error opening file: " + traceback.format_exc(), log.error)
event.call_from_thread(self.chunk.file.fatal, e.strerror)
return
try:
self.rtmp.connectstream(startat)
for buf in iter(self.rtmp.read, ""):
output.write(buf)
if self.stopped:
output.close()
break
ratelimit.sleep(len(buf),
sleepfunc=monkey.get_original("time", "sleep"))
else:
output.close()
event.call_from_thread(self.finalize)
finally:
self.rtmp.close()
output.close()
tlog("download rtmp finished")
return
def test_attributes(self):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
self.assertEqual(socket.AF_INET, s.type)
self.assertEqual(socket.SOCK_DGRAM, s.family)
self.assertEqual(0, s.proto)
if hasattr(socket, 'SOCK_NONBLOCK'):
s.settimeout(1)
self.assertEqual(socket.AF_INET, s.type)
s.setblocking(0)
std_socket = monkey.get_original('socket', 'socket')(socket.AF_INET, socket.SOCK_DGRAM, 0)
std_socket.setblocking(0)
self.assertEqual(std_socket.type, s.type)
s.close()
def _process(self):
"""runs in a thread, careful"""
self.output.close()
tlog("Thread started, creating RTMPClient")
self.rtmp = RTMPClient(self.url.encode("utf-8"), **self.options)
try:
self.rtmp.connect()
except RTMPError:
raise
event.call_from_thread(self.chunk.file.set_offline, "cannot connect")
return
path = self.chunk.file.get_download_file()
startat = 0
try:
if os.path.isfile(path) and self.chunk.pos > 0:
startat = self.rtmp.resumefrom(path)
except RTMPResumeError as e:
tlog("no resume, beginning from start " + e.message)
try:
if startat > 0:
output = open(path, "r+b")
output.seek(self.rtmp.tell())
self.last_index = self.rtmp.tell()
else:
output = open(path, "wb+")
except (OSError, IOError) as e:
tlog("error opening file: " + traceback.format_exc(), log.error)
event.call_from_thread(self.chunk.file.fatal, e.strerror)
return
try:
self.rtmp.connectstream(startat)
for buf in iter(self.rtmp.read, ""):
output.write(buf)
if self.stopped:
output.close()
break
ratelimit.sleep(len(buf), sleepfunc=monkey.get_original("time", "sleep"))
else:
output.close()
event.call_from_thread(self.finalize)
finally:
self.rtmp.close()
output.close()
tlog("download rtmp finished")
return
def _get_original(qual_name):
mod, name = qual_name.split('.')
original = getattr(__import__(mod), name)
try:
from gevent.monkey import get_original
original = get_original(mod, name)
except (ImportError, SyntaxError):
pass
try:
from eventlet.patcher import original
original = getattr(original(mod), name)
except (ImportError, SyntaxError):
pass
return original
def _get_original(qual_name):
mod, name = qual_name.split('.')
original = getattr(__import__(mod), name)
try:
from gevent.monkey import get_original
original = get_original(mod, name)
except (ImportError, SyntaxError):
pass
try:
from eventlet.patcher import original
original = getattr(original(mod), name)
except (ImportError, SyntaxError):
pass
return original
def assertMonkeyPatchedFuncSignatures(self,
mod_name,
func_names=(),
exclude=()):
# We use inspect.getargspec because it's the only thing available
# in Python 2.7, but it is deprecated
# pylint:disable=deprecated-method
import inspect
import warnings
from gevent.monkey import get_original
# XXX: Very similar to gevent.monkey.patch_module. Should refactor?
gevent_module = getattr(__import__('gevent.' + mod_name), mod_name)
module_name = getattr(gevent_module, '__target__', mod_name)
funcs_given = True
if not func_names:
funcs_given = False
func_names = getattr(gevent_module, '__implements__')
for func_name in func_names:
if func_name in exclude:
continue
gevent_func = getattr(gevent_module, func_name)
if not inspect.isfunction(gevent_func) and not funcs_given:
continue
func = get_original(module_name, func_name)
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
gevent_sig = inspect.getargspec(gevent_func)
sig = inspect.getargspec(func)
except TypeError:
if funcs_given:
raise
# Can't do this one. If they specifically asked for it,
# it's an error, otherwise it's not.
# Python 3 can check a lot more than Python 2 can.
continue
self.assertEqual(sig.args, gevent_sig.args, func_name)
# The next three might not actually matter?
self.assertEqual(sig.varargs, gevent_sig.varargs, func_name)
self.assertEqual(sig.keywords, gevent_sig.keywords, func_name)
self.assertEqual(sig.defaults, gevent_sig.defaults, func_name)
def assertMonkeyPatchedFuncSignatures(self, mod_name, func_names=(), exclude=()):
# We use inspect.getargspec because it's the only thing available
# in Python 2.7, but it is deprecated
# pylint:disable=deprecated-method,too-many-locals
import inspect
import warnings
from gevent.monkey import get_original
# XXX: Very similar to gevent.monkey.patch_module. Should refactor?
gevent_module = getattr(__import__('gevent.' + mod_name), mod_name)
module_name = getattr(gevent_module, '__target__', mod_name)
funcs_given = True
if not func_names:
funcs_given = False
func_names = getattr(gevent_module, '__implements__')
for func_name in func_names:
if func_name in exclude:
continue
gevent_func = getattr(gevent_module, func_name)
if not inspect.isfunction(gevent_func) and not funcs_given:
continue
func = get_original(module_name, func_name)
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
gevent_sig = inspect.getargspec(gevent_func)
sig = inspect.getargspec(func)
except TypeError:
if funcs_given:
raise
# Can't do this one. If they specifically asked for it,
# it's an error, otherwise it's not.
# Python 3 can check a lot more than Python 2 can.
continue
self.assertEqual(sig.args, gevent_sig.args, func_name)
# The next three might not actually matter?
self.assertEqual(sig.varargs, gevent_sig.varargs, func_name)
self.assertEqual(sig.keywords, gevent_sig.keywords, func_name)
self.assertEqual(sig.defaults, gevent_sig.defaults, func_name)
def _green_monitor_thread(self):
sleep = monkey.get_original('time', 'sleep')
while True:
# Check every 2 seconds for blocked green threads.
# This could be a knob in the future.
sleep(2)
# If there have been no greenlet switches since last time we
# checked it means we are likely stuck in the same green thread
# for more time than we would like to!
if self._green_counter == 0:
active = self._green_active
# greenlet hub is OK since its the thread waiting for IO.
if active not in (None, self._green_hub):
frame = sys._current_frames()[self._green_main_threadid]
stack = traceback.format_stack(frame)
err_log = ["Green thread seems blocked:\n"] + stack
self.logger.warn(''.join(err_log))
# A race condition may happen here but its fairly rare.
self._green_counter = 0
def get_system_poll():
"""Returns the original select.poll() object. If select.poll is
monkey patched by eventlet or gevent library, it gets the original
select.poll and returns an object of it using the
eventlet.patcher.original/gevent.monkey.get_original functions.
As a last resort, if there is any exception it returns the
SelectPoll() object.
"""
try:
if _using_eventlet_green_select():
_system_poll = eventlet_patcher.original("select").poll
elif gevent_monkey and gevent_monkey.is_object_patched(
'select', 'poll'):
_system_poll = gevent_monkey.get_original('select', 'poll')
else:
_system_poll = select.poll
except:
_system_poll = SelectPoll
return _system_poll()
def get_system_poll():
"""Returns the original select.poll() object. If select.poll is
monkey patched by eventlet or gevent library, it gets the original
select.poll and returns an object of it using the
eventlet.patcher.original/gevent.monkey.get_original functions.
As a last resort, if there is any exception it returns the
SelectPoll() object.
"""
try:
if _using_eventlet_green_select():
_system_poll = eventlet_patcher.original("select").poll
elif gevent_monkey and gevent_monkey.is_object_patched(
'select', 'poll'):
_system_poll = gevent_monkey.get_original('select', 'poll')
else:
_system_poll = select.poll
except:
_system_poll = SelectPoll
return _system_poll()
def _test(self, func, *args):
gevent_func = getattr(gevent_socket, func)
real_func = monkey.get_original('socket', func)
real_result, time_real = run(real_func, *args)
gevent_result, time_gevent = run(gevent_func, *args)
if not DEBUG and self.should_log_results(real_result, gevent_result):
log('')
log_call(real_result, time_real, real_func, *args)
log_call(gevent_result, time_gevent, gevent_func, *args)
self.assertEqualResults(real_result, gevent_result, func)
if self.verbose_dns and time_gevent > time_real + 0.01 and time_gevent > 0.02:
msg = 'gevent:%s%s took %dms versus %dms stdlib' % (func, args, time_gevent * 1000.0, time_real * 1000.0)
if time_gevent > time_real + 1:
word = 'VERY'
else:
word = 'quite'
log('\nWARNING: %s slow: %s', word, msg)
return gevent_result
def test_subprocess_in_native_thread(self):
# gevent.subprocess doesn't work from a background
# native thread. See #688
from gevent import monkey
# must be a native thread; defend against monkey-patching
ex = []
Thread = monkey.get_original('threading', 'Thread')
def fn():
with self.assertRaises(TypeError) as exc:
gevent.subprocess.Popen('echo 123', shell=True)
raise AssertionError("Should not be able to construct Popen")
ex.append(exc.exception)
thread = Thread(target=fn)
thread.start()
thread.join()
self.assertEqual(len(ex), 1)
self.assertTrue(isinstance(ex[0], TypeError), ex)
self.assertEqual(ex[0].args[0], 'child watchers are only available on the default loop')
def _create_accept(self):
# TODO: raise appropriate errors
# Handle case where we are being used inside gevent
sock = None
try:
from gevent import monkey
if monkey.is_object_patched('socket', 'socket'):
sock = monkey.get_original('socket', 'socket')()
except:
pass
# connect to sam
sock = pysocket.create_connection(self._samAddr)
# say hello
repl = _sam_cmd(sock, _greeting())
if repl.opts["RESULT"] != "OK":
raise pysocket.error(errno.ENOTCONN, "failed to accept: %s" % repl.opts['RESULT'])
# send command
cmd = 'STREAM ACCEPT ID={} SILENT=false'.format(self.socketname)
repl = _sam_cmd(sock, cmd)
if repl.opts['RESULT'] != 'OK':
raise pysocket.error(errno.ENOTCONN, "failed to accept: %s" % repl.opts['RESULT'])
sock.setblocking(self._blocking_flag)
return (sock, '')
from gevent.events import MemoryUsageUnderThreshold
from gevent.events import IPeriodicMonitorThread
from gevent.events import implementer
from gevent._tracer import GreenletTracer
from gevent._compat import thread_mod_name
from gevent._compat import perf_counter
from gevent._compat import get_this_psutil_process
__all__ = [
'PeriodicMonitoringThread',
]
get_thread_ident = get_original(thread_mod_name, 'get_ident')
start_new_thread = get_original(thread_mod_name, 'start_new_thread')
thread_sleep = get_original('time', 'sleep')
class MonitorWarning(RuntimeWarning):
"""The type of warnings we emit."""
class _MonitorEntry(object):
__slots__ = ('function', 'period', 'last_run_time')
def __init__(self, function, period):
self.function = function
def test_patch_subprocess_twice(self):
self.assertNotIn('gevent', repr(Popen))
self.assertIs(Popen, monkey.get_original('subprocess', 'Popen'))
monkey.patch_subprocess()
self.assertIs(Popen, monkey.get_original('subprocess', 'Popen'))
# -*- coding: utf-8 -*-
""" Gevent support pieces
"""
from __future__ import absolute_import
from weakref import WeakKeyDictionary
import gevent
from gevent.monkey import get_original
current_thread = get_original('threading', 'current_thread')
RLock = get_original('threading', 'RLock')
class Condition(object):
""" A gevent-aware version of threading.Condition.
This allows greenlets to wait on notifications generated by bona fide
threads.
(Calling threading.Condition.wait from a greenlet will block all greenlets
in the thread.)
This is currently only a partial re-implementation which supports only
``notifyAll``. It does not currently support ``notify``.
"""
def __init__(self, lock=None):
if lock is None:
lock = RLock() # a real threading.Lock
self.lock = lock
self.waiters_by_thread = WeakKeyDictionary()
def assertNotMonkeyPatched(self):
self.assertIs(thread.start_new_thread,
get_original('thread', 'start_new_thread'))
# coding:utf8
import os
import select
import subprocess
from wsgiauth.basic import BasicAuth
from gevent.monkey import get_original
try:
select.poll = get_original('select', 'poll')
except AttributeError:
select.poll = get_original('select', 'kqueue')
subprocess.Popen = get_original('subprocess', 'Popen')
from sina import Sina
from sina.config import DEFAULT_CONFIG
from vilya.config import DEVELOP_MODE
from vilya.libs.permdir import get_repo_root
from vilya.models.project import CodeDoubanProject
from vilya.models.gist import Gist
from vilya.models.user import User
DOUBAN_REALM = "douban wsgi basic auth"
DEFAULT_CONFIG['project_root'] = get_repo_root()
app = Sina(DEFAULT_CONFIG)
# @app.get_repo_path
def get_repo_path_handler(environ, path):
return ''
def test_patch_subprocess_twice(self):
self.assertNotIn('gevent', repr(Popen))
self.assertIs(Popen, monkey.get_original('subprocess', 'Popen'))
monkey.patch_subprocess()
self.assertIs(Popen, monkey.get_original('subprocess', 'Popen'))
from ddtrace.vendor import six
from ddtrace.vendor.six.moves import _thread
from ddtrace.profiling import recorder
from ddtrace.profiling.collector import stack
from . import test_collector
TESTING_GEVENT = os.getenv("DD_PROFILE_TEST_GEVENT", False)
try:
from gevent import monkey
except ImportError:
sleep = time.sleep
else:
sleep = monkey.get_original("time", "sleep")
def func1():
return func2()
def func2():
return func3()
def func3():
return func4()
def func4():
"""
from __future__ import absolute_import
from collections import deque
from gevent import monkey
from gevent._compat import thread_mod_name
__all__ = [
'Lock',
'Queue',
]
start_new_thread, Lock, get_thread_ident, = monkey.get_original(
thread_mod_name, [
'start_new_thread',
'allocate_lock',
'get_ident',
])
# pylint 2.0.dev2 things collections.dequeue.popleft() doesn't return
# pylint:disable=assignment-from-no-return
class _Condition(object):
# pylint:disable=method-hidden
def __init__(self, lock):
self.__lock = lock
self.__waiters = []
# If the lock defines _release_save() and/or _acquire_restore(),
__extra__.remove(name)
else:
globals()[name] = value
__all__ = __implements__ + __imports__
mswindows = sys.platform == 'win32'
if mswindows:
import msvcrt
else:
import fcntl
import pickle
from gevent import monkey
fork = monkey.get_original('os', 'fork')
def call(*popenargs, **kwargs):
"""Run command with arguments. Wait for command to complete, then
return the returncode attribute.
The arguments are the same as for the Popen constructor. Example:
retcode = call(["ls", "-l"])
"""
return Popen(*popenargs, **kwargs).wait()
def check_call(*popenargs, **kwargs):
"""Run command with arguments. Wait for command to complete. If
# -*- coding: utf-8 -*-
try:
# from gevent.monkey import patch_all
# patch_all(subprocess=False, aggressive=False)
from gevent.monkey import get_original
from gevent.server import StreamServer
import select
select.poll = get_original("select", "poll")
import subprocess
subprocess.Popen = get_original("subprocess", "Popen")
except ImportError:
print "You need install gevent manually! System shutdown."
from binascii import hexlify
from maria import Maria
from maria.config import config
from vilya.libs.permdir import get_repo_root
from vilya.models.sshkey import SSHKey
from vilya.models.project import CodeDoubanProject as Project
from vilya.models.gist import Gist
config.project_root = get_repo_root()
config.log_file = None
config.host_key_path = "./host.key"
app = Maria(config=config)
from __future__ import absolute_import
from collections import deque
from gevent import monkey
from gevent._compat import thread_mod_name
__all__ = [
'Lock',
'Queue',
]
start_new_thread, Lock, get_thread_ident, = monkey.get_original(thread_mod_name, [
'start_new_thread', 'allocate_lock', 'get_ident',
])
# pylint 2.0.dev2 things collections.dequeue.popleft() doesn't return
# pylint:disable=assignment-from-no-return
class _Condition(object):
# pylint:disable=method-hidden
def __init__(self, lock):
self.__lock = lock
self.__waiters = []
# If the lock defines _release_save() and/or _acquire_restore(),
pass
if value is _NONE:
__extra__.remove(name)
else:
globals()[name] = value
__all__ = __implements__ + __imports__
mswindows = sys.platform == 'win32'
if mswindows:
import msvcrt
else:
import fcntl
import pickle
from gevent import monkey
fork = monkey.get_original('os', 'fork')
def call(*popenargs, **kwargs):
"""Run command with arguments. Wait for command to complete, then
return the returncode attribute.
The arguments are the same as for the Popen constructor. Example:
retcode = call(["ls", "-l"])
"""
return Popen(*popenargs, **kwargs).wait()
def check_call(*popenargs, **kwargs):
"""Run command with arguments. Wait for command to complete. If
# -*- coding: utf-8 -*-
"""
This file *does not* run ``gevent.monkey.patch_all()``.
It is intended to be used by ``python -m gevent.monkey <this file>``
to prove that the threadpool and getting the original value of things
works.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
from gevent import monkey
from gevent import get_hub
from gevent.thread import get_ident as gr_ident
std_thread_mod = 'thread' if bytes is str else '_thread'
thr_ident = monkey.get_original(std_thread_mod, 'get_ident')
print(thr_ident is gr_ident)
def thread_is_greenlet():
return thr_ident() == gr_ident()
is_greenlet = get_hub().threadpool.apply(thread_is_greenlet)
print(is_greenlet)
print(len(sys._current_frames()))
# -*- coding: utf-8 -*-
try:
import subprocess
from gevent import monkey
subprocess.Popen = monkey.get_original('subprocess', 'Popen')
except:
pass
from sina import Sina
from sina.config import DEFAULT_CONFIG
app = Sina(DEFAULT_CONFIG)
# gunicorn run_gunicorn:app -b 0.0.0.0:8000 -k gevent
# gunicorn run_gunicorn:app -b 0.0.0.0:8000
from gevent import monkey
monkey.patch_all()
# monkey.patch_thread(threading=False, _threading_local=False)
#
import threading
import time
g_local = monkey.get_original('thread', '_local')()
# g_local = threading.local()
g_local.a = 'haha'
def work(index):
global g_local
print threading.currentThread()
print type(g_local)
g_local.a = index
time.sleep(index)
print g_local.a
def start_thread(index):
t = threading.Thread(target=work, args=(index,))
t.start()
return t
def main():
threads = [
start_thread(index)
else:
globals()[name] = value
__all__ = __implements__ + __imports__
mswindows = sys.platform == "win32"
if mswindows:
import msvcrt
else:
import fcntl
import pickle
from gevent import monkey
fork = monkey.get_original("os", "fork")
def call(*popenargs, **kwargs):
"""Run command with arguments. Wait for command to complete, then
return the returncode attribute.
The arguments are the same as for the Popen constructor. Example:
retcode = call(["ls", "-l"])
"""
return Popen(*popenargs, **kwargs).wait()
def check_call(*popenargs, **kwargs):
"""Run command with arguments. Wait for command to complete. If
from greenlet import gettrace
from greenlet import settrace
from gevent.monkey import get_original
from gevent._compat import thread_mod_name
from gevent._compat import NativeStrIO
from gevent.testing import verify
from gevent.testing.skipping import skipWithoutPSUtil
from gevent import _monitor as monitor
from gevent import config as GEVENT_CONFIG
get_ident = get_original(thread_mod_name, 'get_ident')
class MockHub(object):
_threadpool = None
_resolver = None
def __init__(self):
self.thread_ident = get_ident()
self.exception_stream = NativeStrIO()
self.dead = False
def __bool__(self):
return not self.dead
__nonzero__ = __bool__
import os.path
import copy
import urlparse
from gevent.greenlet import Greenlet, SpawnedLink
from gevent.hub import greenlet, getcurrent, get_hub
from gevent import threadpool, monkey
# from collections import defaultdict
from .base import EventObject, OptionData, merge_dict, EventMgr
from .loader import Loader
from .downloader import Downloader
from .parser import Parser, ParseRule
from .urlmgr import UrlMgr
from .log import Logger
from .scheduler import Scheduler
thread_get_ident = monkey.get_original('thread', 'get_ident')
# class ArgsLink(SpawnedLink):
# def __init__(self, callback, *args, **kwargs):
# super(ArgsLink, self).__init__(callback)
# self._args = args
# self._kwargs = kwargs
# # print 'c$$$$$$$$$$$$$$$', self, hasattr(self, 'callback')
# def __call__(self, source):
# # print '$$$$$$$$$$$$$$$', self, hasattr(self, 'callback')
# # g = greenlet(self.callback, get_hub())
# self._args = (source, ) + self._args
# # g.switch(*self._args, **self._kwargs)
from gevent.events import notify
from gevent.events import EventLoopBlocked
from gevent.events import MemoryUsageThresholdExceeded
from gevent.events import MemoryUsageUnderThreshold
from gevent.events import IPeriodicMonitorThread
from gevent.events import implementer
from gevent._compat import thread_mod_name
from gevent._compat import perf_counter
from gevent._util import gmctime
__all__ = [
'PeriodicMonitoringThread',
]
get_thread_ident = get_original(thread_mod_name, 'get_ident')
start_new_thread = get_original(thread_mod_name, 'start_new_thread')
thread_sleep = get_original('time', 'sleep')
class MonitorWarning(RuntimeWarning):
"""The type of warnings we emit."""
class GreenletTracer(object):
# A counter, incremented by the greenlet trace function
# we install on every greenlet switch. This is reset when the
# periodic monitoring thread runs.
greenlet_switch_counter = 0
from gevent._compat import PY3
__all__ = ['Condition',
'Event',
'Lock',
'RLock',
'Semaphore',
'BoundedSemaphore',
'Queue',
'local',
'stack_size']
thread_name = '_thread' if PY3 else 'thread'
start_new_thread, Lock, get_ident, local, stack_size = monkey.get_original(thread_name, [
'start_new_thread', 'allocate_lock', 'get_ident', '_local', 'stack_size'])
class RLock(object):
def __init__(self):
self.__block = Lock()
self.__owner = None
self.__count = 0
def __repr__(self):
owner = self.__owner
return "<%s owner=%r count=%d>" % (
self.__class__.__name__, owner, self.__count)
def acquire(self, blocking=1):