Beispiel #1
0
 def green_monitor(self):
     """
     Start point method for setting up greenlet trace for finding
     out blocked green threads.
     """
     self._green_hub = gevent.hub.get_hub()
     self._green_active = None
     self._green_counter = 0
     greenlet.settrace(self._green_callback)
     monkey.get_original('_thread', 'start_new_thread')(self._green_monitor_thread, ())
     self._green_main_threadid = monkey.get_original('_thread', 'get_ident')()
Beispiel #2
0
def test_concurrent_real_thread():
    from easypy.concurrency import IS_GEVENT
    from gevent.monkey import get_original
    import logging

    sleep = get_original("time", "sleep")
    current_thread = get_original("threading", "get_ident")
    main_thread = current_thread()

    ran = 0

    def log_and_sleep():
        nonlocal ran
        logging.info("test")
        sleep(.1)
        ran += 1
        return current_thread()

    if IS_GEVENT:
        before = ran
        with concurrent(log_and_sleep, real_thread_no_greenlet=True) as c:
            pass
        result = c.result()
        assert ran == before + 1
        assert main_thread != result

        before = ran
        with concurrent(log_and_sleep) as c:
            pass
        result = c.result()
        assert ran == before + 1
        assert main_thread == result
    else:
        before = ran
        with concurrent(log_and_sleep, real_thread_no_greenlet=True) as c:
            pass
        result = c.result()
        assert ran == before + 1
        assert main_thread != result

        before = ran
        with concurrent(log_and_sleep) as c:
            pass
        result = c.result()
        assert ran == before + 1
        assert main_thread != result

    assert ran
 def _create_accept(self):
     # TODO: raise appropriate errors
     # Handle case where we are being used inside gevent
     sock = None
     try:
         from gevent import monkey
         if monkey.is_object_patched('socket', 'socket'):
             sock = monkey.get_original('socket', 'socket')()
     except:
         pass
     # connect to sam
     sock = pysocket.create_connection(self._samAddr)
     # say hello
     repl = _sam_cmd(sock, _greeting())
     if repl.opts["RESULT"] != "OK":
         raise pysocket.error(errno.ENOTCONN,
                              "failed to accept: %s" % repl.opts['RESULT'])
     # send command
     cmd = 'STREAM ACCEPT ID={} SILENT=false'.format(self.socketname)
     repl = _sam_cmd(sock, cmd)
     if repl.opts['RESULT'] != 'OK':
         raise pysocket.error(errno.ENOTCONN,
                              "failed to accept: %s" % repl.opts['RESULT'])
     sock.setblocking(self._blocking_flag)
     return (sock, '')
    def _test(self, func_name, *args):
        """
        Runs the function *func_name* with *args* and compares gevent and the system.

        Returns the gevent result.
        """
        gevent_func = getattr(gevent_socket, func_name)
        real_func = monkey.get_original('socket', func_name)

        tester = getattr(self, '_run_test_' + func_name,
                         self._run_test_generic)
        result = tester(func_name, real_func, gevent_func, args)
        _real_result, time_real, gevent_result, time_gevent = result

        if self.verbose_dns and time_gevent > time_real + 0.02 and time_gevent > 0.03:
            msg = 'gevent:%s%s took %dms versus %dms stdlib' % (
                func_name, args, time_gevent * 1000.0, time_real * 1000.0)

            if time_gevent > time_real + 1:
                word = 'VERY'
            else:
                word = 'quite'

            util.log('\nWARNING: %s slow: %s', word, msg, color='warning')

        return gevent_result
Beispiel #5
0
def start(command, quiet=False, **kwargs):
    timeout = kwargs.pop('timeout', None)
    preexec_fn = None
    if not os.environ.get('DO_NOT_SETPGRP'):
        preexec_fn = getattr(os, 'setpgrp', None)
    env = kwargs.pop('env', None)
    setenv = kwargs.pop('setenv', None) or {}
    name = getname(command, env=env, setenv=setenv)
    if preexec_fn is not None:
        setenv['DO_NOT_SETPGRP'] = '1'
    if setenv:
        env = env.copy() if env else os.environ.copy()
        env.update(setenv)

    if not quiet:
        log('+ %s', name)
    popen = Popen(command, preexec_fn=preexec_fn, env=env, **kwargs)
    popen.name = name
    popen.setpgrp_enabled = preexec_fn is not None
    popen.was_killed = False
    if timeout is not None:
        t = get_original('threading', 'Timer')(timeout, kill, args=(popen, ))
        popen.timer = t
        t.setDaemon(True)
        t.start()
        popen.timer = t
    return popen
    def __init__(self, threadpool):
        # Construct in the main thread (owner of the threadpool)
        # The parent greenlet and thread identifier will be set once the
        # new thread begins running.
        RawGreenlet.__init__(self)

        self._hub = threadpool.hub
        # Avoid doing any imports in the background thread if it's not
        # necessary (monkey.get_original imports if not patched).
        # Background imports can hang Python 2 (gevent's thread resolver runs in the BG,
        # and resolving may have to import the idna module, which needs an import lock, so
        # resolving at module scope)
        if monkey.is_module_patched('sys'):
            stderr = monkey.get_original('sys', 'stderr')
        else:
            stderr = sys.stderr
        self._stderr = stderr
        # We can capture the task_queue; even though it can change if the threadpool
        # is re-innitted, we won't be running in that case
        self._task_queue = threadpool.task_queue  # type:gevent._threading.Queue
        self._task_queue_cookie = self._task_queue.allocate_cookie()
        self._unregister_worker = threadpool._unregister_worker

        threadpool._register_worker(self)
        try:
            start_new_thread(self._begin, ())
        except:
            self._unregister_worker(self)
            raise
Beispiel #7
0
 def _child(self, socket, run, args, kwargs):
     """The body of the child process."""
     kill_signo = getattr(self, '_kill_signo', None)
     # Reset environments.
     reset_signal_handlers(exclude=set([kill_signo] if kill_signo else []))
     reset_gevent()
     # Reinit the socket because the hub has been destroyed.
     socket = fromfd(socket.fileno(), socket.family, socket.proto)
     # Make a greenlet but don't start yet.
     greenlet = Quietlet(run, *args, **kwargs)
     # Register kill signal handler.
     if kill_signo:
         killed = (lambda signo, frame, socket=socket, greenlet=greenlet:
                   self._child_killed(socket, greenlet, frame))
         signal.signal(kill_signo, killed)
     # Notify birth.  Use non-gevent socket to avoid gevent conflict.
     _fromfd = get_original('socket', 'fromfd')
     _socket = _fromfd(socket.fileno(), socket.family, socket.proto)
     _socket.sendall(b'\x01')
     # Run the greenlet.
     greenlet.start()
     try:
         greenlet.join(0)  # Catch exceptions before blocking.
         gevent.spawn(self._watch_child_killers, socket, greenlet)
         rv = greenlet.get()  # Run the function.
     except SystemExit as e:
         ok, code, rv = False, e.code, e
     except BaseException as e:
         ok, code, rv = False, 1, e
     else:
         ok, code = True, 0
     # Notify the result.
     put(socket, (ok, rv))
     os._exit(code)
Beispiel #8
0
        def test_subprocess_in_native_thread(self):
            # gevent.subprocess doesn't work from a background
            # native thread. See #688
            from gevent import monkey

            # must be a native thread; defend against monkey-patching
            ex = []
            Thread = monkey.get_original('threading', 'Thread')

            def fn():
                with self.assertRaises(TypeError) as exc:
                    gevent.subprocess.Popen('echo 123', shell=True)
                    raise AssertionError(
                        "Should not be able to construct Popen")
                ex.append(exc.exception)

            thread = Thread(target=fn)
            thread.start()
            thread.join()

            self.assertEqual(len(ex), 1)
            self.assertTrue(isinstance(ex[0], TypeError), ex)
            self.assertEqual(
                ex[0].args[0],
                'child watchers are only available on the default loop')
 def test_main_thread(self):
     current = threading.current_thread()
     self.assertFalse(isinstance(current, threading._DummyThread))
     self.assertTrue(isinstance(current, monkey.get_original('threading', 'Thread')))
     # in 3.4, if the patch is incorrectly done, getting the repr
     # of the thread fails
     repr(current)
Beispiel #10
0
 def test_main_thread(self):
     current = threading.current_thread()
     self.assertFalse(isinstance(current, threading._DummyThread))
     self.assertTrue(
         isinstance(current, monkey.get_original('threading', 'Thread')))
     # in 3.4, if the patch is incorrectly done, getting the repr
     # of the thread fails
     repr(current)
def _patch_no_check(module):
    if hasattr(module, '__all__'):
        attributes = module.__all__
    else
        attributes = dir(module)
    for attrname in attributes:
        obj = getattr(module, attrname)
        if hasattr(obj, '__module__') and obj.__module__.startswith('gevent.'):
            original_obj = monkey.get_original(module.__name__, attrname)
            setattr(module, attrname, _proxy(obj, original_obj))
 def with_orig():
     mod = __import__(mod_name)
     now = getattr(mod, func_name)
     from gevent.monkey import get_original
     orig = get_original(mod_name, func_name)
     try:
         setattr(mod, func_name, orig)
         yield
     finally:
         setattr(mod, func_name, now)
Beispiel #13
0
class _Threadlocal(get_original(thread_mod_name, '_local')):

    def __init__(self):
        # Use a class with an initializer so that we can test
        # for 'is None' instead of catching AttributeError, making
        # the code cleaner and possibly solving some corner cases
        # (like #687)
        super(_Threadlocal, self).__init__()
        self.Hub = None
        self.loop = None
        self.hub = None
Beispiel #14
0
    def test_patch_in_thread(self):
        all_warnings = []
        try:
            get_ident = threading.get_ident
        except AttributeError:
            get_ident = threading._get_ident

        def process_warnings(warnings):
            all_warnings.extend(warnings)

        monkey._process_warnings = process_warnings

        current = threading.current_thread()
        current_id = get_ident()

        def target():
            tcurrent = threading.current_thread()
            monkey.patch_all()
            tcurrent2 = threading.current_thread()
            self.assertIsNot(tcurrent, current)
            # We get a dummy thread now
            self.assertIsNot(tcurrent, tcurrent2)

        thread = threading.Thread(target=target)
        thread.start()
        try:
            thread.join()
        except:  # pylint:disable=bare-except
            # XXX: This can raise LoopExit in some cases.
            greentest.reraiseFlakyTestRaceCondition()

        self.assertNotIsInstance(current, threading._DummyThread)
        self.assertIsInstance(current,
                              monkey.get_original('threading', 'Thread'))

        # We generated some warnings
        if sys.version_info >= (3, 4):
            self.assertEqual(all_warnings, [
                'Monkey-patching outside the main native thread. Some APIs will not be '
                'available. Expect a KeyError to be printed at shutdown.',
                'Monkey-patching not on the main thread; threading.main_thread().join() '
                'will hang from a greenlet'
            ])
        else:
            self.assertEqual(all_warnings, [
                'Monkey-patching outside the main native thread. Some APIs will not be '
                'available. Expect a KeyError to be printed at shutdown.'
            ])

        # Manual clean up so we don't get a KeyError
        del threading._active[current_id]
        threading._active[(getattr(threading, 'get_ident', None)
                           or threading._get_ident)()] = current
Beispiel #15
0
 def test_keyboard_interrupt_stderr_patched(self):
     from gevent import monkey
     monkey.patch_sys(stdin=False, stdout=False, stderr=True)
     try:
         try:
             self.start(raise_, KeyboardInterrupt)
             while True:
                 gevent.sleep(0.1)
         except KeyboardInterrupt:
             pass  # expected
     finally:
         sys.stderr = monkey.get_original('sys', 'stderr')
Beispiel #16
0
 def test_keyboard_interrupt_stderr_patched(self):
     from gevent import monkey
     monkey.patch_sys(stdin=False, stdout=False, stderr=True)
     try:
         try:
             self.start(raise_, KeyboardInterrupt)
             while True:
                 gevent.sleep(0.1)
         except KeyboardInterrupt:
             pass # expected
     finally:
         sys.stderr = monkey.get_original('sys', 'stderr')
Beispiel #17
0
def format_run_info(thread_stacks=True,
                    greenlet_stacks=True,
                    limit=_NONE,
                    current_thread_ident=None):
    """
    format_run_info(thread_stacks=True, greenlet_stacks=True, limit=None) -> [str]

    Request information about the running threads of the current process.

    This is a debugging utility. Its output has no guarantees other than being
    intended for human consumption.

    :keyword bool thread_stacks: If true, then include the stacks for
       running threads.
    :keyword bool greenlet_stacks: If true, then include the stacks for
       running greenlets. (Spawning stacks will always be printed.)
       Setting this to False can reduce the output volume considerably
       without reducing the overall information if *thread_stacks* is true
       and you can associate a greenlet to a thread (using ``thread_ident``
       printed values).
    :keyword int limit: If given, passed directly to `traceback.format_stack`.
       If not given, this defaults to the whole stack under CPython, and a
       smaller stack under PyPy.

    :return: A sequence of text lines detailing the stacks of running
            threads and greenlets. (One greenlet will duplicate one thread,
            the current thread and greenlet. If there are multiple running threads,
            the stack for the current greenlet may be incorrectly duplicated in multiple
            greenlets.)
            Extra information about
            :class:`gevent.Greenlet` object will also be returned.

    .. versionadded:: 1.3a1
    .. versionchanged:: 1.3a2
       Renamed from ``dump_stacks`` to reflect the fact that this
       prints additional information about greenlets, including their
       spawning stack, parent, locals, and any spawn tree locals.
    .. versionchanged:: 1.3b1
       Added the *thread_stacks*, *greenlet_stacks*, and *limit* params.
    """
    if current_thread_ident is None:
        from gevent import monkey
        current_thread_ident = monkey.get_original(thread_mod_name,
                                                   'get_ident')()

    lines = []

    limit = _STACK_LIMIT if limit is _NONE else limit
    _format_thread_info(lines, thread_stacks, limit, current_thread_ident)
    _format_greenlet_info(lines, greenlet_stacks, limit)
    return lines
Beispiel #18
0
def format_run_info(thread_stacks=True,
                    greenlet_stacks=True,
                    limit=_NONE,
                    current_thread_ident=None):
    """
    format_run_info(thread_stacks=True, greenlet_stacks=True, limit=None) -> [str]

    Request information about the running threads of the current process.

    This is a debugging utility. Its output has no guarantees other than being
    intended for human consumption.

    :keyword bool thread_stacks: If true, then include the stacks for
       running threads.
    :keyword bool greenlet_stacks: If true, then include the stacks for
       running greenlets. (Spawning stacks will always be printed.)
       Setting this to False can reduce the output volume considerably
       without reducing the overall information if *thread_stacks* is true
       and you can associate a greenlet to a thread (using ``thread_ident``
       printed values).
    :keyword int limit: If given, passed directly to `traceback.format_stack`.
       If not given, this defaults to the whole stack under CPython, and a
       smaller stack under PyPy.

    :return: A sequence of text lines detailing the stacks of running
            threads and greenlets. (One greenlet will duplicate one thread,
            the current thread and greenlet. If there are multiple running threads,
            the stack for the current greenlet may be incorrectly duplicated in multiple
            greenlets.)
            Extra information about
            :class:`gevent.Greenlet` object will also be returned.

    .. versionadded:: 1.3a1
    .. versionchanged:: 1.3a2
       Renamed from ``dump_stacks`` to reflect the fact that this
       prints additional information about greenlets, including their
       spawning stack, parent, locals, and any spawn tree locals.
    .. versionchanged:: 1.3b1
       Added the *thread_stacks*, *greenlet_stacks*, and *limit* params.
    """
    if current_thread_ident is None:
        from gevent import monkey
        current_thread_ident = monkey.get_original(thread_mod_name, 'get_ident')()

    lines = []

    limit = _STACK_LIMIT if limit is _NONE else limit
    _format_thread_info(lines, thread_stacks, limit, current_thread_ident)
    _format_greenlet_info(lines, greenlet_stacks, limit)
    return lines
 def test_keyboard_interrupt_stderr_patched(self):
     # XXX: This one non-top-level call prevents us from being
     # run in a process with other tests.
     from gevent import monkey
     monkey.patch_sys(stdin=False, stdout=False, stderr=True)
     try:
         try:
             self.start(raise_, KeyboardInterrupt)
             while True:
                 gevent.sleep(0.1)
         except KeyboardInterrupt:
             pass # expected
     finally:
         sys.stderr = monkey.get_original('sys', 'stderr')
    def test_patch_in_thread(self):
        all_warnings = []
        try:
            get_ident = threading.get_ident
        except AttributeError:
            get_ident = threading._get_ident

        def process_warnings(warnings):
            all_warnings.extend(warnings)
        monkey._process_warnings = process_warnings

        current = threading.current_thread()
        current_id = get_ident()

        def target():
            tcurrent = threading.current_thread()
            monkey.patch_all()
            tcurrent2 = threading.current_thread()
            self.assertIsNot(tcurrent, current)
            # We get a dummy thread now
            self.assertIsNot(tcurrent, tcurrent2)

        thread = threading.Thread(target=target)
        thread.start()
        try:
            thread.join()
        except: # pylint:disable=bare-except
            # XXX: This can raise LoopExit in some cases.
            greentest.reraiseFlakyTestRaceCondition()

        self.assertNotIsInstance(current, threading._DummyThread)
        self.assertIsInstance(current, monkey.get_original('threading', 'Thread'))


        # We generated some warnings
        if greentest.PY3:
            self.assertEqual(all_warnings,
                             ['Monkey-patching outside the main native thread. Some APIs will not be '
                              'available. Expect a KeyError to be printed at shutdown.',
                              'Monkey-patching not on the main thread; threading.main_thread().join() '
                              'will hang from a greenlet'])
        else:
            self.assertEqual(all_warnings,
                             ['Monkey-patching outside the main native thread. Some APIs will not be '
                              'available. Expect a KeyError to be printed at shutdown.'])


        # Manual clean up so we don't get a KeyError
        del threading._active[current_id]
        threading._active[(getattr(threading, 'get_ident', None) or threading._get_ident)()] = current
Beispiel #21
0
    def test_attributes(self):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
        self.assertEqual(socket.AF_INET, s.type)
        self.assertEqual(socket.SOCK_DGRAM, s.family)
        self.assertEqual(0, s.proto)

        if hasattr(socket, 'SOCK_NONBLOCK'):
            s.settimeout(1)
            self.assertEqual(socket.AF_INET, s.type)

            s.setblocking(0)
            std_socket = monkey.get_original('socket', 'socket')(socket.AF_INET, socket.SOCK_DGRAM, 0)
            std_socket.setblocking(0)
            self.assertEqual(std_socket.type, s.type)
Beispiel #22
0
    def _process(self):
        """runs in a thread, careful"""
        self.output.close()
        tlog("Thread started, creating RTMPClient")
        self.rtmp = RTMPClient(self.url.encode("utf-8"), **self.options)
        try:
            self.rtmp.connect()
        except RTMPError:
            raise
            event.call_from_thread(self.chunk.file.set_offline,
                                   "cannot connect")
            return
        path = self.chunk.file.get_download_file()
        startat = 0
        try:
            if os.path.isfile(path) and self.chunk.pos > 0:
                startat = self.rtmp.resumefrom(path)
        except RTMPResumeError as e:
            tlog("no resume, beginning from start " + e.message)
        try:
            if startat > 0:
                output = open(path, "r+b")
                output.seek(self.rtmp.tell())
                self.last_index = self.rtmp.tell()
            else:
                output = open(path, "wb+")
        except (OSError, IOError) as e:
            tlog("error opening file: " + traceback.format_exc(), log.error)
            event.call_from_thread(self.chunk.file.fatal, e.strerror)
            return

        try:
            self.rtmp.connectstream(startat)
            for buf in iter(self.rtmp.read, ""):
                output.write(buf)
                if self.stopped:
                    output.close()
                    break
                ratelimit.sleep(len(buf),
                                sleepfunc=monkey.get_original("time", "sleep"))
            else:
                output.close()
                event.call_from_thread(self.finalize)
        finally:
            self.rtmp.close()
            output.close()

        tlog("download rtmp finished")
        return
Beispiel #23
0
    def test_attributes(self):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
        self.assertEqual(socket.AF_INET, s.type)
        self.assertEqual(socket.SOCK_DGRAM, s.family)
        self.assertEqual(0, s.proto)

        if hasattr(socket, 'SOCK_NONBLOCK'):
            s.settimeout(1)
            self.assertEqual(socket.AF_INET, s.type)

            s.setblocking(0)
            std_socket = monkey.get_original('socket', 'socket')(socket.AF_INET, socket.SOCK_DGRAM, 0)
            std_socket.setblocking(0)
            self.assertEqual(std_socket.type, s.type)

        s.close()
Beispiel #24
0
 def _process(self):
     """runs in a thread, careful"""
     self.output.close()
     tlog("Thread started, creating RTMPClient")
     self.rtmp = RTMPClient(self.url.encode("utf-8"), **self.options)
     try:
         self.rtmp.connect()
     except RTMPError:
         raise
         event.call_from_thread(self.chunk.file.set_offline, "cannot connect")
         return
     path = self.chunk.file.get_download_file()
     startat = 0
     try:
         if os.path.isfile(path) and self.chunk.pos > 0:
             startat = self.rtmp.resumefrom(path)
     except RTMPResumeError as e:
         tlog("no resume, beginning from start " + e.message)
     try:
         if startat > 0:
             output = open(path, "r+b")
             output.seek(self.rtmp.tell())
             self.last_index = self.rtmp.tell()
         else:
             output = open(path, "wb+")
     except (OSError, IOError) as e:
         tlog("error opening file: " + traceback.format_exc(), log.error)
         event.call_from_thread(self.chunk.file.fatal, e.strerror)
         return
          
     try:
         self.rtmp.connectstream(startat)
         for buf in iter(self.rtmp.read, ""):
             output.write(buf)
             if self.stopped:
                 output.close()
                 break
             ratelimit.sleep(len(buf), sleepfunc=monkey.get_original("time", "sleep"))
         else:
             output.close()
             event.call_from_thread(self.finalize)
     finally:
         self.rtmp.close()
         output.close()
         
     tlog("download rtmp finished")
     return
Beispiel #25
0
def _get_original(qual_name):
    mod, name = qual_name.split('.')
    original = getattr(__import__(mod), name)

    try:
        from gevent.monkey import get_original
        original = get_original(mod, name)
    except (ImportError, SyntaxError):
        pass

    try:
        from eventlet.patcher import original
        original = getattr(original(mod), name)
    except (ImportError, SyntaxError):
        pass

    return original
Beispiel #26
0
def _get_original(qual_name):
    mod, name = qual_name.split('.')
    original = getattr(__import__(mod), name)

    try:
        from gevent.monkey import get_original
        original = get_original(mod, name)
    except (ImportError, SyntaxError):
        pass

    try:
        from eventlet.patcher import original
        original = getattr(original(mod), name)
    except (ImportError, SyntaxError):
        pass

    return original
Beispiel #27
0
    def assertMonkeyPatchedFuncSignatures(self,
                                          mod_name,
                                          func_names=(),
                                          exclude=()):
        # We use inspect.getargspec because it's the only thing available
        # in Python 2.7, but it is deprecated
        # pylint:disable=deprecated-method
        import inspect
        import warnings
        from gevent.monkey import get_original
        # XXX: Very similar to gevent.monkey.patch_module. Should refactor?
        gevent_module = getattr(__import__('gevent.' + mod_name), mod_name)
        module_name = getattr(gevent_module, '__target__', mod_name)

        funcs_given = True
        if not func_names:
            funcs_given = False
            func_names = getattr(gevent_module, '__implements__')

        for func_name in func_names:
            if func_name in exclude:
                continue
            gevent_func = getattr(gevent_module, func_name)
            if not inspect.isfunction(gevent_func) and not funcs_given:
                continue

            func = get_original(module_name, func_name)

            try:
                with warnings.catch_warnings():
                    warnings.simplefilter("ignore")
                    gevent_sig = inspect.getargspec(gevent_func)
                    sig = inspect.getargspec(func)
            except TypeError:
                if funcs_given:
                    raise
                # Can't do this one. If they specifically asked for it,
                # it's an error, otherwise it's not.
                # Python 3 can check a lot more than Python 2 can.
                continue
            self.assertEqual(sig.args, gevent_sig.args, func_name)
            # The next three might not actually matter?
            self.assertEqual(sig.varargs, gevent_sig.varargs, func_name)
            self.assertEqual(sig.keywords, gevent_sig.keywords, func_name)
            self.assertEqual(sig.defaults, gevent_sig.defaults, func_name)
Beispiel #28
0
    def assertMonkeyPatchedFuncSignatures(self, mod_name, func_names=(), exclude=()):
        # We use inspect.getargspec because it's the only thing available
        # in Python 2.7, but it is deprecated
        # pylint:disable=deprecated-method,too-many-locals
        import inspect
        import warnings
        from gevent.monkey import get_original
        # XXX: Very similar to gevent.monkey.patch_module. Should refactor?
        gevent_module = getattr(__import__('gevent.' + mod_name), mod_name)
        module_name = getattr(gevent_module, '__target__', mod_name)

        funcs_given = True
        if not func_names:
            funcs_given = False
            func_names = getattr(gevent_module, '__implements__')

        for func_name in func_names:
            if func_name in exclude:
                continue
            gevent_func = getattr(gevent_module, func_name)
            if not inspect.isfunction(gevent_func) and not funcs_given:
                continue

            func = get_original(module_name, func_name)

            try:
                with warnings.catch_warnings():
                    warnings.simplefilter("ignore")
                    gevent_sig = inspect.getargspec(gevent_func)
                    sig = inspect.getargspec(func)
            except TypeError:
                if funcs_given:
                    raise
                # Can't do this one. If they specifically asked for it,
                # it's an error, otherwise it's not.
                # Python 3 can check a lot more than Python 2 can.
                continue
            self.assertEqual(sig.args, gevent_sig.args, func_name)
            # The next three might not actually matter?
            self.assertEqual(sig.varargs, gevent_sig.varargs, func_name)
            self.assertEqual(sig.keywords, gevent_sig.keywords, func_name)
            self.assertEqual(sig.defaults, gevent_sig.defaults, func_name)
Beispiel #29
0
    def _green_monitor_thread(self):
        sleep = monkey.get_original('time', 'sleep')
        while True:
            # Check every 2 seconds for blocked green threads.
            # This could be a knob in the future.
            sleep(2)
            # If there have been no greenlet switches since last time we
            # checked it means we are likely stuck in the same green thread
            # for more time than we would like to!
            if self._green_counter == 0:
                active = self._green_active
                # greenlet hub is OK since its the thread waiting for IO.
                if active not in (None, self._green_hub):
                    frame = sys._current_frames()[self._green_main_threadid]
                    stack = traceback.format_stack(frame)
                    err_log = ["Green thread seems blocked:\n"] + stack
                    self.logger.warn(''.join(err_log))

            # A race condition may happen here but its fairly rare.
            self._green_counter = 0
Beispiel #30
0
def get_system_poll():
    """Returns the original select.poll() object. If select.poll is
    monkey patched by eventlet or gevent library, it gets the original
    select.poll and returns an object of it using the
    eventlet.patcher.original/gevent.monkey.get_original functions.

    As a last resort, if there is any exception it returns the
    SelectPoll() object.
    """
    try:
        if _using_eventlet_green_select():
            _system_poll = eventlet_patcher.original("select").poll
        elif gevent_monkey and gevent_monkey.is_object_patched(
                'select', 'poll'):
            _system_poll = gevent_monkey.get_original('select', 'poll')
        else:
            _system_poll = select.poll
    except:
        _system_poll = SelectPoll

    return _system_poll()
Beispiel #31
0
def get_system_poll():
    """Returns the original select.poll() object. If select.poll is
    monkey patched by eventlet or gevent library, it gets the original
    select.poll and returns an object of it using the
    eventlet.patcher.original/gevent.monkey.get_original functions.

    As a last resort, if there is any exception it returns the
    SelectPoll() object.
    """
    try:
        if _using_eventlet_green_select():
            _system_poll = eventlet_patcher.original("select").poll
        elif gevent_monkey and gevent_monkey.is_object_patched(
                'select', 'poll'):
            _system_poll = gevent_monkey.get_original('select', 'poll')
        else:
            _system_poll = select.poll
    except:
        _system_poll = SelectPoll

    return _system_poll()
Beispiel #32
0
    def _test(self, func, *args):
        gevent_func = getattr(gevent_socket, func)
        real_func = monkey.get_original('socket', func)
        real_result, time_real = run(real_func, *args)
        gevent_result, time_gevent = run(gevent_func, *args)
        if not DEBUG and self.should_log_results(real_result, gevent_result):
            log('')
            log_call(real_result, time_real, real_func, *args)
            log_call(gevent_result, time_gevent, gevent_func, *args)
        self.assertEqualResults(real_result, gevent_result, func)

        if self.verbose_dns and time_gevent > time_real + 0.01 and time_gevent > 0.02:
            msg = 'gevent:%s%s took %dms versus %dms stdlib' % (func, args, time_gevent * 1000.0, time_real * 1000.0)

            if time_gevent > time_real + 1:
                word = 'VERY'
            else:
                word = 'quite'

            log('\nWARNING: %s slow: %s', word, msg)

        return gevent_result
Beispiel #33
0
    def test_subprocess_in_native_thread(self):
        # gevent.subprocess doesn't work from a background
        # native thread. See #688
        from gevent import monkey

        # must be a native thread; defend against monkey-patching
        ex = []
        Thread = monkey.get_original('threading', 'Thread')

        def fn():
            with self.assertRaises(TypeError) as exc:
                gevent.subprocess.Popen('echo 123', shell=True)
                raise AssertionError("Should not be able to construct Popen")
            ex.append(exc.exception)

        thread = Thread(target=fn)
        thread.start()
        thread.join()

        self.assertEqual(len(ex), 1)
        self.assertTrue(isinstance(ex[0], TypeError), ex)
        self.assertEqual(ex[0].args[0], 'child watchers are only available on the default loop')
Beispiel #34
0
 def _create_accept(self):
     # TODO: raise appropriate errors
     # Handle case where we are being used inside gevent
     sock = None
     try:
         from gevent import monkey
         if monkey.is_object_patched('socket', 'socket'):
             sock = monkey.get_original('socket', 'socket')()
     except:
         pass
     # connect to sam
     sock = pysocket.create_connection(self._samAddr)
     # say hello
     repl = _sam_cmd(sock, _greeting())
     if repl.opts["RESULT"] != "OK":
         raise pysocket.error(errno.ENOTCONN, "failed to accept: %s" % repl.opts['RESULT'])
     # send command
     cmd = 'STREAM ACCEPT ID={} SILENT=false'.format(self.socketname)
     repl = _sam_cmd(sock, cmd)
     if repl.opts['RESULT'] != 'OK':
         raise pysocket.error(errno.ENOTCONN, "failed to accept: %s" % repl.opts['RESULT'])
     sock.setblocking(self._blocking_flag)
     return (sock, '')
Beispiel #35
0
from gevent.events import MemoryUsageUnderThreshold
from gevent.events import IPeriodicMonitorThread
from gevent.events import implementer

from gevent._tracer import GreenletTracer
from gevent._compat import thread_mod_name
from gevent._compat import perf_counter
from gevent._compat import get_this_psutil_process



__all__ = [
    'PeriodicMonitoringThread',
]

get_thread_ident = get_original(thread_mod_name, 'get_ident')
start_new_thread = get_original(thread_mod_name, 'start_new_thread')
thread_sleep = get_original('time', 'sleep')



class MonitorWarning(RuntimeWarning):
    """The type of warnings we emit."""


class _MonitorEntry(object):

    __slots__ = ('function', 'period', 'last_run_time')

    def __init__(self, function, period):
        self.function = function
Beispiel #36
0
 def test_patch_subprocess_twice(self):
     self.assertNotIn('gevent', repr(Popen))
     self.assertIs(Popen, monkey.get_original('subprocess', 'Popen'))
     monkey.patch_subprocess()
     self.assertIs(Popen, monkey.get_original('subprocess', 'Popen'))
Beispiel #37
0
# -*- coding: utf-8 -*-
""" Gevent support pieces
"""
from __future__ import absolute_import

from weakref import WeakKeyDictionary

import gevent
from gevent.monkey import get_original

current_thread = get_original('threading', 'current_thread')
RLock = get_original('threading', 'RLock')

class Condition(object):
    """ A gevent-aware version of threading.Condition.

    This allows greenlets to wait on notifications generated by bona fide
    threads.

    (Calling threading.Condition.wait from a greenlet will block all greenlets
    in the thread.)

    This is currently only a partial re-implementation which supports only
    ``notifyAll``.  It does not currently support ``notify``.

    """
    def __init__(self, lock=None):
        if lock is None:
            lock = RLock()              # a real threading.Lock
        self.lock = lock
        self.waiters_by_thread = WeakKeyDictionary()
Beispiel #38
0
 def assertNotMonkeyPatched(self):
     self.assertIs(thread.start_new_thread,
                   get_original('thread', 'start_new_thread'))
Beispiel #39
0
# coding:utf8
import os
import select
import subprocess
from wsgiauth.basic import BasicAuth

from gevent.monkey import get_original
try:
    select.poll = get_original('select', 'poll')
except AttributeError:
    select.poll = get_original('select', 'kqueue')
subprocess.Popen = get_original('subprocess', 'Popen')

from sina import Sina
from sina.config import DEFAULT_CONFIG

from vilya.config import DEVELOP_MODE
from vilya.libs.permdir import get_repo_root
from vilya.models.project import CodeDoubanProject
from vilya.models.gist import Gist
from vilya.models.user import User

DOUBAN_REALM = "douban wsgi basic auth"
DEFAULT_CONFIG['project_root'] = get_repo_root()
app = Sina(DEFAULT_CONFIG)


# @app.get_repo_path
def get_repo_path_handler(environ, path):
    return ''
Beispiel #40
0
 def test_patch_subprocess_twice(self):
     self.assertNotIn('gevent', repr(Popen))
     self.assertIs(Popen, monkey.get_original('subprocess', 'Popen'))
     monkey.patch_subprocess()
     self.assertIs(Popen, monkey.get_original('subprocess', 'Popen'))
Beispiel #41
0
from ddtrace.vendor import six
from ddtrace.vendor.six.moves import _thread

from ddtrace.profiling import recorder
from ddtrace.profiling.collector import stack

from . import test_collector

TESTING_GEVENT = os.getenv("DD_PROFILE_TEST_GEVENT", False)

try:
    from gevent import monkey
except ImportError:
    sleep = time.sleep
else:
    sleep = monkey.get_original("time", "sleep")


def func1():
    return func2()


def func2():
    return func3()


def func3():
    return func4()


def func4():
"""
from __future__ import absolute_import

from collections import deque

from gevent import monkey
from gevent._compat import thread_mod_name

__all__ = [
    'Lock',
    'Queue',
]

start_new_thread, Lock, get_thread_ident, = monkey.get_original(
    thread_mod_name, [
        'start_new_thread',
        'allocate_lock',
        'get_ident',
    ])

# pylint 2.0.dev2 things collections.dequeue.popleft() doesn't return
# pylint:disable=assignment-from-no-return


class _Condition(object):
    # pylint:disable=method-hidden

    def __init__(self, lock):
        self.__lock = lock
        self.__waiters = []

        # If the lock defines _release_save() and/or _acquire_restore(),
Beispiel #43
0
        __extra__.remove(name)
    else:
        globals()[name] = value


__all__ = __implements__ + __imports__


mswindows = sys.platform == 'win32'
if mswindows:
    import msvcrt
else:
    import fcntl
    import pickle
    from gevent import monkey
    fork = monkey.get_original('os', 'fork')


def call(*popenargs, **kwargs):
    """Run command with arguments.  Wait for command to complete, then
    return the returncode attribute.

    The arguments are the same as for the Popen constructor.  Example:

    retcode = call(["ls", "-l"])
    """
    return Popen(*popenargs, **kwargs).wait()


def check_call(*popenargs, **kwargs):
    """Run command with arguments.  Wait for command to complete.  If
Beispiel #44
0
# -*- coding: utf-8 -*-

try:
    # from gevent.monkey import patch_all
    # patch_all(subprocess=False, aggressive=False)
    from gevent.monkey import get_original
    from gevent.server import StreamServer
    import select

    select.poll = get_original("select", "poll")
    import subprocess

    subprocess.Popen = get_original("subprocess", "Popen")
except ImportError:
    print "You need install gevent manually! System shutdown."

from binascii import hexlify
from maria import Maria
from maria.config import config
from vilya.libs.permdir import get_repo_root
from vilya.models.sshkey import SSHKey
from vilya.models.project import CodeDoubanProject as Project
from vilya.models.gist import Gist

config.project_root = get_repo_root()
config.log_file = None
config.host_key_path = "./host.key"

app = Maria(config=config)

Beispiel #45
0
from __future__ import absolute_import

from collections import deque

from gevent import monkey
from gevent._compat import thread_mod_name


__all__ = [
    'Lock',
    'Queue',
]


start_new_thread, Lock, get_thread_ident, = monkey.get_original(thread_mod_name, [
    'start_new_thread', 'allocate_lock', 'get_ident',
])


# pylint 2.0.dev2 things collections.dequeue.popleft() doesn't return
# pylint:disable=assignment-from-no-return


class _Condition(object):
    # pylint:disable=method-hidden

    def __init__(self, lock):
        self.__lock = lock
        self.__waiters = []

        # If the lock defines _release_save() and/or _acquire_restore(),
Beispiel #46
0
                pass
    if value is _NONE:
        __extra__.remove(name)
    else:
        globals()[name] = value

__all__ = __implements__ + __imports__

mswindows = sys.platform == 'win32'
if mswindows:
    import msvcrt
else:
    import fcntl
    import pickle
    from gevent import monkey
    fork = monkey.get_original('os', 'fork')


def call(*popenargs, **kwargs):
    """Run command with arguments.  Wait for command to complete, then
    return the returncode attribute.

    The arguments are the same as for the Popen constructor.  Example:

    retcode = call(["ls", "-l"])
    """
    return Popen(*popenargs, **kwargs).wait()


def check_call(*popenargs, **kwargs):
    """Run command with arguments.  Wait for command to complete.  If
Beispiel #47
0
# -*- coding: utf-8 -*-
"""
This file *does not* run ``gevent.monkey.patch_all()``.

It is intended to be used by ``python -m gevent.monkey <this file>``
to prove that the threadpool and getting the original value of things
works.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import sys
from gevent import monkey
from gevent import get_hub

from gevent.thread import get_ident as gr_ident

std_thread_mod = 'thread' if bytes is str else '_thread'
thr_ident = monkey.get_original(std_thread_mod, 'get_ident')

print(thr_ident is gr_ident)

def thread_is_greenlet():
    return thr_ident() == gr_ident()


is_greenlet = get_hub().threadpool.apply(thread_is_greenlet)
print(is_greenlet)
print(len(sys._current_frames()))
Beispiel #48
0
# -*- coding: utf-8 -*-
try:
    import subprocess
    from gevent import monkey
    subprocess.Popen = monkey.get_original('subprocess', 'Popen')
except:
    pass

from sina import Sina
from sina.config import DEFAULT_CONFIG

app = Sina(DEFAULT_CONFIG)

# gunicorn run_gunicorn:app -b 0.0.0.0:8000 -k gevent
# gunicorn run_gunicorn:app -b 0.0.0.0:8000
Beispiel #49
0
from gevent import monkey
monkey.patch_all()
# monkey.patch_thread(threading=False, _threading_local=False)
#
import threading
import time

g_local = monkey.get_original('thread', '_local')()
# g_local = threading.local()
g_local.a = 'haha'

def work(index):
    global g_local

    print threading.currentThread()
    print type(g_local)
    g_local.a = index
    time.sleep(index)

    print g_local.a

def start_thread(index):
    t = threading.Thread(target=work, args=(index,))
    t.start()

    return t


def main():
    threads = [
        start_thread(index)
Beispiel #50
0
    else:
        globals()[name] = value


__all__ = __implements__ + __imports__


mswindows = sys.platform == "win32"
if mswindows:
    import msvcrt
else:
    import fcntl
    import pickle
    from gevent import monkey

    fork = monkey.get_original("os", "fork")


def call(*popenargs, **kwargs):
    """Run command with arguments.  Wait for command to complete, then
    return the returncode attribute.

    The arguments are the same as for the Popen constructor.  Example:

    retcode = call(["ls", "-l"])
    """
    return Popen(*popenargs, **kwargs).wait()


def check_call(*popenargs, **kwargs):
    """Run command with arguments.  Wait for command to complete.  If
Beispiel #51
0

from greenlet import gettrace
from greenlet import settrace

from gevent.monkey import get_original
from gevent._compat import thread_mod_name
from gevent._compat import NativeStrIO

from gevent.testing import verify
from gevent.testing.skipping import skipWithoutPSUtil

from gevent import _monitor as monitor
from gevent import config as GEVENT_CONFIG

get_ident = get_original(thread_mod_name, 'get_ident')

class MockHub(object):
    _threadpool = None
    _resolver = None

    def __init__(self):
        self.thread_ident = get_ident()
        self.exception_stream = NativeStrIO()
        self.dead = False

    def __bool__(self):
        return not self.dead

    __nonzero__ = __bool__
Beispiel #52
0
import os.path
import copy
import urlparse
from gevent.greenlet import Greenlet, SpawnedLink
from gevent.hub import greenlet, getcurrent, get_hub
from gevent import threadpool, monkey
# from collections import defaultdict
from .base import EventObject, OptionData, merge_dict, EventMgr
from .loader import Loader
from .downloader import Downloader
from .parser import Parser, ParseRule
from .urlmgr import UrlMgr
from .log import Logger
from .scheduler import Scheduler

thread_get_ident = monkey.get_original('thread', 'get_ident')


# class ArgsLink(SpawnedLink):

#     def __init__(self, callback, *args, **kwargs):
#         super(ArgsLink, self).__init__(callback)
#         self._args = args
#         self._kwargs = kwargs
#         # print 'c$$$$$$$$$$$$$$$', self, hasattr(self, 'callback')

#     def __call__(self, source):
#         # print '$$$$$$$$$$$$$$$', self, hasattr(self, 'callback')
#         # g = greenlet(self.callback, get_hub())
#         self._args = (source, ) + self._args
#         # g.switch(*self._args, **self._kwargs)
Beispiel #53
0
from gevent.events import notify
from gevent.events import EventLoopBlocked
from gevent.events import MemoryUsageThresholdExceeded
from gevent.events import MemoryUsageUnderThreshold
from gevent.events import IPeriodicMonitorThread
from gevent.events import implementer

from gevent._compat import thread_mod_name
from gevent._compat import perf_counter
from gevent._util import gmctime

__all__ = [
    'PeriodicMonitoringThread',
]

get_thread_ident = get_original(thread_mod_name, 'get_ident')
start_new_thread = get_original(thread_mod_name, 'start_new_thread')
thread_sleep = get_original('time', 'sleep')


class MonitorWarning(RuntimeWarning):
    """The type of warnings we emit."""


class GreenletTracer(object):

    # A counter, incremented by the greenlet trace function
    # we install on every greenlet switch. This is reset when the
    # periodic monitoring thread runs.
    greenlet_switch_counter = 0
Beispiel #54
0
from gevent._compat import PY3


__all__ = ['Condition',
           'Event',
           'Lock',
           'RLock',
           'Semaphore',
           'BoundedSemaphore',
           'Queue',
           'local',
           'stack_size']


thread_name = '_thread' if PY3 else 'thread'
start_new_thread, Lock, get_ident, local, stack_size = monkey.get_original(thread_name, [
    'start_new_thread', 'allocate_lock', 'get_ident', '_local', 'stack_size'])


class RLock(object):

    def __init__(self):
        self.__block = Lock()
        self.__owner = None
        self.__count = 0

    def __repr__(self):
        owner = self.__owner
        return "<%s owner=%r count=%d>" % (
            self.__class__.__name__, owner, self.__count)

    def acquire(self, blocking=1):