Exemplo n.º 1
0
def install_optimal_reactor(verbose = False):
    """
    Try to install the optimal Twisted reactor for platform.
    
    :param verbose: If ``True``, print what happens.
    :type verbose: bool
    """
    import sys
    from twisted.python import reflect
    if 'twisted.internet.reactor' in sys.modules:
        current_reactor = reflect.qual(sys.modules['twisted.internet.reactor'].__class__).split('.')[-1]
    else:
        current_reactor = None
    if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
        if current_reactor != 'KQueueReactor':
            try:
                v = sys.version_info
                if v[0] == 1 or v[0] == 2 and v[1] < 6 or v[0] == 2 and v[1] == 6 and v[2] < 5:
                    raise Exception('Python version too old ({0}) to use kqueue reactor'.format(sys.version))
                from twisted.internet import kqreactor
                kqreactor.install()
            except Exception as e:
                print 'WARNING: Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor ({0}).'.format(e)
            else:
                if verbose:
                    print 'Running on *BSD or MacOSX and optimal reactor (kqueue) was installed.'
        elif verbose:
            print 'Running on *BSD or MacOSX and optimal reactor (kqueue) already installed.'
    elif sys.platform in ('win32',):
        if current_reactor != 'IOCPReactor':
            try:
                from twisted.internet.iocpreactor import reactor as iocpreactor
                iocpreactor.install()
            except Exception as e:
                print 'WARNING: Running on Windows, but cannot install IOCP Twisted reactor ({0}).'.format(e)
            else:
                if verbose:
                    print 'Running on Windows and optimal reactor (ICOP) was installed.'
        elif verbose:
            print 'Running on Windows and optimal reactor (ICOP) already installed.'
    elif sys.platform.startswith('linux'):
        if current_reactor != 'EPollReactor':
            try:
                from twisted.internet import epollreactor
                epollreactor.install()
            except Exception as e:
                print 'WARNING: Running on Linux, but cannot install Epoll Twisted reactor ({0}).'.format(e)
            else:
                if verbose:
                    print 'Running on Linux and optimal reactor (epoll) was installed.'
        elif verbose:
            print 'Running on Linux and optimal reactor (epoll) already installed.'
    else:
        try:
            from twisted.internet import default as defaultreactor
            defaultreactor.install()
        except Exception as e:
            print 'WARNING: Could not install default Twisted reactor for this platform ({0}).'.format(e)

    return
Exemplo n.º 2
0
 def test_install(self):
     """
     L{install} installs a reactor.
     """
     with NoReactor():
         install()
         self.assertIn("twisted.internet.reactor", sys.modules)
Exemplo n.º 3
0
    def engine_stopped(self):
        logger.error("SpiderRunner engine_stopped")
        try:
            if self.spider.spider and hasattr(self.spider.spider,
                                              'engine_stopped'):
                self.spider.spider.engine_stopped()
            self.handler.parent.writeLog(
                self, str("---- Spider %s engine_stopped ----" % self.name))
            self.management_info['current_endtime'] = datetime.datetime.now(
            ).strftime("%Y%m%d%H%M%S")
            self.is_running = False
            self.management_info['is_running'] = False

            running_spiders = [
                x for x in self.handler.spider_runners if x.spider.crawling
            ]
            if len(running_spiders) == 0:
                from twisted.internet import reactor
                if reactor.running:
                    reactor.stop()
                    import sys
                    del sys.modules['twisted.internet.reactor']
                    from twisted.internet import reactor
                    default.install()
            pass
        except Exception as ex:
            logger.error("SpiderRunner Exception : %s", str(ex))
Exemplo n.º 4
0
 def test_install(self):
     """
     L{install} installs a reactor.
     """
     with NoReactor():
         install()
         self.assertIn("twisted.internet.reactor", sys.modules)
Exemplo n.º 5
0
def run_twisted(host, port, barrier, profile):

    if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
        from twisted.internet import kqreactor
        kqreactor.install()
    elif sys.platform in ['win32']:
        from twisted.internet.iocpreactor import reactor as iocpreactor
        iocpreactor.install()
    elif sys.platform.startswith('linux'):
        from twisted.internet import epollreactor
        epollreactor.install()
    else:
        from twisted.internet import default as defaultreactor
        defaultreactor.install()

    from twisted.web.server import Site
    from twisted.web.resource import Resource
    from twisted.internet import reactor

    class TestResource(Resource):

        def __init__(self, name):
            super().__init__()
            self.name = name
            self.isLeaf = name is not None

        def render_GET(self, request):
            txt = 'Hello, ' + self.name
            request.setHeader(b'Content-Type', b'text/plain; charset=utf-8')
            return txt.encode('utf8')

        def getChild(self, name, request):
            return TestResource(name=name.decode('utf-8'))

    class PrepareResource(Resource):

        isLeaf = True

        def render_GET(self, request):
            gc.collect()
            return b'OK'

    class StopResource(Resource):

        isLeaf = True

        def render_GET(self, request):
            reactor.callLater(0.1, reactor.stop)
            return b'OK'

    root = Resource()
    root.putChild(b'test', TestResource(None))
    root.putChild(b'prepare', PrepareResource())
    root.putChild(b'stop', StopResource())
    site = Site(root)
    reactor.listenTCP(port, site, interface=host)
    barrier.wait()

    reactor.run()
Exemplo n.º 6
0
def run_twisted(host, port, barrier, profile):

    if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
        from twisted.internet import kqreactor
        kqreactor.install()
    elif sys.platform in ['win32']:
        from twisted.internet.iocpreactor import reactor as iocpreactor
        iocpreactor.install()
    elif sys.platform.startswith('linux'):
        from twisted.internet import epollreactor
        epollreactor.install()
    else:
        from twisted.internet import default as defaultreactor
        defaultreactor.install()

    from twisted.web.server import Site
    from twisted.web.resource import Resource
    from twisted.internet import reactor

    class TestResource(Resource):

        def __init__(self, name):
            super().__init__()
            self.name = name
            self.isLeaf = name is not None

        def render_GET(self, request):
            txt = 'Hello, ' + self.name
            request.setHeader(b'Content-Type', b'text/plain; charset=utf-8')
            return txt.encode('utf8')

        def getChild(self, name, request):
            return TestResource(name=name.decode('utf-8'))

    class PrepareResource(Resource):

        isLeaf = True

        def render_GET(self, request):
            gc.collect()
            return b'OK'

    class StopResource(Resource):

        isLeaf = True

        def render_GET(self, request):
            reactor.callLater(0.1, reactor.stop)
            return b'OK'

    root = Resource()
    root.putChild(b'test', TestResource(None))
    root.putChild(b'prepare', PrepareResource())
    root.putChild(b'stop', StopResource())
    site = Site(root)
    reactor.listenTCP(port, site, interface=host)
    barrier.wait()

    reactor.run()
Exemplo n.º 7
0
    def __init__(self, spidercls, settings=None, init_reactor: bool = False):
        if isinstance(spidercls, Spider):
            raise ValueError('The spidercls argument must be a class, not an object')

        if isinstance(settings, dict) or settings is None:
            settings = Settings(settings)

        self.spidercls = spidercls
        self.settings = settings.copy()
        self.spidercls.update_settings(self.settings)

        self.signals = SignalManager(self)

        self.stats = load_object(self.settings['STATS_CLASS'])(self)

        handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
        logging.root.addHandler(handler)

        d = dict(overridden_settings(self.settings))
        logger.info("Overridden settings:\n%(settings)s",
                    {'settings': pprint.pformat(d)})

        if get_scrapy_root_handler() is not None:
            # scrapy root handler already installed: update it with new settings
            install_scrapy_root_handler(self.settings)
        # lambda is assigned to Crawler attribute because this way it is not
        # garbage collected after leaving __init__ scope
        self.__remove_handler = lambda: logging.root.removeHandler(handler)
        self.signals.connect(self.__remove_handler, signals.engine_stopped)

        lf_cls = load_object(self.settings['LOG_FORMATTER'])
        self.logformatter = lf_cls.from_crawler(self)

        self.request_fingerprinter = create_instance(
            load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
            settings=self.settings,
            crawler=self,
        )

        reactor_class = self.settings.get("TWISTED_REACTOR")
        if init_reactor:
            # this needs to be done after the spider settings are merged,
            # but before something imports twisted.internet.reactor
            if reactor_class:
                install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
            else:
                from twisted.internet import default
                default.install()
            log_reactor_info()
        if reactor_class:
            verify_installed_reactor(reactor_class)

        self.extensions = ExtensionManager.from_crawler(self)

        self.settings.freeze()
        self.crawling = False
        self.spider = None
        self.engine = None
Exemplo n.º 8
0
def _get_reactor_impl():
    import inspect
    from unittest import mock
    from twisted.internet.default import install

    with mock.patch("twisted.internet.main.installReactor") as mock_install:
        install()
    reactor = mock_install.call_args[0][0]
    reactor_module_name = inspect.getmodule(reactor).__name__
    reactor_module = astroid.MANAGER.ast_from_module_name(reactor_module_name)
    return reactor_module[type(reactor).__name__].instantiate_class()
def execute_command(server_config, command, **kwargs):
    # We use lsof to determine before/after file descriptors
    if not which('lsof'):
            raise Exception('Could not find "lsof". Please make sure it is installed and is in PATH.')
    before_fds = _get_open_fds()  # see the comment in the finally clause below
    if isinstance(command, basestring):
        command = Command(command, **kwargs)
    timeout = 300 if command.name in ['stop', 'pull'] else 10
    factory = CommandExecutorFactory(server_config, command, timeout)

    # reactors aren't designed to be re-startable. In order to be
    # able to call execute_command multiple times, we need to froce
    # re-installation of the reactor; hence this hackery.
    # TODO: look into implementing restartable reactors. According to the
    #       Twisted FAQ, there is no good reason why there isn't one:
    #       http://twistedmatrix.com/trac/wiki/FrequentlyAskedQuestions#WhycanttheTwistedsreactorberestarted
    from twisted.internet import default
    del sys.modules['twisted.internet.reactor']
    default.install()
    global reactor  # pylint: disable=W0603
    reactor = sys.modules['twisted.internet.reactor']

    try:
        reactor.connectTCP(server_config.host, server_config.port, factory)
        reactor.run()
        return factory.result
    finally:
        # re-startable reactor hack part 2.
        # twisted hijacks SIGINT and doesn't bother to un-hijack it when the reactor
        # stops. So we have to do it for it *rolls eye*.
        import signal
        signal.signal(signal.SIGINT, signal.default_int_handler)
        # OK, the reactor is also leaking file descriptors. Tracking down all
        # of them is non trivial, so instead we're just comparing the before
        # and after lists of open FDs for the current process, and closing all
        # new ones, as execute_command should never leave anything open after
        # it exits (even when downloading data files from the server).
        # TODO: This is way too hacky even compared to the rest of this function.
        #       Additionally, the current implementation ties this to UNIX,
        #       so in the long run, we need to do this properly and get the FDs
        #       from the reactor.
        after_fds = _get_open_fds()
        for fd in after_fds - before_fds:
            try:
                os.close(int(fd[1:]))
            except OSError:
                pass
Exemplo n.º 10
0
def execute_command(server_config, command, **kwargs):
    before_fds = _get_open_fds()  # see the comment in the finally clause below
    if isinstance(command, basestring):
        command = Command(command, **kwargs)
    timeout = 300 if command.name in ['stop', 'pull'] else 10
    factory = CommandExecutorFactory(server_config, command, timeout)

    # reactors aren't designed to be re-startable. In order to be
    # able to call execute_command multiple times, we need to froce
    # re-installation of the reactor; hence this hackery.
    # TODO: look into implementing restartable reactors. According to the
    #       Twisted FAQ, there is no good reason why there isn't one:
    #       http://twistedmatrix.com/trac/wiki/FrequentlyAskedQuestions#WhycanttheTwistedsreactorberestarted
    from twisted.internet import default
    del sys.modules['twisted.internet.reactor']
    default.install()
    global reactor  # pylint: disable=W0603
    reactor = sys.modules['twisted.internet.reactor']

    try:
        reactor.connectTCP(server_config.host, server_config.port, factory)
        reactor.run()
        return factory.result
    finally:
        # re-startable reactor hack part 2.
        # twisted hijacks SIGINT and doesn't bother to un-hijack it when the reactor
        # stops. So we have to do it for it *rolls eye*.
        import signal
        signal.signal(signal.SIGINT, signal.default_int_handler)
        # OK, the reactor is also leaking file descriptors. Tracking down all
        # of them is non trivial, so instead we're just comparing the before
        # and after lists of open FDs for the current process, and closing all
        # new ones, as execute_command should never leave anything open after
        # it exits (even when downloading data files from the server).
        # TODO: This is way too hacky even compared to the rest of this function.
        #       Additionally, the current implementation ties this to UNIX,
        #       so in the long run, we need to do this properly and get the FDs
        #       from the reactor.
        after_fds = _get_open_fds()
        for fd in after_fds - before_fds:
            try:
                os.close(int(fd[1:]))
            except OSError:
                pass
Exemplo n.º 11
0
    def startReactor(self):
        """
        If C{self.reactor} is L{None}, install the default reactor and set
        C{self.reactor} to the default reactor.

        Register C{self.whenRunning} with the reactor so that it is called once
        the reactor is running, then start the reactor.
        """
        if self.reactor is None:
            defaultReactor.install()
            from twisted.internet import reactor
            self.reactor = reactor
        else:
            reactor = self.reactor

        reactor.callWhenRunning(self.whenRunning)

        self.log.info("Starting reactor...")
        reactor.run()
Exemplo n.º 12
0
    def startReactor(self):
        """
        If C{self.reactor} is L{None}, install the default reactor and set
        C{self.reactor} to the default reactor.

        Register C{self.whenRunning} with the reactor so that it is called once
        the reactor is running, then start the reactor.
        """
        if self.reactor is None:
            defaultReactor.install()
            from twisted.internet import reactor
            self.reactor = reactor
        else:
            reactor = self.reactor

        reactor.callWhenRunning(self.whenRunning)

        self.log.info("Starting reactor...")
        reactor.run()
Exemplo n.º 13
0
def fetchFreeProxy(num=500):
    global availables
    import sys

    del sys.modules['twisted.internet.reactor']
    from twisted.internet import reactor
    from twisted.internet import default

    default.install()

    res = requests.get(
        "http://www.89ip.cn/tqdl.html?api=1&num=%d&port=&address=&isp=" %
        (num))
    proxies = []

    # add new proxy into proxy list
    for proxy in re.findall("([0-9\.:]{10,})", res.content):
        proxies.append('http://' + proxy)

    # merge old available proxy into proxy list
    for item in availables:
        proxies.append(item['proxy'])

    # clean availables
    availables = []

    proxies = list(set(proxies))
    r = requests.session()

    url = "https://www.baidu.com/img/bd_logo1.png?where=super"

    deferred_list = []
    if True:
        for proxy in proxies:
            for i in range(0, 1):
                deferred = checkStatus(url=url, proxy=proxy, timeout=10)
                deferred.addCallback(callback)  # 请求返回后的回调函数
                deferred.addErrback(errback)
                deferred_list.append(deferred)  # 把所有的请求加到列表里,后面要检测
        dlist = defer.DeferredList(deferred_list)  # 检测所有的请求
        dlist.addBoth(lambda _: reactor.stop())  # 检测到所有请求都执行完,执行的方法
        reactor.run()
Exemplo n.º 14
0
def install_twisted_reactor():
    """
    Install the Twisted reactor designed for this platform.
    """

    import sys

    if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
         from twisted.internet import kqreactor
         kqreactor.install()
    elif sys.platform in ['win32']:
         from twisted.internet.iocpreactor import reactor as iocpreactor
         iocpreactor.install()
    elif sys.platform.startswith('linux'):
         from twisted.internet import epollreactor
         epollreactor.install()
    else:
         from twisted.internet import default as defaultreactor
         defaultreactor.install()

    from twisted.internet import reactor
    
    return reactor
Exemplo n.º 15
0
import sys
import time
import json
import logging
from functools import partial

try:
    # remove any already installed reactor
    del sys.modules['twisted.internet.reactor']
except KeyError: pass
# reinstall reactor
from twisted.internet.default import install
#from twisted.internet.pollreactor import install
#from twisted.internet.epollreactor import install
install()
del install

from twisted.internet import reactor
from twisted.internet.error import TimeoutError

# apply monkey patches for scrapy
from . import _monkeypatches
del _monkeypatches

from scrapy.spiders import Spider
from scrapy.exceptions import CloseSpider
from scrapy.settings import Settings

from scrapy.http import Request
from ..spamfilter import isspam_link, isspam_text
Exemplo n.º 16
0
 def installer():
     installed.append(True)
     return install()
Exemplo n.º 17
0
def install_optimal_reactor(verbose=False):
    """
    Try to install the optimal Twisted reactor for platform.

    :param verbose: If ``True``, print what happens.
    :type verbose: bool
    """
    import sys
    from twisted.python import reflect
    import txaio
    txaio.use_twisted()  # just to be sure...
    # XXX should I configure txaio.config.loop in here too, or just in
    # install_reactor()? (I am: see bottom of function)

    # determine currently installed reactor, if any
    ##
    if 'twisted.internet.reactor' in sys.modules:
        current_reactor = reflect.qual(
            sys.modules['twisted.internet.reactor'].__class__).split('.')[-1]
    else:
        current_reactor = None

    # depending on platform, install optimal reactor
    ##
    if 'bsd' in sys.platform or sys.platform.startswith('darwin'):

        # *BSD and MacOSX
        ##
        if current_reactor != 'KQueueReactor':
            try:
                v = sys.version_info
                if v[0] == 1 or (v[0] == 2
                                 and v[1] < 6) or (v[0] == 2 and v[1] == 6
                                                   and v[2] < 5):
                    raise Exception(
                        "Python version too old ({0}) to use kqueue reactor".
                        format(sys.version))
                from twisted.internet import kqreactor
                kqreactor.install()
            except Exception as e:
                print(
                    "WARNING: Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor ({0})."
                    .format(e))
            else:
                if verbose:
                    print(
                        "Running on *BSD or MacOSX and optimal reactor (kqueue) was installed."
                    )
        else:
            if verbose:
                print(
                    "Running on *BSD or MacOSX and optimal reactor (kqueue) already installed."
                )

    elif sys.platform in ['win32']:

        # Windows
        ##
        if current_reactor != 'IOCPReactor':
            try:
                from twisted.internet.iocpreactor import reactor as iocpreactor
                iocpreactor.install()
            except Exception as e:
                print(
                    "WARNING: Running on Windows, but cannot install IOCP Twisted reactor ({0})."
                    .format(e))
            else:
                if verbose:
                    print(
                        "Running on Windows and optimal reactor (ICOP) was installed."
                    )
        else:
            if verbose:
                print(
                    "Running on Windows and optimal reactor (ICOP) already installed."
                )

    elif sys.platform.startswith('linux'):

        # Linux
        ##
        if current_reactor != 'EPollReactor':
            try:
                from twisted.internet import epollreactor
                epollreactor.install()
            except Exception as e:
                print(
                    "WARNING: Running on Linux, but cannot install Epoll Twisted reactor ({0})."
                    .format(e))
            else:
                if verbose:
                    print(
                        "Running on Linux and optimal reactor (epoll) was installed."
                    )
        else:
            if verbose:
                print(
                    "Running on Linux and optimal reactor (epoll) already installed."
                )

    else:
        try:
            from twisted.internet import default as defaultreactor
            defaultreactor.install()
        except Exception as e:
            print(
                "WARNING: Could not install default Twisted reactor for this platform ({0})."
                .format(e))

    from twisted.internet import reactor
    txaio.config.loop = reactor
Exemplo n.º 18
0
def install_optimal_reactor(verbose=False):
    """
    Try to install the optimal Twisted reactor for this platform.

    :param verbose: If ``True``, print what happens.
    :type verbose: bool
    """
    log = make_logger()

    import sys
    from twisted.python import reflect
    import txaio
    txaio.use_twisted()  # just to be sure...
    # XXX should I configure txaio.config.loop in here too, or just in
    # install_reactor()? (I am: see bottom of function)

    # determine currently installed reactor, if any
    ##
    if 'twisted.internet.reactor' in sys.modules:
        current_reactor = reflect.qual(
            sys.modules['twisted.internet.reactor'].__class__).split('.')[-1]
    else:
        current_reactor = None

    # depending on platform, install optimal reactor
    ##
    if 'bsd' in sys.platform or sys.platform.startswith('darwin'):

        # *BSD and MacOSX
        ##
        if current_reactor != 'KQueueReactor':
            try:
                from twisted.internet import kqreactor
                kqreactor.install()
            except:
                log.critical(
                    "Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor"
                )
                log.debug(traceback.format_exc())
            else:
                log.debug(
                    "Running on *BSD or MacOSX and optimal reactor (kqueue) was installed."
                )
        else:
            log.debug(
                "Running on *BSD or MacOSX and optimal reactor (kqueue) already installed."
            )

    elif sys.platform in ['win32']:

        # Windows
        ##
        if current_reactor != 'IOCPReactor':
            try:
                from twisted.internet.iocpreactor import reactor as iocpreactor
                iocpreactor.install()
            except:
                log.critical(
                    "Running on Windows, but cannot install IOCP Twisted reactor"
                )
                log.debug(traceback.format_exc())
            else:
                log.debug(
                    "Running on Windows and optimal reactor (ICOP) was installed."
                )
        else:
            log.debug(
                "Running on Windows and optimal reactor (ICOP) already installed."
            )

    elif sys.platform.startswith('linux'):

        # Linux
        ##
        if current_reactor != 'EPollReactor':
            try:
                from twisted.internet import epollreactor
                epollreactor.install()
            except:
                log.critical(
                    "Running on Linux, but cannot install Epoll Twisted reactor"
                )
                log.debug(traceback.format_exc())
            else:
                log.debug(
                    "Running on Linux and optimal reactor (epoll) was installed."
                )
        else:
            log.debug(
                "Running on Linux and optimal reactor (epoll) already installed."
            )

    else:
        try:
            from twisted.internet import default as defaultreactor
            defaultreactor.install()
        except:
            log.critical(
                "Could not install default Twisted reactor for this platform")
            log.debug(traceback.format_exc())

    from twisted.internet import reactor
    txaio.config.loop = reactor
Exemplo n.º 19
0
def install_optimal_reactor(verbose=False):
    """
    Try to install the optimal Twisted reactor for this platform.

    :param verbose: If ``True``, print what happens.
    :type verbose: bool
    """
    log = make_logger()

    import sys
    from twisted.python import reflect
    import txaio
    txaio.use_twisted()  # just to be sure...
    # XXX should I configure txaio.config.loop in here too, or just in
    # install_reactor()? (I am: see bottom of function)

    # determine currently installed reactor, if any
    ##
    if 'twisted.internet.reactor' in sys.modules:
        current_reactor = reflect.qual(sys.modules['twisted.internet.reactor'].__class__).split('.')[-1]
    else:
        current_reactor = None

    # depending on platform, install optimal reactor
    ##
    if 'bsd' in sys.platform or sys.platform.startswith('darwin'):

        # *BSD and MacOSX
        ##
        if current_reactor != 'KQueueReactor':
            try:
                from twisted.internet import kqreactor
                kqreactor.install()
            except:
                log.critical("Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor")
                log.debug(traceback.format_exc())
            else:
                log.debug("Running on *BSD or MacOSX and optimal reactor (kqueue) was installed.")
        else:
            log.debug("Running on *BSD or MacOSX and optimal reactor (kqueue) already installed.")

    elif sys.platform in ['win32']:

        # Windows
        ##
        if current_reactor != 'IOCPReactor':
            try:
                from twisted.internet.iocpreactor import reactor as iocpreactor
                iocpreactor.install()
            except:
                log.critical("Running on Windows, but cannot install IOCP Twisted reactor")
                log.debug(traceback.format_exc())
            else:
                log.debug("Running on Windows and optimal reactor (ICOP) was installed.")
        else:
            log.debug("Running on Windows and optimal reactor (ICOP) already installed.")

    elif sys.platform.startswith('linux'):

        # Linux
        ##
        if current_reactor != 'EPollReactor':
            try:
                from twisted.internet import epollreactor
                epollreactor.install()
            except:
                log.critical("Running on Linux, but cannot install Epoll Twisted reactor")
                log.debug(traceback.format_exc())
            else:
                log.debug("Running on Linux and optimal reactor (epoll) was installed.")
        else:
            log.debug("Running on Linux and optimal reactor (epoll) already installed.")

    else:
        try:
            from twisted.internet import default as defaultreactor
            defaultreactor.install()
        except:
            log.critical("Could not install default Twisted reactor for this platform")
            log.debug(traceback.format_exc())

    from twisted.internet import reactor
    txaio.config.loop = reactor
Exemplo n.º 20
0
import sys
import ujson

if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
    from twisted.internet import kqreactor
    kqreactor.install()
elif sys.platform in ['win32']:
    from twisted.internet.iocpreactor import reactor as iocpreactor
    iocpreactor.install()
elif sys.platform.startswith('linux'):
    from twisted.internet import epollreactor
    epollreactor.install()
else:
    from twisted.internet import default as defaultreactor
    defaultreactor.install()

from twisted.web.server import Site  # noqa
from twisted.web.resource import Resource  # noqa
from twisted.internet import reactor  # noqa


class TestResource(Resource):
    def render_GET(self, request):
        body = ujson.dumps({'test': True}).encode('utf8')
        request.setHeader(b'Content-Type', b'application/json; charset=utf-8')
        return body

    def getChild(self, path, request):
        if path == b'payload':
            return Payload()
Exemplo n.º 21
0
beforehand. Regardless of which reactor is installed, importing this module is
the correct way to get a reference to it.

New application code should prefer to pass and accept the reactor as a
parameter where it is needed, rather than relying on being able to import this
module to get a reference.  This simplifies unit testing and may make it easier
to one day support multiple reactors (as a performance enhancement), though
this is not currently possible.

@see: L{IReactorCore<twisted.internet.interfaces.IReactorCore>}
@see: L{IReactorTime<twisted.internet.interfaces.IReactorTime>}
@see: L{IReactorProcess<twisted.internet.interfaces.IReactorProcess>}
@see: L{IReactorTCP<twisted.internet.interfaces.IReactorTCP>}
@see: L{IReactorSSL<twisted.internet.interfaces.IReactorSSL>}
@see: L{IReactorUDP<twisted.internet.interfaces.IReactorUDP>}
@see: L{IReactorMulticast<twisted.internet.interfaces.IReactorMulticast>}
@see: L{IReactorUNIX<twisted.internet.interfaces.IReactorUNIX>}
@see: L{IReactorUNIXDatagram<twisted.internet.interfaces.IReactorUNIXDatagram>}
@see: L{IReactorFDSet<twisted.internet.interfaces.IReactorFDSet>}
@see: L{IReactorThreads<twisted.internet.interfaces.IReactorThreads>}
@see: L{IReactorArbitrary<twisted.internet.interfaces.IReactorArbitrary>}
@see: L{IReactorPluggableResolver<twisted.internet.interfaces.IReactorPluggableResolver>}
"""

import sys

del sys.modules["twisted.internet.reactor"]
from twisted.internet import default

default.install()
Exemplo n.º 22
0
# https://github.com/pyinstaller/pyinstaller/issues/3390

import sys

# this creates module: sys.modules['twisted.internet.reactor']
if sys.platform in ['win32']:
    from twisted.internet.iocpreactor import reactor as _iocpreactor
    _iocpreactor.install()
else:
    from twisted.internet import default
    default.install()
Exemplo n.º 23
0
def install_optimal_reactor(verbose=False):
    """
   Try to install the optimal Twisted reactor for platform.

   :param verbose: If ``True``, print what happens.
   :type verbose: bool
   """
    import sys
    from twisted.python import reflect
    from twisted.python import log

    ## determine currently installed reactor, if any
    ##
    if "twisted.internet.reactor" in sys.modules:
        current_reactor = reflect.qual(sys.modules["twisted.internet.reactor"].__class__).split(".")[-1]
    else:
        current_reactor = None

    ## depending on platform, install optimal reactor
    ##
    if "bsd" in sys.platform or sys.platform.startswith("darwin"):

        ## *BSD and MacOSX
        ##
        if current_reactor != "KQueueReactor":
            try:
                v = sys.version_info
                if v[0] == 1 or (v[0] == 2 and v[1] < 6) or (v[0] == 2 and v[1] == 6 and v[2] < 5):
                    raise Exception("Python version too old ({0}) to use kqueue reactor".format(sys.version))
                from twisted.internet import kqreactor

                kqreactor.install()
            except Exception as e:
                log.err(
                    "WARNING: Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor ({0}).".format(e)
                )
            else:
                if verbose:
                    log.msg("Running on *BSD or MacOSX and optimal reactor (kqueue) was installed.")
        else:
            if verbose:
                log.msg("Running on *BSD or MacOSX and optimal reactor (kqueue) already installed.")

    elif sys.platform in ["win32"]:

        ## Windows
        ##
        if current_reactor != "IOCPReactor":
            try:
                from twisted.internet.iocpreactor import reactor as iocpreactor

                iocpreactor.install()
            except Exception as e:
                log.err("WARNING: Running on Windows, but cannot install IOCP Twisted reactor ({0}).".format(e))
            else:
                if verbose:
                    log.msg("Running on Windows and optimal reactor (ICOP) was installed.")
        else:
            if verbose:
                log.msg("Running on Windows and optimal reactor (ICOP) already installed.")

    elif sys.platform.startswith("linux"):

        ## Linux
        ##
        if current_reactor != "EPollReactor":
            try:
                from twisted.internet import epollreactor

                epollreactor.install()
            except Exception as e:
                log.err("WARNING: Running on Linux, but cannot install Epoll Twisted reactor ({0}).".format(e))
            else:
                if verbose:
                    log.msg("Running on Linux and optimal reactor (epoll) was installed.")
        else:
            if verbose:
                log.msg("Running on Linux and optimal reactor (epoll) already installed.")

    else:
        try:
            from twisted.internet import default as defaultreactor

            defaultreactor.install()
        except Exception as e:
            log.err("WARNING: Could not install default Twisted reactor for this platform ({0}).".format(e))
Exemplo n.º 24
0
 def installer():
     installed.append(True)
     return install()
Exemplo n.º 25
0
import sys
import time
import json
import logging
from functools import partial

try:
    # remove any already installed reactor
    del sys.modules['twisted.internet.reactor']
except KeyError:
    pass
# reinstall reactor
from twisted.internet.default import install
#from twisted.internet.pollreactor import install
#from twisted.internet.epollreactor import install
install()
del install

from twisted.internet import reactor
from twisted.internet.error import TimeoutError

# apply monkey patches for scrapy
from . import _monkeypatches
del _monkeypatches

from scrapy.spiders import Spider
from scrapy.exceptions import CloseSpider
from scrapy.settings import Settings

from scrapy.http import Request
from ..spamfilter import isspam_link, isspam_text
Exemplo n.º 26
0
def install_optimal_reactor(verbose=False):
    """
    Try to install the optimal Twisted reactor for this platform.

    :param verbose: If ``True``, print what happens.
    :type verbose: bool
    """
    log = txaio.make_logger()

    import sys
    from twisted.python import reflect

    # determine currently installed reactor, if any
    ##
    if "twisted.internet.reactor" in sys.modules:
        current_reactor = reflect.qual(sys.modules["twisted.internet.reactor"].__class__).split(".")[-1]
    else:
        current_reactor = None

    # depending on platform, install optimal reactor
    ##
    if "bsd" in sys.platform or sys.platform.startswith("darwin"):

        # *BSD and MacOSX
        ##
        if current_reactor != "KQueueReactor":
            try:
                from twisted.internet import kqreactor

                kqreactor.install()
            except:
                log.critical("Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor")
                log.warn("{tb}", tb=traceback.format_exc())
            else:
                log.debug("Running on *BSD or MacOSX and optimal reactor (kqueue) was installed.")
        else:
            log.debug("Running on *BSD or MacOSX and optimal reactor (kqueue) already installed.")

    elif sys.platform in ["win32"]:

        # Windows
        ##
        if current_reactor != "IOCPReactor":
            try:
                from twisted.internet.iocpreactor import reactor as iocpreactor

                iocpreactor.install()
            except:
                log.critical("Running on Windows, but cannot install IOCP Twisted reactor")
                log.warn("{tb}", tb=traceback.format_exc())
            else:
                log.debug("Running on Windows and optimal reactor (ICOP) was installed.")
        else:
            log.debug("Running on Windows and optimal reactor (ICOP) already installed.")

    elif sys.platform.startswith("linux"):

        # Linux
        ##
        if current_reactor != "EPollReactor":
            try:
                from twisted.internet import epollreactor

                epollreactor.install()
            except:
                log.critical("Running on Linux, but cannot install Epoll Twisted reactor")
                log.warn("{tb}", tb=traceback.format_exc())
            else:
                log.debug("Running on Linux and optimal reactor (epoll) was installed.")
        else:
            log.debug("Running on Linux and optimal reactor (epoll) already installed.")

    else:
        try:
            from twisted.internet import default as defaultreactor

            defaultreactor.install()
        except:
            log.critical("Could not install default Twisted reactor for this platform")
            log.warn("{tb}", tb=traceback.format_exc())

    from twisted.internet import reactor

    txaio.config.loop = reactor
Exemplo n.º 27
0
def install_optimal_reactor(verbose=False):
    """
    Try to install the optimal Twisted reactor for platform.

    :param verbose: If ``True``, print what happens.
    :type verbose: bool
    """
    import sys
    from twisted.python import reflect
    import txaio
    txaio.use_twisted()  # just to be sure...
    # XXX should I configure txaio.config.loop in here too, or just in
    # install_reactor()? (I am: see bottom of function)

    # determine currently installed reactor, if any
    ##
    if 'twisted.internet.reactor' in sys.modules:
        current_reactor = reflect.qual(sys.modules['twisted.internet.reactor'].__class__).split('.')[-1]
    else:
        current_reactor = None

    # depending on platform, install optimal reactor
    ##
    if 'bsd' in sys.platform or sys.platform.startswith('darwin'):

        # *BSD and MacOSX
        ##
        if current_reactor != 'KQueueReactor':
            try:
                v = sys.version_info
                if v[0] == 1 or (v[0] == 2 and v[1] < 6) or (v[0] == 2 and v[1] == 6 and v[2] < 5):
                    raise Exception("Python version too old ({0}) to use kqueue reactor".format(sys.version))
                from twisted.internet import kqreactor
                kqreactor.install()
            except Exception as e:
                print("WARNING: Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor ({0}).".format(e))
            else:
                if verbose:
                    print("Running on *BSD or MacOSX and optimal reactor (kqueue) was installed.")
        else:
            if verbose:
                print("Running on *BSD or MacOSX and optimal reactor (kqueue) already installed.")

    elif sys.platform in ['win32']:

        # Windows
        ##
        if current_reactor != 'IOCPReactor':
            try:
                from twisted.internet.iocpreactor import reactor as iocpreactor
                iocpreactor.install()
            except Exception as e:
                print("WARNING: Running on Windows, but cannot install IOCP Twisted reactor ({0}).".format(e))
            else:
                if verbose:
                    print("Running on Windows and optimal reactor (ICOP) was installed.")
        else:
            if verbose:
                print("Running on Windows and optimal reactor (ICOP) already installed.")

    elif sys.platform.startswith('linux'):

        # Linux
        ##
        if current_reactor != 'EPollReactor':
            try:
                from twisted.internet import epollreactor
                epollreactor.install()
            except Exception as e:
                print("WARNING: Running on Linux, but cannot install Epoll Twisted reactor ({0}).".format(e))
            else:
                if verbose:
                    print("Running on Linux and optimal reactor (epoll) was installed.")
        else:
            if verbose:
                print("Running on Linux and optimal reactor (epoll) already installed.")

    else:
        try:
            from twisted.internet import default as defaultreactor
            defaultreactor.install()
        except Exception as e:
            print("WARNING: Could not install default Twisted reactor for this platform ({0}).".format(e))

    from twisted.internet import reactor
    txaio.config.loop = reactor