def install_optimal_reactor(verbose = False):
"""
Try to install the optimal Twisted reactor for platform.
:param verbose: If ``True``, print what happens.
:type verbose: bool
"""
import sys
from twisted.python import reflect
if 'twisted.internet.reactor' in sys.modules:
current_reactor = reflect.qual(sys.modules['twisted.internet.reactor'].__class__).split('.')[-1]
else:
current_reactor = None
if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
if current_reactor != 'KQueueReactor':
try:
v = sys.version_info
if v[0] == 1 or v[0] == 2 and v[1] < 6 or v[0] == 2 and v[1] == 6 and v[2] < 5:
raise Exception('Python version too old ({0}) to use kqueue reactor'.format(sys.version))
from twisted.internet import kqreactor
kqreactor.install()
except Exception as e:
print 'WARNING: Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor ({0}).'.format(e)
else:
if verbose:
print 'Running on *BSD or MacOSX and optimal reactor (kqueue) was installed.'
elif verbose:
print 'Running on *BSD or MacOSX and optimal reactor (kqueue) already installed.'
elif sys.platform in ('win32',):
if current_reactor != 'IOCPReactor':
try:
from twisted.internet.iocpreactor import reactor as iocpreactor
iocpreactor.install()
except Exception as e:
print 'WARNING: Running on Windows, but cannot install IOCP Twisted reactor ({0}).'.format(e)
else:
if verbose:
print 'Running on Windows and optimal reactor (ICOP) was installed.'
elif verbose:
print 'Running on Windows and optimal reactor (ICOP) already installed.'
elif sys.platform.startswith('linux'):
if current_reactor != 'EPollReactor':
try:
from twisted.internet import epollreactor
epollreactor.install()
except Exception as e:
print 'WARNING: Running on Linux, but cannot install Epoll Twisted reactor ({0}).'.format(e)
else:
if verbose:
print 'Running on Linux and optimal reactor (epoll) was installed.'
elif verbose:
print 'Running on Linux and optimal reactor (epoll) already installed.'
else:
try:
from twisted.internet import default as defaultreactor
defaultreactor.install()
except Exception as e:
print 'WARNING: Could not install default Twisted reactor for this platform ({0}).'.format(e)
return
def test_install(self):
"""
L{install} installs a reactor.
"""
with NoReactor():
install()
self.assertIn("twisted.internet.reactor", sys.modules)
def engine_stopped(self):
logger.error("SpiderRunner engine_stopped")
try:
if self.spider.spider and hasattr(self.spider.spider,
'engine_stopped'):
self.spider.spider.engine_stopped()
self.handler.parent.writeLog(
self, str("---- Spider %s engine_stopped ----" % self.name))
self.management_info['current_endtime'] = datetime.datetime.now(
).strftime("%Y%m%d%H%M%S")
self.is_running = False
self.management_info['is_running'] = False
running_spiders = [
x for x in self.handler.spider_runners if x.spider.crawling
]
if len(running_spiders) == 0:
from twisted.internet import reactor
if reactor.running:
reactor.stop()
import sys
del sys.modules['twisted.internet.reactor']
from twisted.internet import reactor
default.install()
pass
except Exception as ex:
logger.error("SpiderRunner Exception : %s", str(ex))
def test_install(self):
"""
L{install} installs a reactor.
"""
with NoReactor():
install()
self.assertIn("twisted.internet.reactor", sys.modules)
def run_twisted(host, port, barrier, profile):
if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
from twisted.internet import kqreactor
kqreactor.install()
elif sys.platform in ['win32']:
from twisted.internet.iocpreactor import reactor as iocpreactor
iocpreactor.install()
elif sys.platform.startswith('linux'):
from twisted.internet import epollreactor
epollreactor.install()
else:
from twisted.internet import default as defaultreactor
defaultreactor.install()
from twisted.web.server import Site
from twisted.web.resource import Resource
from twisted.internet import reactor
class TestResource(Resource):
def __init__(self, name):
super().__init__()
self.name = name
self.isLeaf = name is not None
def render_GET(self, request):
txt = 'Hello, ' + self.name
request.setHeader(b'Content-Type', b'text/plain; charset=utf-8')
return txt.encode('utf8')
def getChild(self, name, request):
return TestResource(name=name.decode('utf-8'))
class PrepareResource(Resource):
isLeaf = True
def render_GET(self, request):
gc.collect()
return b'OK'
class StopResource(Resource):
isLeaf = True
def render_GET(self, request):
reactor.callLater(0.1, reactor.stop)
return b'OK'
root = Resource()
root.putChild(b'test', TestResource(None))
root.putChild(b'prepare', PrepareResource())
root.putChild(b'stop', StopResource())
site = Site(root)
reactor.listenTCP(port, site, interface=host)
barrier.wait()
reactor.run()
def run_twisted(host, port, barrier, profile):
if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
from twisted.internet import kqreactor
kqreactor.install()
elif sys.platform in ['win32']:
from twisted.internet.iocpreactor import reactor as iocpreactor
iocpreactor.install()
elif sys.platform.startswith('linux'):
from twisted.internet import epollreactor
epollreactor.install()
else:
from twisted.internet import default as defaultreactor
defaultreactor.install()
from twisted.web.server import Site
from twisted.web.resource import Resource
from twisted.internet import reactor
class TestResource(Resource):
def __init__(self, name):
super().__init__()
self.name = name
self.isLeaf = name is not None
def render_GET(self, request):
txt = 'Hello, ' + self.name
request.setHeader(b'Content-Type', b'text/plain; charset=utf-8')
return txt.encode('utf8')
def getChild(self, name, request):
return TestResource(name=name.decode('utf-8'))
class PrepareResource(Resource):
isLeaf = True
def render_GET(self, request):
gc.collect()
return b'OK'
class StopResource(Resource):
isLeaf = True
def render_GET(self, request):
reactor.callLater(0.1, reactor.stop)
return b'OK'
root = Resource()
root.putChild(b'test', TestResource(None))
root.putChild(b'prepare', PrepareResource())
root.putChild(b'stop', StopResource())
site = Site(root)
reactor.listenTCP(port, site, interface=host)
barrier.wait()
reactor.run()
def __init__(self, spidercls, settings=None, init_reactor: bool = False):
if isinstance(spidercls, Spider):
raise ValueError('The spidercls argument must be a class, not an object')
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.spidercls = spidercls
self.settings = settings.copy()
self.spidercls.update_settings(self.settings)
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
d = dict(overridden_settings(self.settings))
logger.info("Overridden settings:\n%(settings)s",
{'settings': pprint.pformat(d)})
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
install_scrapy_root_handler(self.settings)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
self.__remove_handler = lambda: logging.root.removeHandler(handler)
self.signals.connect(self.__remove_handler, signals.engine_stopped)
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.request_fingerprinter = create_instance(
load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
settings=self.settings,
crawler=self,
)
reactor_class = self.settings.get("TWISTED_REACTOR")
if init_reactor:
# this needs to be done after the spider settings are merged,
# but before something imports twisted.internet.reactor
if reactor_class:
install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
else:
from twisted.internet import default
default.install()
log_reactor_info()
if reactor_class:
verify_installed_reactor(reactor_class)
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
self.crawling = False
self.spider = None
self.engine = None
def _get_reactor_impl():
import inspect
from unittest import mock
from twisted.internet.default import install
with mock.patch("twisted.internet.main.installReactor") as mock_install:
install()
reactor = mock_install.call_args[0][0]
reactor_module_name = inspect.getmodule(reactor).__name__
reactor_module = astroid.MANAGER.ast_from_module_name(reactor_module_name)
return reactor_module[type(reactor).__name__].instantiate_class()
def execute_command(server_config, command, **kwargs):
# We use lsof to determine before/after file descriptors
if not which('lsof'):
raise Exception('Could not find "lsof". Please make sure it is installed and is in PATH.')
before_fds = _get_open_fds() # see the comment in the finally clause below
if isinstance(command, basestring):
command = Command(command, **kwargs)
timeout = 300 if command.name in ['stop', 'pull'] else 10
factory = CommandExecutorFactory(server_config, command, timeout)
# reactors aren't designed to be re-startable. In order to be
# able to call execute_command multiple times, we need to froce
# re-installation of the reactor; hence this hackery.
# TODO: look into implementing restartable reactors. According to the
# Twisted FAQ, there is no good reason why there isn't one:
# http://twistedmatrix.com/trac/wiki/FrequentlyAskedQuestions#WhycanttheTwistedsreactorberestarted
from twisted.internet import default
del sys.modules['twisted.internet.reactor']
default.install()
global reactor # pylint: disable=W0603
reactor = sys.modules['twisted.internet.reactor']
try:
reactor.connectTCP(server_config.host, server_config.port, factory)
reactor.run()
return factory.result
finally:
# re-startable reactor hack part 2.
# twisted hijacks SIGINT and doesn't bother to un-hijack it when the reactor
# stops. So we have to do it for it *rolls eye*.
import signal
signal.signal(signal.SIGINT, signal.default_int_handler)
# OK, the reactor is also leaking file descriptors. Tracking down all
# of them is non trivial, so instead we're just comparing the before
# and after lists of open FDs for the current process, and closing all
# new ones, as execute_command should never leave anything open after
# it exits (even when downloading data files from the server).
# TODO: This is way too hacky even compared to the rest of this function.
# Additionally, the current implementation ties this to UNIX,
# so in the long run, we need to do this properly and get the FDs
# from the reactor.
after_fds = _get_open_fds()
for fd in after_fds - before_fds:
try:
os.close(int(fd[1:]))
except OSError:
pass
def execute_command(server_config, command, **kwargs):
before_fds = _get_open_fds() # see the comment in the finally clause below
if isinstance(command, basestring):
command = Command(command, **kwargs)
timeout = 300 if command.name in ['stop', 'pull'] else 10
factory = CommandExecutorFactory(server_config, command, timeout)
# reactors aren't designed to be re-startable. In order to be
# able to call execute_command multiple times, we need to froce
# re-installation of the reactor; hence this hackery.
# TODO: look into implementing restartable reactors. According to the
# Twisted FAQ, there is no good reason why there isn't one:
# http://twistedmatrix.com/trac/wiki/FrequentlyAskedQuestions#WhycanttheTwistedsreactorberestarted
from twisted.internet import default
del sys.modules['twisted.internet.reactor']
default.install()
global reactor # pylint: disable=W0603
reactor = sys.modules['twisted.internet.reactor']
try:
reactor.connectTCP(server_config.host, server_config.port, factory)
reactor.run()
return factory.result
finally:
# re-startable reactor hack part 2.
# twisted hijacks SIGINT and doesn't bother to un-hijack it when the reactor
# stops. So we have to do it for it *rolls eye*.
import signal
signal.signal(signal.SIGINT, signal.default_int_handler)
# OK, the reactor is also leaking file descriptors. Tracking down all
# of them is non trivial, so instead we're just comparing the before
# and after lists of open FDs for the current process, and closing all
# new ones, as execute_command should never leave anything open after
# it exits (even when downloading data files from the server).
# TODO: This is way too hacky even compared to the rest of this function.
# Additionally, the current implementation ties this to UNIX,
# so in the long run, we need to do this properly and get the FDs
# from the reactor.
after_fds = _get_open_fds()
for fd in after_fds - before_fds:
try:
os.close(int(fd[1:]))
except OSError:
pass
def startReactor(self):
"""
If C{self.reactor} is L{None}, install the default reactor and set
C{self.reactor} to the default reactor.
Register C{self.whenRunning} with the reactor so that it is called once
the reactor is running, then start the reactor.
"""
if self.reactor is None:
defaultReactor.install()
from twisted.internet import reactor
self.reactor = reactor
else:
reactor = self.reactor
reactor.callWhenRunning(self.whenRunning)
self.log.info("Starting reactor...")
reactor.run()
def startReactor(self):
"""
If C{self.reactor} is L{None}, install the default reactor and set
C{self.reactor} to the default reactor.
Register C{self.whenRunning} with the reactor so that it is called once
the reactor is running, then start the reactor.
"""
if self.reactor is None:
defaultReactor.install()
from twisted.internet import reactor
self.reactor = reactor
else:
reactor = self.reactor
reactor.callWhenRunning(self.whenRunning)
self.log.info("Starting reactor...")
reactor.run()
def fetchFreeProxy(num=500):
global availables
import sys
del sys.modules['twisted.internet.reactor']
from twisted.internet import reactor
from twisted.internet import default
default.install()
res = requests.get(
"http://www.89ip.cn/tqdl.html?api=1&num=%d&port=&address=&isp=" %
(num))
proxies = []
# add new proxy into proxy list
for proxy in re.findall("([0-9\.:]{10,})", res.content):
proxies.append('http://' + proxy)
# merge old available proxy into proxy list
for item in availables:
proxies.append(item['proxy'])
# clean availables
availables = []
proxies = list(set(proxies))
r = requests.session()
url = "https://www.baidu.com/img/bd_logo1.png?where=super"
deferred_list = []
if True:
for proxy in proxies:
for i in range(0, 1):
deferred = checkStatus(url=url, proxy=proxy, timeout=10)
deferred.addCallback(callback) # 请求返回后的回调函数
deferred.addErrback(errback)
deferred_list.append(deferred) # 把所有的请求加到列表里，后面要检测
dlist = defer.DeferredList(deferred_list) # 检测所有的请求
dlist.addBoth(lambda _: reactor.stop()) # 检测到所有请求都执行完，执行的方法
reactor.run()
def install_twisted_reactor():
"""
Install the Twisted reactor designed for this platform.
"""
import sys
if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
from twisted.internet import kqreactor
kqreactor.install()
elif sys.platform in ['win32']:
from twisted.internet.iocpreactor import reactor as iocpreactor
iocpreactor.install()
elif sys.platform.startswith('linux'):
from twisted.internet import epollreactor
epollreactor.install()
else:
from twisted.internet import default as defaultreactor
defaultreactor.install()
from twisted.internet import reactor
return reactor
import sys
import time
import json
import logging
from functools import partial
try:
# remove any already installed reactor
del sys.modules['twisted.internet.reactor']
except KeyError: pass
# reinstall reactor
from twisted.internet.default import install
#from twisted.internet.pollreactor import install
#from twisted.internet.epollreactor import install
install()
del install
from twisted.internet import reactor
from twisted.internet.error import TimeoutError
# apply monkey patches for scrapy
from . import _monkeypatches
del _monkeypatches
from scrapy.spiders import Spider
from scrapy.exceptions import CloseSpider
from scrapy.settings import Settings
from scrapy.http import Request
from ..spamfilter import isspam_link, isspam_text
def installer():
installed.append(True)
return install()
def install_optimal_reactor(verbose=False):
"""
Try to install the optimal Twisted reactor for platform.
:param verbose: If ``True``, print what happens.
:type verbose: bool
"""
import sys
from twisted.python import reflect
import txaio
txaio.use_twisted() # just to be sure...
# XXX should I configure txaio.config.loop in here too, or just in
# install_reactor()? (I am: see bottom of function)
# determine currently installed reactor, if any
##
if 'twisted.internet.reactor' in sys.modules:
current_reactor = reflect.qual(
sys.modules['twisted.internet.reactor'].__class__).split('.')[-1]
else:
current_reactor = None
# depending on platform, install optimal reactor
##
if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
# *BSD and MacOSX
##
if current_reactor != 'KQueueReactor':
try:
v = sys.version_info
if v[0] == 1 or (v[0] == 2
and v[1] < 6) or (v[0] == 2 and v[1] == 6
and v[2] < 5):
raise Exception(
"Python version too old ({0}) to use kqueue reactor".
format(sys.version))
from twisted.internet import kqreactor
kqreactor.install()
except Exception as e:
print(
"WARNING: Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor ({0})."
.format(e))
else:
if verbose:
print(
"Running on *BSD or MacOSX and optimal reactor (kqueue) was installed."
)
else:
if verbose:
print(
"Running on *BSD or MacOSX and optimal reactor (kqueue) already installed."
)
elif sys.platform in ['win32']:
# Windows
##
if current_reactor != 'IOCPReactor':
try:
from twisted.internet.iocpreactor import reactor as iocpreactor
iocpreactor.install()
except Exception as e:
print(
"WARNING: Running on Windows, but cannot install IOCP Twisted reactor ({0})."
.format(e))
else:
if verbose:
print(
"Running on Windows and optimal reactor (ICOP) was installed."
)
else:
if verbose:
print(
"Running on Windows and optimal reactor (ICOP) already installed."
)
elif sys.platform.startswith('linux'):
# Linux
##
if current_reactor != 'EPollReactor':
try:
from twisted.internet import epollreactor
epollreactor.install()
except Exception as e:
print(
"WARNING: Running on Linux, but cannot install Epoll Twisted reactor ({0})."
.format(e))
else:
if verbose:
print(
"Running on Linux and optimal reactor (epoll) was installed."
)
else:
if verbose:
print(
"Running on Linux and optimal reactor (epoll) already installed."
)
else:
try:
from twisted.internet import default as defaultreactor
defaultreactor.install()
except Exception as e:
print(
"WARNING: Could not install default Twisted reactor for this platform ({0})."
.format(e))
from twisted.internet import reactor
txaio.config.loop = reactor
def install_optimal_reactor(verbose=False):
"""
Try to install the optimal Twisted reactor for this platform.
:param verbose: If ``True``, print what happens.
:type verbose: bool
"""
log = make_logger()
import sys
from twisted.python import reflect
import txaio
txaio.use_twisted() # just to be sure...
# XXX should I configure txaio.config.loop in here too, or just in
# install_reactor()? (I am: see bottom of function)
# determine currently installed reactor, if any
##
if 'twisted.internet.reactor' in sys.modules:
current_reactor = reflect.qual(
sys.modules['twisted.internet.reactor'].__class__).split('.')[-1]
else:
current_reactor = None
# depending on platform, install optimal reactor
##
if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
# *BSD and MacOSX
##
if current_reactor != 'KQueueReactor':
try:
from twisted.internet import kqreactor
kqreactor.install()
except:
log.critical(
"Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor"
)
log.debug(traceback.format_exc())
else:
log.debug(
"Running on *BSD or MacOSX and optimal reactor (kqueue) was installed."
)
else:
log.debug(
"Running on *BSD or MacOSX and optimal reactor (kqueue) already installed."
)
elif sys.platform in ['win32']:
# Windows
##
if current_reactor != 'IOCPReactor':
try:
from twisted.internet.iocpreactor import reactor as iocpreactor
iocpreactor.install()
except:
log.critical(
"Running on Windows, but cannot install IOCP Twisted reactor"
)
log.debug(traceback.format_exc())
else:
log.debug(
"Running on Windows and optimal reactor (ICOP) was installed."
)
else:
log.debug(
"Running on Windows and optimal reactor (ICOP) already installed."
)
elif sys.platform.startswith('linux'):
# Linux
##
if current_reactor != 'EPollReactor':
try:
from twisted.internet import epollreactor
epollreactor.install()
except:
log.critical(
"Running on Linux, but cannot install Epoll Twisted reactor"
)
log.debug(traceback.format_exc())
else:
log.debug(
"Running on Linux and optimal reactor (epoll) was installed."
)
else:
log.debug(
"Running on Linux and optimal reactor (epoll) already installed."
)
else:
try:
from twisted.internet import default as defaultreactor
defaultreactor.install()
except:
log.critical(
"Could not install default Twisted reactor for this platform")
log.debug(traceback.format_exc())
from twisted.internet import reactor
txaio.config.loop = reactor
def install_optimal_reactor(verbose=False):
"""
Try to install the optimal Twisted reactor for this platform.
:param verbose: If ``True``, print what happens.
:type verbose: bool
"""
log = make_logger()
import sys
from twisted.python import reflect
import txaio
txaio.use_twisted() # just to be sure...
# XXX should I configure txaio.config.loop in here too, or just in
# install_reactor()? (I am: see bottom of function)
# determine currently installed reactor, if any
##
if 'twisted.internet.reactor' in sys.modules:
current_reactor = reflect.qual(sys.modules['twisted.internet.reactor'].__class__).split('.')[-1]
else:
current_reactor = None
# depending on platform, install optimal reactor
##
if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
# *BSD and MacOSX
##
if current_reactor != 'KQueueReactor':
try:
from twisted.internet import kqreactor
kqreactor.install()
except:
log.critical("Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor")
log.debug(traceback.format_exc())
else:
log.debug("Running on *BSD or MacOSX and optimal reactor (kqueue) was installed.")
else:
log.debug("Running on *BSD or MacOSX and optimal reactor (kqueue) already installed.")
elif sys.platform in ['win32']:
# Windows
##
if current_reactor != 'IOCPReactor':
try:
from twisted.internet.iocpreactor import reactor as iocpreactor
iocpreactor.install()
except:
log.critical("Running on Windows, but cannot install IOCP Twisted reactor")
log.debug(traceback.format_exc())
else:
log.debug("Running on Windows and optimal reactor (ICOP) was installed.")
else:
log.debug("Running on Windows and optimal reactor (ICOP) already installed.")
elif sys.platform.startswith('linux'):
# Linux
##
if current_reactor != 'EPollReactor':
try:
from twisted.internet import epollreactor
epollreactor.install()
except:
log.critical("Running on Linux, but cannot install Epoll Twisted reactor")
log.debug(traceback.format_exc())
else:
log.debug("Running on Linux and optimal reactor (epoll) was installed.")
else:
log.debug("Running on Linux and optimal reactor (epoll) already installed.")
else:
try:
from twisted.internet import default as defaultreactor
defaultreactor.install()
except:
log.critical("Could not install default Twisted reactor for this platform")
log.debug(traceback.format_exc())
from twisted.internet import reactor
txaio.config.loop = reactor
import sys
import ujson
if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
from twisted.internet import kqreactor
kqreactor.install()
elif sys.platform in ['win32']:
from twisted.internet.iocpreactor import reactor as iocpreactor
iocpreactor.install()
elif sys.platform.startswith('linux'):
from twisted.internet import epollreactor
epollreactor.install()
else:
from twisted.internet import default as defaultreactor
defaultreactor.install()
from twisted.web.server import Site # noqa
from twisted.web.resource import Resource # noqa
from twisted.internet import reactor # noqa
class TestResource(Resource):
def render_GET(self, request):
body = ujson.dumps({'test': True}).encode('utf8')
request.setHeader(b'Content-Type', b'application/json; charset=utf-8')
return body
def getChild(self, path, request):
if path == b'payload':
return Payload()
beforehand. Regardless of which reactor is installed, importing this module is
the correct way to get a reference to it.
New application code should prefer to pass and accept the reactor as a
parameter where it is needed, rather than relying on being able to import this
module to get a reference. This simplifies unit testing and may make it easier
to one day support multiple reactors (as a performance enhancement), though
this is not currently possible.
@see: L{IReactorCore<twisted.internet.interfaces.IReactorCore>}
@see: L{IReactorTime<twisted.internet.interfaces.IReactorTime>}
@see: L{IReactorProcess<twisted.internet.interfaces.IReactorProcess>}
@see: L{IReactorTCP<twisted.internet.interfaces.IReactorTCP>}
@see: L{IReactorSSL<twisted.internet.interfaces.IReactorSSL>}
@see: L{IReactorUDP<twisted.internet.interfaces.IReactorUDP>}
@see: L{IReactorMulticast<twisted.internet.interfaces.IReactorMulticast>}
@see: L{IReactorUNIX<twisted.internet.interfaces.IReactorUNIX>}
@see: L{IReactorUNIXDatagram<twisted.internet.interfaces.IReactorUNIXDatagram>}
@see: L{IReactorFDSet<twisted.internet.interfaces.IReactorFDSet>}
@see: L{IReactorThreads<twisted.internet.interfaces.IReactorThreads>}
@see: L{IReactorArbitrary<twisted.internet.interfaces.IReactorArbitrary>}
@see: L{IReactorPluggableResolver<twisted.internet.interfaces.IReactorPluggableResolver>}
"""
import sys
del sys.modules["twisted.internet.reactor"]
from twisted.internet import default
default.install()
# https://github.com/pyinstaller/pyinstaller/issues/3390
import sys
# this creates module: sys.modules['twisted.internet.reactor']
if sys.platform in ['win32']:
from twisted.internet.iocpreactor import reactor as _iocpreactor
_iocpreactor.install()
else:
from twisted.internet import default
default.install()
def install_optimal_reactor(verbose=False):
"""
Try to install the optimal Twisted reactor for platform.
:param verbose: If ``True``, print what happens.
:type verbose: bool
"""
import sys
from twisted.python import reflect
from twisted.python import log
## determine currently installed reactor, if any
##
if "twisted.internet.reactor" in sys.modules:
current_reactor = reflect.qual(sys.modules["twisted.internet.reactor"].__class__).split(".")[-1]
else:
current_reactor = None
## depending on platform, install optimal reactor
##
if "bsd" in sys.platform or sys.platform.startswith("darwin"):
## *BSD and MacOSX
##
if current_reactor != "KQueueReactor":
try:
v = sys.version_info
if v[0] == 1 or (v[0] == 2 and v[1] < 6) or (v[0] == 2 and v[1] == 6 and v[2] < 5):
raise Exception("Python version too old ({0}) to use kqueue reactor".format(sys.version))
from twisted.internet import kqreactor
kqreactor.install()
except Exception as e:
log.err(
"WARNING: Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor ({0}).".format(e)
)
else:
if verbose:
log.msg("Running on *BSD or MacOSX and optimal reactor (kqueue) was installed.")
else:
if verbose:
log.msg("Running on *BSD or MacOSX and optimal reactor (kqueue) already installed.")
elif sys.platform in ["win32"]:
## Windows
##
if current_reactor != "IOCPReactor":
try:
from twisted.internet.iocpreactor import reactor as iocpreactor
iocpreactor.install()
except Exception as e:
log.err("WARNING: Running on Windows, but cannot install IOCP Twisted reactor ({0}).".format(e))
else:
if verbose:
log.msg("Running on Windows and optimal reactor (ICOP) was installed.")
else:
if verbose:
log.msg("Running on Windows and optimal reactor (ICOP) already installed.")
elif sys.platform.startswith("linux"):
## Linux
##
if current_reactor != "EPollReactor":
try:
from twisted.internet import epollreactor
epollreactor.install()
except Exception as e:
log.err("WARNING: Running on Linux, but cannot install Epoll Twisted reactor ({0}).".format(e))
else:
if verbose:
log.msg("Running on Linux and optimal reactor (epoll) was installed.")
else:
if verbose:
log.msg("Running on Linux and optimal reactor (epoll) already installed.")
else:
try:
from twisted.internet import default as defaultreactor
defaultreactor.install()
except Exception as e:
log.err("WARNING: Could not install default Twisted reactor for this platform ({0}).".format(e))
def installer():
installed.append(True)
return install()
import sys
import time
import json
import logging
from functools import partial
try:
# remove any already installed reactor
del sys.modules['twisted.internet.reactor']
except KeyError:
pass
# reinstall reactor
from twisted.internet.default import install
#from twisted.internet.pollreactor import install
#from twisted.internet.epollreactor import install
install()
del install
from twisted.internet import reactor
from twisted.internet.error import TimeoutError
# apply monkey patches for scrapy
from . import _monkeypatches
del _monkeypatches
from scrapy.spiders import Spider
from scrapy.exceptions import CloseSpider
from scrapy.settings import Settings
from scrapy.http import Request
from ..spamfilter import isspam_link, isspam_text
def install_optimal_reactor(verbose=False):
"""
Try to install the optimal Twisted reactor for this platform.
:param verbose: If ``True``, print what happens.
:type verbose: bool
"""
log = txaio.make_logger()
import sys
from twisted.python import reflect
# determine currently installed reactor, if any
##
if "twisted.internet.reactor" in sys.modules:
current_reactor = reflect.qual(sys.modules["twisted.internet.reactor"].__class__).split(".")[-1]
else:
current_reactor = None
# depending on platform, install optimal reactor
##
if "bsd" in sys.platform or sys.platform.startswith("darwin"):
# *BSD and MacOSX
##
if current_reactor != "KQueueReactor":
try:
from twisted.internet import kqreactor
kqreactor.install()
except:
log.critical("Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor")
log.warn("{tb}", tb=traceback.format_exc())
else:
log.debug("Running on *BSD or MacOSX and optimal reactor (kqueue) was installed.")
else:
log.debug("Running on *BSD or MacOSX and optimal reactor (kqueue) already installed.")
elif sys.platform in ["win32"]:
# Windows
##
if current_reactor != "IOCPReactor":
try:
from twisted.internet.iocpreactor import reactor as iocpreactor
iocpreactor.install()
except:
log.critical("Running on Windows, but cannot install IOCP Twisted reactor")
log.warn("{tb}", tb=traceback.format_exc())
else:
log.debug("Running on Windows and optimal reactor (ICOP) was installed.")
else:
log.debug("Running on Windows and optimal reactor (ICOP) already installed.")
elif sys.platform.startswith("linux"):
# Linux
##
if current_reactor != "EPollReactor":
try:
from twisted.internet import epollreactor
epollreactor.install()
except:
log.critical("Running on Linux, but cannot install Epoll Twisted reactor")
log.warn("{tb}", tb=traceback.format_exc())
else:
log.debug("Running on Linux and optimal reactor (epoll) was installed.")
else:
log.debug("Running on Linux and optimal reactor (epoll) already installed.")
else:
try:
from twisted.internet import default as defaultreactor
defaultreactor.install()
except:
log.critical("Could not install default Twisted reactor for this platform")
log.warn("{tb}", tb=traceback.format_exc())
from twisted.internet import reactor
txaio.config.loop = reactor
def install_optimal_reactor(verbose=False):
"""
Try to install the optimal Twisted reactor for platform.
:param verbose: If ``True``, print what happens.
:type verbose: bool
"""
import sys
from twisted.python import reflect
import txaio
txaio.use_twisted() # just to be sure...
# XXX should I configure txaio.config.loop in here too, or just in
# install_reactor()? (I am: see bottom of function)
# determine currently installed reactor, if any
##
if 'twisted.internet.reactor' in sys.modules:
current_reactor = reflect.qual(sys.modules['twisted.internet.reactor'].__class__).split('.')[-1]
else:
current_reactor = None
# depending on platform, install optimal reactor
##
if 'bsd' in sys.platform or sys.platform.startswith('darwin'):
# *BSD and MacOSX
##
if current_reactor != 'KQueueReactor':
try:
v = sys.version_info
if v[0] == 1 or (v[0] == 2 and v[1] < 6) or (v[0] == 2 and v[1] == 6 and v[2] < 5):
raise Exception("Python version too old ({0}) to use kqueue reactor".format(sys.version))
from twisted.internet import kqreactor
kqreactor.install()
except Exception as e:
print("WARNING: Running on *BSD or MacOSX, but cannot install kqueue Twisted reactor ({0}).".format(e))
else:
if verbose:
print("Running on *BSD or MacOSX and optimal reactor (kqueue) was installed.")
else:
if verbose:
print("Running on *BSD or MacOSX and optimal reactor (kqueue) already installed.")
elif sys.platform in ['win32']:
# Windows
##
if current_reactor != 'IOCPReactor':
try:
from twisted.internet.iocpreactor import reactor as iocpreactor
iocpreactor.install()
except Exception as e:
print("WARNING: Running on Windows, but cannot install IOCP Twisted reactor ({0}).".format(e))
else:
if verbose:
print("Running on Windows and optimal reactor (ICOP) was installed.")
else:
if verbose:
print("Running on Windows and optimal reactor (ICOP) already installed.")
elif sys.platform.startswith('linux'):
# Linux
##
if current_reactor != 'EPollReactor':
try:
from twisted.internet import epollreactor
epollreactor.install()
except Exception as e:
print("WARNING: Running on Linux, but cannot install Epoll Twisted reactor ({0}).".format(e))
else:
if verbose:
print("Running on Linux and optimal reactor (epoll) was installed.")
else:
if verbose:
print("Running on Linux and optimal reactor (epoll) already installed.")
else:
try:
from twisted.internet import default as defaultreactor
defaultreactor.install()
except Exception as e:
print("WARNING: Could not install default Twisted reactor for this platform ({0}).".format(e))
from twisted.internet import reactor
txaio.config.loop = reactor