def start(self, stop_after_crawl=True): """ This method starts a Twisted `reactor`_, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. If `stop_after_crawl` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. :param boolean stop_after_crawl: stop or not the reactor when all crawlers have finished """ if stop_after_crawl: d = self.join() # Don't start the reactor if the deferreds are already fired if d.called: return d.addBoth(lambda _: self._stop_reactor()) cache_size = self.settings.getint('DNSCACHE_SIZE') if self.settings.getbool('DNSCACHE_ENABLED') else 0 reactor.installResolver(CachingThreadedResolver(reactor, cache_size, self.settings.getfloat('DNS_TIMEOUT'))) tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE')) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call
def start(self, stop_after_crawl=True): """ This method starts a Twisted `reactor`_, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. If `stop_after_crawl` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. :param boolean stop_after_crawl: stop or not the reactor when all crawlers have finished """ if stop_after_crawl: d = self.join() # Don't start the reactor if the deferreds are already fired if d.called: return d.addBoth(self._stop_reactor) # 对reactor进行定制化处理,只能针对ipv4,设置一个内部解释器用于域名的查找 reactor.installResolver(self._get_dns_resolver()) #返回一个线程池跟reactor.callInThread有关系 tp = reactor.getThreadPool() #调节线程池的大小adjustPoolsize(self, minthreads=None, maxthreads=None) tp.adjustPoolsize( maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE')) # 添加系统事件触发事件当系统关闭的时候,系统事件激活之前,reactor将会被激活进行停止的操作 reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call
def buildHostMap(ipFile, outFilename): reactor.installResolver(client.createResolver(servers=[('127.0.0.1', 53), ('8.8.8.8', 53), ('8.8.4.4', 53)])) CHUNK_SIZE = 20000 ipList = [] with open(ipFile, 'r') as f: for line in f: if line[0].isdigit(): ipList.append(line.split(',')[0]) outFile = open(outFilename, 'w') def ret(dummy, chunk, outFile): return defer.DeferredList([reverseLookup(ip, outFile) for ip in chunk]) ipList = [ipList[i:i+CHUNK_SIZE] for i in xrange(0, len(ipList), CHUNK_SIZE-1)] d = defer.Deferred() for chunk in ipList: d.addCallback(ret, chunk, outFile) # cleanup after ourselves d.addCallback(die, outFile) d.callback(None) reactor.run()
def start(self, stop_after_crawl=True): """ This method starts a Twisted `reactor`_, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. If `stop_after_crawl` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. :param boolean stop_after_crawl: stop or not the reactor when all crawlers have finished """ if stop_after_crawl: d = self.join() # Don't start the reactor if the deferreds are already fired if d.called: return d.addBoth(self._stop_reactor) reactor.installResolver(self._get_dns_resolver()) tp = reactor.getThreadPool() tp.adjustPoolsize( maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE')) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call
def start(self, stop_after_crawl=True): """ This method starts a Twisted `reactor`_, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. :param boolean stop_after_crawl: stop or not the reactor when all crawlers have finished """ if stop_after_crawl: d = self.join() # Don't start the reactor if the deferreds are already fired if d.called: return d.addBoth(self._stop_reactor) ## 为 reactor 安装解析器 ## 是 Twisted 模块的事件管理器,只要把需要执行的事件方法注册到 reactor 中,然后 ## 调用它的 run 方法,它就会帮你执行注册好的事件方法,如果遇到网络 IO 等待,它会 ## 自动帮你切换可执行的事件方法,非常高效 reactor.installResolver(self._get_dns_resolver()) ## 获取线程池 tp = reactor.getThreadPool() ## 调整 reactor 的线程池大小(通过修改 REACTOR_THREADPOOL_MAXSIZE 调整) tp.adjustPoolsize( maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE')) ## 添加系统事件触发器 reactor.addSystemEventTrigger('before', 'shutdown', self.stop) ## 开始执行 reactor.run(installSignalHandlers=False) # blocking call
def action_JAP_UPDATE(self, request): data = request.args["data"][0] decoder = JAP_LOCAL.JSONDecoder() configuration = decoder.decode(data) JAP_LOCAL.setConfiguration("./JAP.json", configuration, getDefaultConfiguration) logger = logging.getLogger("JAP") if configuration["LOGGER"]["LEVEL"] == "DEBUG": logger.setLevel(logging.DEBUG) elif configuration["LOGGER"]["LEVEL"] == "INFO": logger.setLevel(logging.INFO) elif configuration["LOGGER"]["LEVEL"] == "WARNING": logger.setLevel(logging.WARNING) elif configuration["LOGGER"]["LEVEL"] == "ERROR": logger.setLevel(logging.ERROR) elif configuration["LOGGER"]["LEVEL"] == "CRITICAL": logger.setLevel(logging.CRITICAL) else: logger.setLevel(logging.NOTSET) resolver = JAP_LOCAL.createResolver(configuration) reactor.installResolver(resolver) return ""
def installResolver(reactor=None, resolv='/etc/resolv.conf', servers=[], hosts='/etc/hosts'): theResolver = Resolver(resolv, servers) hostResolver = hostsModule.Resolver(hosts) L = [hostResolver, cache.CacheResolver(), theResolver] if reactor is None: from twisted.internet import reactor reactor.installResolver(ResolverChain(L))
def _start_reactor(self, stop_after_crawl=True): if stop_after_crawl: d = defer.DeferredList(self.crawl_deferreds) if d.called: # Don't start the reactor if the deferreds are already fired return d.addBoth(lambda _: self._stop_reactor()) if self.settings.getbool('DNSCACHE_ENABLED'): reactor.installResolver(CachingThreadedResolver(reactor)) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call
def start(self, stop_after_crawl=True): if stop_after_crawl: d = defer.DeferredList(self.crawl_deferreds) if d.called: # Don't start the reactor if the deferreds are already fired return d.addBoth(lambda _: self._stop_reactor()) if self.settings.getbool('DNSCACHE_ENABLED'): reactor.installResolver(CachingThreadedResolver(reactor)) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call
def setUp(self): self.fromAddress = q2q.Q2QAddress(self.fromDomain, self.fromResource) self.toAddress = q2q.Q2QAddress(self.toDomain, self.toResource) # A mapping of host names to port numbers Our connectTCP will always # connect to 127.0.0.1 and on a port which is a value in this # dictionary. fakeDNS = FakeConnectTCP(reactor.connectTCP) reactor.connectTCP = fakeDNS.connectTCP # ALSO WE MUST DO OTHER SIMILAR THINGS self._oldResolver = reactor.resolver reactor.installResolver(fakeDNS) # Set up a know-nothing service object for the client half of the # conversation. self.serverService2 = self._makeQ2QService( self.fromDomain, self.fromIP, noResources ) # Do likewise for the server half of the conversation. Also, allow # test methods to set up some trivial resources which we can attempt to # access from the client. self.resourceMap = {} self.serverService = self._makeQ2QService( self.toDomain, self.toIP, self.protocolFactoryLookup ) self.msvc = service.MultiService() self.serverService2.setServiceParent(self.msvc) self.serverService.setServiceParent(self.msvc) # Let the kernel allocate a random port for each of # these service's listeners self.msvc.startService() fakeDNS.addHostPort( self.fromDomain, 8788, self.serverService2.q2qPort.getHost().port) fakeDNS.addHostPort( self.toDomain, 8788, self.serverService.q2qPort.getHost().port) self._addQ2QProtocol('pony', OneTrickPonyServerFactory()) self.dataEater = DataEater() self._addQ2QProtocol('eat', self.dataEater) self._addQ2QProtocol('error', ErroneousServerFactory())
def start(self, stop_after_crawl=True): if stop_after_crawl: d = self.join() # Don't start the reactor if the deferreds are already fired if d.called: return d.addBoth(lambda _: self._stop_reactor()) if self.settings.getbool('DNSCACHE_ENABLED'): reactor.installResolver(CachingThreadedResolver(reactor)) tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE')) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call
def start(self, stop_after_crawl=True): if stop_after_crawl: d = self.join() # Don't start the reactor if the deferreds are already fired if d.called: return d.addBoth(lambda _: self._stop_reactor()) if self.settings.getbool('DNSCACHE_ENABLED'): reactor.installResolver(CachingThreadedResolver(reactor)) tp = reactor.getThreadPool() tp.adjustPoolsize( maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE')) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call
def makeService(config): # install simple, blocking DNS resolver. from twisted.internet import reactor from twisted.internet.base import BlockingResolver reactor.installResolver(BlockingResolver()) try: # Create dbus mainloop import dbus.mainloop.glib dbus.mainloop.glib.DBusGMainLoop(set_as_default=True) except ImportError: warnings.warn('Failed to import the dbus module, some functionality might not work.') # Check if it is the right thing if not hasattr(config.module, 'Application'): raise usage.UsageError("Invalid application module: " + config.appName) # Instantiate the main application s = config.module.Application(config.appName, config.opts, config.appOpts) # Set quitflag s.quitFlag = launcher.QuitFlag(s.path("temp").child("quitflag")) # Set the name s.setName(config.appName) # make sure the relevant paths exist for kind in ["temp", "db"]: path = s.path(kind) if not path.exists(): path.createDirectory() # Set up logging logFile = s.path("logfile") if not logFile.parent().exists(): logFile.parent().createDirectory() filename = s.path("logfile").path if config.opts['no-logrotate']: observer = RotatableFileLogObserver(filename) else: logFile = LogFile.fromFullPath(filename, maxRotatedFiles=9) observer = log.FileLogObserver(logFile).emit log.addObserver(observer) return s
def setUp(self): self.fromAddress = q2q.Q2QAddress(self.fromDomain, self.fromResource) self.toAddress = q2q.Q2QAddress(self.toDomain, self.toResource) # A mapping of host names to port numbers Our connectTCP will always # connect to 127.0.0.1 and on a port which is a value in this # dictionary. fakeDNS = FakeConnectTCP(reactor.connectTCP) reactor.connectTCP = fakeDNS.connectTCP # ALSO WE MUST DO OTHER SIMILAR THINGS self._oldResolver = reactor.resolver reactor.installResolver(fakeDNS) # Set up a know-nothing service object for the client half of the # conversation. self.serverService2 = self._makeQ2QService(self.fromDomain, self.fromIP, noResources) # Do likewise for the server half of the conversation. Also, allow # test methods to set up some trivial resources which we can attempt to # access from the client. self.resourceMap = {} self.serverService = self._makeQ2QService(self.toDomain, self.toIP, self.protocolFactoryLookup) self.msvc = service.MultiService() self.serverService2.setServiceParent(self.msvc) self.serverService.setServiceParent(self.msvc) # Let the kernel allocate a random port for each of # these service's listeners self.msvc.startService() fakeDNS.addHostPort(self.fromDomain, 8788, self.serverService2.q2qPort.getHost().port) fakeDNS.addHostPort(self.toDomain, 8788, self.serverService.q2qPort.getHost().port) self._addQ2QProtocol('pony', OneTrickPonyServerFactory()) self.dataEater = DataEater() self._addQ2QProtocol('eat', self.dataEater) self._addQ2QProtocol('error', ErroneousServerFactory())
def tearDown(self): reactor.installResolver(self._oldResolver) del reactor.connectTCP return self.msvc.stopService()
def start(self): super(CrawlerProcess, self).start() if self.settings.getbool('DNSCACHE_ENABLED'): reactor.installResolver(CachingThreadedResolver(reactor)) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call
############################# threadable.init() if aztk_config.setup.get("site", "environment") == "sandbox": reactor.suggestThreadPoolSize(3) else: try: reactor.suggestThreadPoolSize(25) except Exception, ex: print Exception print ex sys.exit(1) ################################## ## Setup Threaded Name Resolver ## ################################## reactor.installResolver(client.ThreadedResolver(reactor)) __version__ = "$Revision: 1.18 $" class Application(object): """ This is the main application definition. All instances of an AZKT server will be started with this class. """ def __init__(self, profile): """ @param profile: Server profile as mentioned in install.cfg """ self.profile = profile
import logging import JAP.JAP_LOCAL import JAP.JAP_LOCAL_WS configuration = JAP.JAP_LOCAL.getConfiguration( "./JAP_LOCAL_WS.json", JAP.JAP_LOCAL_WS.getDefaultConfiguration) logging.basicConfig() logger = logging.getLogger("JAP") if configuration["LOGGER"]["LEVEL"] == "DEBUG": logger.setLevel(logging.DEBUG) elif configuration["LOGGER"]["LEVEL"] == "INFO": logger.setLevel(logging.INFO) elif configuration["LOGGER"]["LEVEL"] == "WARNING": logger.setLevel(logging.WARNING) elif configuration["LOGGER"]["LEVEL"] == "ERROR": logger.setLevel(logging.ERROR) elif configuration["LOGGER"]["LEVEL"] == "CRITICAL": logger.setLevel(logging.CRITICAL) else: logger.setLevel(logging.NOTSET) resolver = JAP.JAP_LOCAL.createResolver(configuration) reactor.installResolver(resolver) factory = JAP.JAP_LOCAL_WS.WSInputProtocolFactory(configuration) factory.protocol = JAP.JAP_LOCAL_WS.WSInputProtocol reactor.listenTCP(configuration["LOCAL_PROXY_SERVER"]["PORT"], factory, 50, configuration["LOCAL_PROXY_SERVER"]["ADDRESS"]) reactor.run()
This application does a lot of things. 1. Creates a server (nwn format) that clients can connect to... 2. Ingests data... a. Connects to Baron to get its data b. Watches for wx32 files from KELO (provided by LDM) c. Connects to IEM service for KIMT data d. Listens for TWI poller connections """ #from twisted.internet.app import Application from twisted.application import service, internet from twisted.cred import checkers, portal # Hack around 2.0 problem from twisted.internet import reactor, base reactor.installResolver(base.BlockingResolver()) # Import nwnserver stuff! from nwnserver import hubclient, proxyserver, pollerserv, filewatcher import os import sys sys.path.insert(0, os.getcwd()) import secret # Create the Python Hub application application = service.Application("PythonHub") serviceCollection = service.IServiceCollection(application) #______________________________________________________________________ # 1. Setup hub server! # Accepts connects on port 14996
### import supybot.log as log import supybot.conf as conf import supybot.drivers as drivers import supybot.ircmsgs as ircmsgs from twisted.names import client from twisted.internet import reactor, error from twisted.protocols.basic import LineReceiver from twisted.internet.protocol import ReconnectingClientFactory # This hack prevents the standard Twisted resolver from starting any # threads, which allows for a clean shut-down in Twisted>=2.0 reactor.installResolver(client.createResolver()) try: from OpenSSL import SSL from twisted.internet import ssl except ImportError: drivers.log.debug('PyOpenSSL is not available, ' 'cannot connect to SSL servers.') SSL = None class TwistedRunnerDriver(drivers.IrcDriver): def name(self): return self.__class__.__name__ def run(self):
def start_reactor(self): if self.settings.getbool('DNSCACHE_ENABLED'): reactor.installResolver(CachingThreadedResolver(reactor)) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call
def __init__(self): self.spider_hostnames = defaultdict(set) self.resolver = _CachingThreadedResolver(reactor) reactor.installResolver(self.resolver) dispatcher.connect(self.request_received, signals.request_received) dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
This application does a lot of things. 1. Creates a server (nwn format) that clients can connect to... 2. Ingests data... a. Connects to Baron to get its data b. Watches for wx32 files from KELO (provided by LDM) c. Connects to IEM service for KIMT data d. Listens for TWI poller connections """ #from twisted.internet.app import Application from twisted.application import service, internet from twisted.cred import checkers, portal # Hack around 2.0 problem from twisted.internet import reactor, base reactor.installResolver(base.BlockingResolver()) # Import nwnserver stuff! from nwnserver import hubclient, proxyserver, pollerserv, filewatcher import os import sys sys.path.insert(0, os.getcwd()) import secret # Create the Python Hub application application = service.Application("PythonHub") serviceCollection = service.IServiceCollection(application) #______________________________________________________________________ # 1. Setup hub server!
import JAP.JAP_LOCAL import JAP.JAP_LOCAL_SSH configuration = JAP.JAP_LOCAL.getConfiguration("./JAP_LOCAL_SSH.json", JAP.JAP_LOCAL_SSH.getDefaultConfiguration) logging.basicConfig() logger = logging.getLogger("JAP") if configuration["LOGGER"]["LEVEL"] == "DEBUG": logger.setLevel(logging.DEBUG) elif configuration["LOGGER"]["LEVEL"] == "INFO": logger.setLevel(logging.INFO) elif configuration["LOGGER"]["LEVEL"] == "WARNING": logger.setLevel(logging.WARNING) elif configuration["LOGGER"]["LEVEL"] == "ERROR": logger.setLevel(logging.ERROR) elif configuration["LOGGER"]["LEVEL"] == "CRITICAL": logger.setLevel(logging.CRITICAL) else: logger.setLevel(logging.NOTSET) resolver = JAP.JAP_LOCAL.createResolver(configuration) reactor.installResolver(resolver) factory = JAP.JAP_LOCAL_SSH.SSHInputProtocolFactory(configuration) factory.protocol = JAP.JAP_LOCAL_SSH.SSHInputProtocol reactor.listenTCP( configuration["LOCAL_PROXY_SERVER"]["PORT"], factory, 50, configuration["LOCAL_PROXY_SERVER"]["ADDRESS"] ) reactor.run()
def makeService(options): from twisted.names.client import getResolver resolver = getResolver() # HACK: I want a better resolver than the threaded one, and lack a # non-terrible place to install it. from twisted.internet import reactor if not options['keep-resolver']: # pragma: no cover reactor.installResolver(resolver) # HACK: warn about suboptimal reactor usage if not options['force-select']: # pragma: no cover from twisted.internet import selectreactor if isinstance(reactor, selectreactor.SelectReactor): print('The select reactor is probably a bad idea.') print('Please use a reactor with better support for lots of fds.') print('(-r epoll for example)') print('You can bypass this check using --force-select') print() raise ValueError('unfortunate reactor choice') m = service.MultiService() checkerFactories = plugin.getCheckerFactories() default_user_reason = options['conf'].get('user-reason', '') default_oper_reason = options['conf'].get('oper-reason', '') scansets = {} for name, d in options['conf']['scansets'].items(): scans = [] for args in d['protocols']: poolname = args.pop(0) checkername = args.pop(0) checker = checkerFactories[checkername](*args) scans.append((poolname, checker.check)) user_reason = d.get('user-reason', default_user_reason) oper_reason = d.get('oper-reason', default_oper_reason) scansets[name] = scanner.ScanSet(d['timeout'], scans, user_reason, oper_reason) # XXXX the target_blah passing here is horrible, but I intend to # make this less global in the future anyway, so laaaaaaaaater env = {} for k in [ 'target_ip', 'target_port', 'target_url', 'target_strings', 'max_bytes', 'bind_address' ]: env[k] = options['conf'].get(k) theScanner = scanner.Scanner(reactor, resolver, options['conf']['pools'], scansets, env) for name, net in options['conf'].get('irc', {}).items(): # XXX terrible, see also similar complaints in ircpresence. # Split this up later. factory = ircpresence.Factory( net['nick'], net['channel'], password=net.get('pass'), opername=net.get('opername', net['nick']), operpass=net.get('operpass'), operkey=net.get('operkey'), opermode=net.get('opermode'), away=net.get('away'), connregex=net.get('connregex'), scanner=theScanner, masks=options['conf'].get('masks', {}), actions=net.get('actions'), onconnectmsgs=net.get('onconnectmsgs', ()), verbose=options['irc-log'], flood_exempt=net.get('flood_exempt', False), username=net.get('username'), cache_time=net.get("scan-cache-time", 120), cache_size=net.get("scan-cache-size", 1_000_000)) if net.get('ssl', False): ctxf = ssl.ClientContextFactory() serv = internet.SSLClient(net['host'], net['port'], factory, ctxf) else: serv = internet.TCPClient(net['host'], net['port'], factory) serv.setName(name) serv.setServiceParent(m) return m
############################# threadable.init() if aztk_config.setup.get("site", "environment") == "sandbox": reactor.suggestThreadPoolSize(3) else: try: reactor.suggestThreadPoolSize(25) except Exception, ex: print Exception print ex sys.exit(1) ################################## ## Setup Threaded Name Resolver ## ################################## reactor.installResolver(base.ThreadedResolver(reactor)) __version__ = "$Revision: 1.18 $" class Application(object): """ This is the main application definition. All instances of an AZKT server will be started with this class. """ def __init__(self, profile): """ @param profile: Server profile as mentioned in install.cfg """ self.profile = profile
# POSSIBILITY OF SUCH DAMAGE. ### import supybot.log as log import supybot.conf as conf import supybot.drivers as drivers import supybot.ircmsgs as ircmsgs from twisted.names import client from twisted.internet import reactor, error from twisted.protocols.basic import LineReceiver from twisted.internet.protocol import ReconnectingClientFactory # This hack prevents the standard Twisted resolver from starting any # threads, which allows for a clean shut-down in Twisted>=2.0 reactor.installResolver(client.createResolver()) try: from OpenSSL import SSL from twisted.internet import ssl except ImportError: drivers.log.debug('PyOpenSSL is not available, ' 'cannot connect to SSL servers.') SSL = None class TwistedRunnerDriver(drivers.IrcDriver): def name(self): return self.__class__.__name__ def run(self):
def start(self): super(CrawlerProcess, self).start() if self.dnscache_enable.to_value(): reactor.installResolver(CachingThreadedResolver(reactor)) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call
from tests.mockserver import MockServer, MockDNSServer class LocalhostSpider(Spider): name = "localhost_spider" def start_requests(self): yield Request(self.url) def parse(self, response): netloc = urlparse(response.url).netloc self.logger.info("Host: %s" % netloc.split(":")[0]) self.logger.info("Type: %s" % type(response.ip_address)) self.logger.info("IP address: %s" % response.ip_address) if __name__ == "__main__": with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: port = urlparse(mock_http_server.http_address).port url = "http://not.a.real.domain:{port}/echo".format(port=port) servers = [(mock_dns_server.host, mock_dns_server.port)] reactor.installResolver(createResolver(servers=servers)) configure_logging() runner = CrawlerRunner() d = runner.crawl(LocalhostSpider, url=url) d.addBoth(lambda _: reactor.stop()) reactor.run()
def start(self): super(CrawlerProcess, self).start() if self.settings.getbool("DNSCACHE_ENABLED"): reactor.installResolver(CachingThreadedResolver(reactor)) reactor.addSystemEventTrigger("before", "shutdown", self.stop) reactor.run(installSignalHandlers=False) # blocking call