def start(self, stop_after_crawl=True):
"""
This method starts a Twisted `reactor`_, adjusts its pool size to
:setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based
on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
If `stop_after_crawl` is True, the reactor will be stopped after all
crawlers have finished, using :meth:`join`.
:param boolean stop_after_crawl: stop or not the reactor when all
crawlers have finished
"""
if stop_after_crawl:
d = self.join()
# Don't start the reactor if the deferreds are already fired
if d.called:
return
d.addBoth(lambda _: self._stop_reactor())
cache_size = self.settings.getint('DNSCACHE_SIZE') if self.settings.getbool('DNSCACHE_ENABLED') else 0
reactor.installResolver(CachingThreadedResolver(reactor, cache_size,
self.settings.getfloat('DNS_TIMEOUT')))
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
def start(self, stop_after_crawl=True):
"""
This method starts a Twisted `reactor`_, adjusts its pool size to
:setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based
on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
If `stop_after_crawl` is True, the reactor will be stopped after all
crawlers have finished, using :meth:`join`.
:param boolean stop_after_crawl: stop or not the reactor when all
crawlers have finished
"""
if stop_after_crawl:
d = self.join()
# Don't start the reactor if the deferreds are already fired
if d.called:
return
d.addBoth(self._stop_reactor)
# 对reactor进行定制化处理，只能针对ipv4，设置一个内部解释器用于域名的查找
reactor.installResolver(self._get_dns_resolver())
#返回一个线程池跟reactor.callInThread有关系
tp = reactor.getThreadPool()
#调节线程池的大小adjustPoolsize(self, minthreads=None, maxthreads=None)
tp.adjustPoolsize(
maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
# 添加系统事件触发事件当系统关闭的时候，系统事件激活之前，reactor将会被激活进行停止的操作
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
def buildHostMap(ipFile, outFilename):
reactor.installResolver(client.createResolver(servers=[('127.0.0.1', 53), ('8.8.8.8', 53), ('8.8.4.4', 53)]))
CHUNK_SIZE = 20000
ipList = []
with open(ipFile, 'r') as f:
for line in f:
if line[0].isdigit():
ipList.append(line.split(',')[0])
outFile = open(outFilename, 'w')
def ret(dummy, chunk, outFile):
return defer.DeferredList([reverseLookup(ip, outFile) for ip in chunk])
ipList = [ipList[i:i+CHUNK_SIZE] for i in xrange(0, len(ipList), CHUNK_SIZE-1)]
d = defer.Deferred()
for chunk in ipList:
d.addCallback(ret, chunk, outFile)
# cleanup after ourselves
d.addCallback(die, outFile)
d.callback(None)
reactor.run()
def start(self, stop_after_crawl=True):
"""
This method starts a Twisted `reactor`_, adjusts its pool size to
:setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based
on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
If `stop_after_crawl` is True, the reactor will be stopped after all
crawlers have finished, using :meth:`join`.
:param boolean stop_after_crawl: stop or not the reactor when all
crawlers have finished
"""
if stop_after_crawl:
d = self.join()
# Don't start the reactor if the deferreds are already fired
if d.called:
return
d.addBoth(self._stop_reactor)
reactor.installResolver(self._get_dns_resolver())
tp = reactor.getThreadPool()
tp.adjustPoolsize(
maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
def start(self, stop_after_crawl=True):
"""
This method starts a Twisted `reactor`_, adjusts its pool size to
:setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based
on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
If ``stop_after_crawl`` is True, the reactor will be stopped after all
crawlers have finished, using :meth:`join`.
:param boolean stop_after_crawl: stop or not the reactor when all
crawlers have finished
"""
if stop_after_crawl:
d = self.join()
# Don't start the reactor if the deferreds are already fired
if d.called:
return
d.addBoth(self._stop_reactor)
## 为 reactor 安装解析器
## 是 Twisted 模块的事件管理器，只要把需要执行的事件方法注册到 reactor 中，然后
## 调用它的 run 方法，它就会帮你执行注册好的事件方法，如果遇到网络 IO 等待，它会
## 自动帮你切换可执行的事件方法，非常高效
reactor.installResolver(self._get_dns_resolver())
## 获取线程池
tp = reactor.getThreadPool()
## 调整 reactor 的线程池大小（通过修改 REACTOR_THREADPOOL_MAXSIZE 调整）
tp.adjustPoolsize(
maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
## 添加系统事件触发器
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
## 开始执行
reactor.run(installSignalHandlers=False) # blocking call
def action_JAP_UPDATE(self, request):
data = request.args["data"][0]
decoder = JAP_LOCAL.JSONDecoder()
configuration = decoder.decode(data)
JAP_LOCAL.setConfiguration("./JAP.json", configuration,
getDefaultConfiguration)
logger = logging.getLogger("JAP")
if configuration["LOGGER"]["LEVEL"] == "DEBUG":
logger.setLevel(logging.DEBUG)
elif configuration["LOGGER"]["LEVEL"] == "INFO":
logger.setLevel(logging.INFO)
elif configuration["LOGGER"]["LEVEL"] == "WARNING":
logger.setLevel(logging.WARNING)
elif configuration["LOGGER"]["LEVEL"] == "ERROR":
logger.setLevel(logging.ERROR)
elif configuration["LOGGER"]["LEVEL"] == "CRITICAL":
logger.setLevel(logging.CRITICAL)
else:
logger.setLevel(logging.NOTSET)
resolver = JAP_LOCAL.createResolver(configuration)
reactor.installResolver(resolver)
return ""
def action_JAP_UPDATE(self, request):
data = request.args["data"][0]
decoder = JAP_LOCAL.JSONDecoder()
configuration = decoder.decode(data)
JAP_LOCAL.setConfiguration("./JAP.json", configuration, getDefaultConfiguration)
logger = logging.getLogger("JAP")
if configuration["LOGGER"]["LEVEL"] == "DEBUG":
logger.setLevel(logging.DEBUG)
elif configuration["LOGGER"]["LEVEL"] == "INFO":
logger.setLevel(logging.INFO)
elif configuration["LOGGER"]["LEVEL"] == "WARNING":
logger.setLevel(logging.WARNING)
elif configuration["LOGGER"]["LEVEL"] == "ERROR":
logger.setLevel(logging.ERROR)
elif configuration["LOGGER"]["LEVEL"] == "CRITICAL":
logger.setLevel(logging.CRITICAL)
else:
logger.setLevel(logging.NOTSET)
resolver = JAP_LOCAL.createResolver(configuration)
reactor.installResolver(resolver)
return ""
def installResolver(reactor=None,
resolv='/etc/resolv.conf',
servers=[],
hosts='/etc/hosts'):
theResolver = Resolver(resolv, servers)
hostResolver = hostsModule.Resolver(hosts)
L = [hostResolver, cache.CacheResolver(), theResolver]
if reactor is None:
from twisted.internet import reactor
reactor.installResolver(ResolverChain(L))
def _start_reactor(self, stop_after_crawl=True):
if stop_after_crawl:
d = defer.DeferredList(self.crawl_deferreds)
if d.called:
# Don't start the reactor if the deferreds are already fired
return
d.addBoth(lambda _: self._stop_reactor())
if self.settings.getbool('DNSCACHE_ENABLED'):
reactor.installResolver(CachingThreadedResolver(reactor))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
def start(self, stop_after_crawl=True):
if stop_after_crawl:
d = defer.DeferredList(self.crawl_deferreds)
if d.called:
# Don't start the reactor if the deferreds are already fired
return
d.addBoth(lambda _: self._stop_reactor())
if self.settings.getbool('DNSCACHE_ENABLED'):
reactor.installResolver(CachingThreadedResolver(reactor))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
def setUp(self):
self.fromAddress = q2q.Q2QAddress(self.fromDomain, self.fromResource)
self.toAddress = q2q.Q2QAddress(self.toDomain, self.toResource)
# A mapping of host names to port numbers Our connectTCP will always
# connect to 127.0.0.1 and on a port which is a value in this
# dictionary.
fakeDNS = FakeConnectTCP(reactor.connectTCP)
reactor.connectTCP = fakeDNS.connectTCP
# ALSO WE MUST DO OTHER SIMILAR THINGS
self._oldResolver = reactor.resolver
reactor.installResolver(fakeDNS)
# Set up a know-nothing service object for the client half of the
# conversation.
self.serverService2 = self._makeQ2QService(
self.fromDomain, self.fromIP, noResources
)
# Do likewise for the server half of the conversation. Also, allow
# test methods to set up some trivial resources which we can attempt to
# access from the client.
self.resourceMap = {}
self.serverService = self._makeQ2QService(
self.toDomain, self.toIP, self.protocolFactoryLookup
)
self.msvc = service.MultiService()
self.serverService2.setServiceParent(self.msvc)
self.serverService.setServiceParent(self.msvc)
# Let the kernel allocate a random port for each of
# these service's listeners
self.msvc.startService()
fakeDNS.addHostPort(
self.fromDomain, 8788,
self.serverService2.q2qPort.getHost().port)
fakeDNS.addHostPort(
self.toDomain, 8788,
self.serverService.q2qPort.getHost().port)
self._addQ2QProtocol('pony', OneTrickPonyServerFactory())
self.dataEater = DataEater()
self._addQ2QProtocol('eat', self.dataEater)
self._addQ2QProtocol('error', ErroneousServerFactory())
def start(self, stop_after_crawl=True):
if stop_after_crawl:
d = self.join()
# Don't start the reactor if the deferreds are already fired
if d.called:
return
d.addBoth(lambda _: self._stop_reactor())
if self.settings.getbool('DNSCACHE_ENABLED'):
reactor.installResolver(CachingThreadedResolver(reactor))
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
def start(self, stop_after_crawl=True):
if stop_after_crawl:
d = self.join()
# Don't start the reactor if the deferreds are already fired
if d.called:
return
d.addBoth(lambda _: self._stop_reactor())
if self.settings.getbool('DNSCACHE_ENABLED'):
reactor.installResolver(CachingThreadedResolver(reactor))
tp = reactor.getThreadPool()
tp.adjustPoolsize(
maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
def makeService(config):
# install simple, blocking DNS resolver.
from twisted.internet import reactor
from twisted.internet.base import BlockingResolver
reactor.installResolver(BlockingResolver())
try:
# Create dbus mainloop
import dbus.mainloop.glib
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
except ImportError:
warnings.warn('Failed to import the dbus module, some functionality might not work.')
# Check if it is the right thing
if not hasattr(config.module, 'Application'):
raise usage.UsageError("Invalid application module: " + config.appName)
# Instantiate the main application
s = config.module.Application(config.appName, config.opts, config.appOpts)
# Set quitflag
s.quitFlag = launcher.QuitFlag(s.path("temp").child("quitflag"))
# Set the name
s.setName(config.appName)
# make sure the relevant paths exist
for kind in ["temp", "db"]:
path = s.path(kind)
if not path.exists():
path.createDirectory()
# Set up logging
logFile = s.path("logfile")
if not logFile.parent().exists():
logFile.parent().createDirectory()
filename = s.path("logfile").path
if config.opts['no-logrotate']:
observer = RotatableFileLogObserver(filename)
else:
logFile = LogFile.fromFullPath(filename, maxRotatedFiles=9)
observer = log.FileLogObserver(logFile).emit
log.addObserver(observer)
return s
def setUp(self):
self.fromAddress = q2q.Q2QAddress(self.fromDomain, self.fromResource)
self.toAddress = q2q.Q2QAddress(self.toDomain, self.toResource)
# A mapping of host names to port numbers Our connectTCP will always
# connect to 127.0.0.1 and on a port which is a value in this
# dictionary.
fakeDNS = FakeConnectTCP(reactor.connectTCP)
reactor.connectTCP = fakeDNS.connectTCP
# ALSO WE MUST DO OTHER SIMILAR THINGS
self._oldResolver = reactor.resolver
reactor.installResolver(fakeDNS)
# Set up a know-nothing service object for the client half of the
# conversation.
self.serverService2 = self._makeQ2QService(self.fromDomain,
self.fromIP, noResources)
# Do likewise for the server half of the conversation. Also, allow
# test methods to set up some trivial resources which we can attempt to
# access from the client.
self.resourceMap = {}
self.serverService = self._makeQ2QService(self.toDomain, self.toIP,
self.protocolFactoryLookup)
self.msvc = service.MultiService()
self.serverService2.setServiceParent(self.msvc)
self.serverService.setServiceParent(self.msvc)
# Let the kernel allocate a random port for each of
# these service's listeners
self.msvc.startService()
fakeDNS.addHostPort(self.fromDomain, 8788,
self.serverService2.q2qPort.getHost().port)
fakeDNS.addHostPort(self.toDomain, 8788,
self.serverService.q2qPort.getHost().port)
self._addQ2QProtocol('pony', OneTrickPonyServerFactory())
self.dataEater = DataEater()
self._addQ2QProtocol('eat', self.dataEater)
self._addQ2QProtocol('error', ErroneousServerFactory())
def tearDown(self):
reactor.installResolver(self._oldResolver)
del reactor.connectTCP
return self.msvc.stopService()
def start(self):
super(CrawlerProcess, self).start()
if self.settings.getbool('DNSCACHE_ENABLED'):
reactor.installResolver(CachingThreadedResolver(reactor))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
def tearDown(self):
reactor.installResolver(self._oldResolver)
del reactor.connectTCP
return self.msvc.stopService()
#############################
threadable.init()
if aztk_config.setup.get("site", "environment") == "sandbox":
reactor.suggestThreadPoolSize(3)
else:
try:
reactor.suggestThreadPoolSize(25)
except Exception, ex:
print Exception
print ex
sys.exit(1)
##################################
## Setup Threaded Name Resolver ##
##################################
reactor.installResolver(client.ThreadedResolver(reactor))
__version__ = "$Revision: 1.18 $"
class Application(object):
"""
This is the main application definition. All instances of an AZKT
server will be started with this class.
"""
def __init__(self, profile):
"""
@param profile: Server profile as mentioned in install.cfg
"""
self.profile = profile
import logging
import JAP.JAP_LOCAL
import JAP.JAP_LOCAL_WS
configuration = JAP.JAP_LOCAL.getConfiguration(
"./JAP_LOCAL_WS.json", JAP.JAP_LOCAL_WS.getDefaultConfiguration)
logging.basicConfig()
logger = logging.getLogger("JAP")
if configuration["LOGGER"]["LEVEL"] == "DEBUG":
logger.setLevel(logging.DEBUG)
elif configuration["LOGGER"]["LEVEL"] == "INFO":
logger.setLevel(logging.INFO)
elif configuration["LOGGER"]["LEVEL"] == "WARNING":
logger.setLevel(logging.WARNING)
elif configuration["LOGGER"]["LEVEL"] == "ERROR":
logger.setLevel(logging.ERROR)
elif configuration["LOGGER"]["LEVEL"] == "CRITICAL":
logger.setLevel(logging.CRITICAL)
else:
logger.setLevel(logging.NOTSET)
resolver = JAP.JAP_LOCAL.createResolver(configuration)
reactor.installResolver(resolver)
factory = JAP.JAP_LOCAL_WS.WSInputProtocolFactory(configuration)
factory.protocol = JAP.JAP_LOCAL_WS.WSInputProtocol
reactor.listenTCP(configuration["LOCAL_PROXY_SERVER"]["PORT"], factory, 50,
configuration["LOCAL_PROXY_SERVER"]["ADDRESS"])
reactor.run()
This application does a lot of things.
1. Creates a server (nwn format) that clients can connect to...
2. Ingests data...
a. Connects to Baron to get its data
b. Watches for wx32 files from KELO (provided by LDM)
c. Connects to IEM service for KIMT data
d. Listens for TWI poller connections
"""
#from twisted.internet.app import Application
from twisted.application import service, internet
from twisted.cred import checkers, portal
# Hack around 2.0 problem
from twisted.internet import reactor, base
reactor.installResolver(base.BlockingResolver())
# Import nwnserver stuff!
from nwnserver import hubclient, proxyserver, pollerserv, filewatcher
import os
import sys
sys.path.insert(0, os.getcwd())
import secret
# Create the Python Hub application
application = service.Application("PythonHub")
serviceCollection = service.IServiceCollection(application)
#______________________________________________________________________
# 1. Setup hub server!
# Accepts connects on port 14996
###
import supybot.log as log
import supybot.conf as conf
import supybot.drivers as drivers
import supybot.ircmsgs as ircmsgs
from twisted.names import client
from twisted.internet import reactor, error
from twisted.protocols.basic import LineReceiver
from twisted.internet.protocol import ReconnectingClientFactory
# This hack prevents the standard Twisted resolver from starting any
# threads, which allows for a clean shut-down in Twisted>=2.0
reactor.installResolver(client.createResolver())
try:
from OpenSSL import SSL
from twisted.internet import ssl
except ImportError:
drivers.log.debug('PyOpenSSL is not available, '
'cannot connect to SSL servers.')
SSL = None
class TwistedRunnerDriver(drivers.IrcDriver):
def name(self):
return self.__class__.__name__
def run(self):
def start_reactor(self):
if self.settings.getbool('DNSCACHE_ENABLED'):
reactor.installResolver(CachingThreadedResolver(reactor))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
def __init__(self):
self.spider_hostnames = defaultdict(set)
self.resolver = _CachingThreadedResolver(reactor)
reactor.installResolver(self.resolver)
dispatcher.connect(self.request_received, signals.request_received)
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
#############################
threadable.init()
if aztk_config.setup.get("site", "environment") == "sandbox":
reactor.suggestThreadPoolSize(3)
else:
try:
reactor.suggestThreadPoolSize(25)
except Exception, ex:
print Exception
print ex
sys.exit(1)
##################################
## Setup Threaded Name Resolver ##
##################################
reactor.installResolver(client.ThreadedResolver(reactor))
__version__ = "$Revision: 1.18 $"
class Application(object):
"""
This is the main application definition. All instances of an AZKT
server will be started with this class.
"""
def __init__(self, profile):
"""
@param profile: Server profile as mentioned in install.cfg
"""
self.profile = profile
This application does a lot of things.
1. Creates a server (nwn format) that clients can connect to...
2. Ingests data...
a. Connects to Baron to get its data
b. Watches for wx32 files from KELO (provided by LDM)
c. Connects to IEM service for KIMT data
d. Listens for TWI poller connections
"""
#from twisted.internet.app import Application
from twisted.application import service, internet
from twisted.cred import checkers, portal
# Hack around 2.0 problem
from twisted.internet import reactor, base
reactor.installResolver(base.BlockingResolver())
# Import nwnserver stuff!
from nwnserver import hubclient, proxyserver, pollerserv, filewatcher
import os
import sys
sys.path.insert(0, os.getcwd())
import secret
# Create the Python Hub application
application = service.Application("PythonHub")
serviceCollection = service.IServiceCollection(application)
#______________________________________________________________________
# 1. Setup hub server!
import JAP.JAP_LOCAL
import JAP.JAP_LOCAL_SSH
configuration = JAP.JAP_LOCAL.getConfiguration("./JAP_LOCAL_SSH.json", JAP.JAP_LOCAL_SSH.getDefaultConfiguration)
logging.basicConfig()
logger = logging.getLogger("JAP")
if configuration["LOGGER"]["LEVEL"] == "DEBUG":
logger.setLevel(logging.DEBUG)
elif configuration["LOGGER"]["LEVEL"] == "INFO":
logger.setLevel(logging.INFO)
elif configuration["LOGGER"]["LEVEL"] == "WARNING":
logger.setLevel(logging.WARNING)
elif configuration["LOGGER"]["LEVEL"] == "ERROR":
logger.setLevel(logging.ERROR)
elif configuration["LOGGER"]["LEVEL"] == "CRITICAL":
logger.setLevel(logging.CRITICAL)
else:
logger.setLevel(logging.NOTSET)
resolver = JAP.JAP_LOCAL.createResolver(configuration)
reactor.installResolver(resolver)
factory = JAP.JAP_LOCAL_SSH.SSHInputProtocolFactory(configuration)
factory.protocol = JAP.JAP_LOCAL_SSH.SSHInputProtocol
reactor.listenTCP(
configuration["LOCAL_PROXY_SERVER"]["PORT"], factory, 50, configuration["LOCAL_PROXY_SERVER"]["ADDRESS"]
)
reactor.run()
def makeService(options):
from twisted.names.client import getResolver
resolver = getResolver()
# HACK: I want a better resolver than the threaded one, and lack a
# non-terrible place to install it.
from twisted.internet import reactor
if not options['keep-resolver']: # pragma: no cover
reactor.installResolver(resolver)
# HACK: warn about suboptimal reactor usage
if not options['force-select']: # pragma: no cover
from twisted.internet import selectreactor
if isinstance(reactor, selectreactor.SelectReactor):
print('The select reactor is probably a bad idea.')
print('Please use a reactor with better support for lots of fds.')
print('(-r epoll for example)')
print('You can bypass this check using --force-select')
print()
raise ValueError('unfortunate reactor choice')
m = service.MultiService()
checkerFactories = plugin.getCheckerFactories()
default_user_reason = options['conf'].get('user-reason', '')
default_oper_reason = options['conf'].get('oper-reason', '')
scansets = {}
for name, d in options['conf']['scansets'].items():
scans = []
for args in d['protocols']:
poolname = args.pop(0)
checkername = args.pop(0)
checker = checkerFactories[checkername](*args)
scans.append((poolname, checker.check))
user_reason = d.get('user-reason', default_user_reason)
oper_reason = d.get('oper-reason', default_oper_reason)
scansets[name] = scanner.ScanSet(d['timeout'], scans, user_reason,
oper_reason)
# XXXX the target_blah passing here is horrible, but I intend to
# make this less global in the future anyway, so laaaaaaaaater
env = {}
for k in [
'target_ip', 'target_port', 'target_url', 'target_strings',
'max_bytes', 'bind_address'
]:
env[k] = options['conf'].get(k)
theScanner = scanner.Scanner(reactor, resolver, options['conf']['pools'],
scansets, env)
for name, net in options['conf'].get('irc', {}).items():
# XXX terrible, see also similar complaints in ircpresence.
# Split this up later.
factory = ircpresence.Factory(
net['nick'],
net['channel'],
password=net.get('pass'),
opername=net.get('opername', net['nick']),
operpass=net.get('operpass'),
operkey=net.get('operkey'),
opermode=net.get('opermode'),
away=net.get('away'),
connregex=net.get('connregex'),
scanner=theScanner,
masks=options['conf'].get('masks', {}),
actions=net.get('actions'),
onconnectmsgs=net.get('onconnectmsgs', ()),
verbose=options['irc-log'],
flood_exempt=net.get('flood_exempt', False),
username=net.get('username'),
cache_time=net.get("scan-cache-time", 120),
cache_size=net.get("scan-cache-size", 1_000_000))
if net.get('ssl', False):
ctxf = ssl.ClientContextFactory()
serv = internet.SSLClient(net['host'], net['port'], factory, ctxf)
else:
serv = internet.TCPClient(net['host'], net['port'], factory)
serv.setName(name)
serv.setServiceParent(m)
return m
#############################
threadable.init()
if aztk_config.setup.get("site", "environment") == "sandbox":
reactor.suggestThreadPoolSize(3)
else:
try:
reactor.suggestThreadPoolSize(25)
except Exception, ex:
print Exception
print ex
sys.exit(1)
##################################
## Setup Threaded Name Resolver ##
##################################
reactor.installResolver(base.ThreadedResolver(reactor))
__version__ = "$Revision: 1.18 $"
class Application(object):
"""
This is the main application definition. All instances of an AZKT
server will be started with this class.
"""
def __init__(self, profile):
"""
@param profile: Server profile as mentioned in install.cfg
"""
self.profile = profile
def __init__(self):
self.spider_hostnames = defaultdict(set)
self.resolver = _CachingThreadedResolver(reactor)
reactor.installResolver(self.resolver)
dispatcher.connect(self.request_received, signals.request_received)
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
# POSSIBILITY OF SUCH DAMAGE.
###
import supybot.log as log
import supybot.conf as conf
import supybot.drivers as drivers
import supybot.ircmsgs as ircmsgs
from twisted.names import client
from twisted.internet import reactor, error
from twisted.protocols.basic import LineReceiver
from twisted.internet.protocol import ReconnectingClientFactory
# This hack prevents the standard Twisted resolver from starting any
# threads, which allows for a clean shut-down in Twisted>=2.0
reactor.installResolver(client.createResolver())
try:
from OpenSSL import SSL
from twisted.internet import ssl
except ImportError:
drivers.log.debug('PyOpenSSL is not available, '
'cannot connect to SSL servers.')
SSL = None
class TwistedRunnerDriver(drivers.IrcDriver):
def name(self):
return self.__class__.__name__
def run(self):
def start(self):
super(CrawlerProcess, self).start()
if self.dnscache_enable.to_value():
reactor.installResolver(CachingThreadedResolver(reactor))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
from tests.mockserver import MockServer, MockDNSServer
class LocalhostSpider(Spider):
name = "localhost_spider"
def start_requests(self):
yield Request(self.url)
def parse(self, response):
netloc = urlparse(response.url).netloc
self.logger.info("Host: %s" % netloc.split(":")[0])
self.logger.info("Type: %s" % type(response.ip_address))
self.logger.info("IP address: %s" % response.ip_address)
if __name__ == "__main__":
with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server:
port = urlparse(mock_http_server.http_address).port
url = "http://not.a.real.domain:{port}/echo".format(port=port)
servers = [(mock_dns_server.host, mock_dns_server.port)]
reactor.installResolver(createResolver(servers=servers))
configure_logging()
runner = CrawlerRunner()
d = runner.crawl(LocalhostSpider, url=url)
d.addBoth(lambda _: reactor.stop())
reactor.run()
def start(self):
super(CrawlerProcess, self).start()
if self.settings.getbool("DNSCACHE_ENABLED"):
reactor.installResolver(CachingThreadedResolver(reactor))
reactor.addSystemEventTrigger("before", "shutdown", self.stop)
reactor.run(installSignalHandlers=False) # blocking call
