class Connector(base.BaseConnector):
_addressType = address.IPv4Address
def __init__(self,
host,
port,
factory,
timeout,
bindAddress,
reactor=None):
if isinstance(port, types.StringTypes):
try:
port = socket.getservbyname(port, 'tcp')
except socket.error, e:
raise error.ServiceNameUnknownError(string="%s (%r)" %
(e, port))
# do a host lookup to make sure we have the correct address family
try:
addressInfo = socket.getaddrinfo(host, port)
except socket.gaierror:
raise error.DNSLookupError(host)
else:
assert len(addressInfo) > 0
if addressInfo[0][0] == socket.AF_INET6:
self._addressType = address.IPv6Address
self.host, self.port = addressInfo[0][4][:2]
self.bindAddress = bindAddress
base.BaseConnector.__init__(self, factory, timeout, reactor)
def connect(self):
"""Start connection to remote server."""
self.factory.doStart()
self.factory.startedConnecting(self)
if not self.servers:
if self.domain is None:
self.connectionFailed(
error.DNSLookupError("Domain is not defined."),
)
return
d = client.lookupService(
"_%s._%s.%s"
% (
nativeString(self.service),
nativeString(self.protocol),
nativeString(self.domain),
),
)
d.addCallbacks(self._cbGotServers, self._ebGotServers)
d.addCallback(lambda x, self=self: self._reallyConnect())
if self._defaultPort:
d.addErrback(self._ebServiceUnknown)
d.addErrback(self.connectionFailed)
elif self.connector is None:
self._reallyConnect()
else:
self.connector.connect()
def getHostByName(self, name, timeout = (1, 3, 11, 45)):
try:
address = socket.gethostbyname(name)
except socket.error:
msg = "address %r not found" % (name,)
err = error.DNSLookupError(msg)
return defer.fail(err)
else:
return defer.succeed(address)
def test_robotstxt_immediate_error(self):
self.crawler.settings.set('ROBOTSTXT_OBEY', True)
err = error.DNSLookupError('Robotstxt address not found')
def immediate_failure(request, spider):
deferred = Deferred()
deferred.errback(failure.Failure(err))
return deferred
self.crawler.engine.download.side_effect = immediate_failure
middleware = RobotsTxtMiddleware(self.crawler)
return self.assertNotIgnored(Request('http://site.local'), middleware)
def getHostByName(
self, name: str,
timeout: Sequence[int] = (1, 3, 11, 45)) -> Deferred[str]:
try:
address = socket.gethostbyname(name)
except OSError:
msg = f"address {name!r} not found"
err = error.DNSLookupError(msg)
return defer.fail(err)
else:
return defer.succeed(address)
def query_complete(address, name):
if address is None or address == "":
from twisted.internet import error
msg = "address %r not found" % (name, )
err = error.DNSLookupError(msg)
from twisted.internet import defer
return defer.fail(err)
else:
return address
def test_robotstxt_error(self):
self.crawler.settings.set('ROBOTSTXT_OBEY', True)
err = error.DNSLookupError('Robotstxt address not found')
def return_failure(request, spider):
deferred = Deferred()
reactor.callFromThread(deferred.errback, failure.Failure(err))
return deferred
self.crawler.engine.download.side_effect = return_failure
middleware = RobotsTxtMiddleware(self.crawler)
middleware._logerror = mock.MagicMock(side_effect=middleware._logerror)
deferred = middleware.process_request(Request('http://site.local'), None)
deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called))
return deferred
def testFallbackFailsFirstTime(self):
"""Test a cache with fallback."""
original = mox.MockAnything()
original.getHostByName('google.com').AndReturn(
defer.fail(failure.Failure(error.DNSLookupError('Fake DNS failure'))))
original.getHostByName('google.com').AndReturn(
defer.succeed('1.2.3.4'))
mox.Replay(original)
cache = dnsCache.CachingDNS(original, timeout = 1)
result = cache.getHostByName('google.com')
self.assertTrue(isinstance(result.result, failure.Failure))
result.addErrback(lambda _: None) # Consume the error.
result = cache.getHostByName('google.com')
self.assertEquals(result.result, '1.2.3.4')
mox.Verify(original)
def _cbGotServers(self, result):
answers, auth, add = result
if len(answers) == 1 and answers[0].type == dns.SRV \
and answers[0].payload \
and answers[0].payload.target == dns.Name(b'.'):
# decidedly not available
raise error.DNSLookupError("Service %s not available for domain %s."
% (repr(self.service), repr(self.domain)))
self.servers = []
self.orderedServers = []
for a in answers:
if a.type != dns.SRV or not a.payload:
continue
self.orderedServers.append(a.payload)
def testTwoTries(self):
"""Test a two try cache."""
original = mox.MockAnything()
original.getHostByName('google.com').AndReturn(
defer.fail(
failure.Failure(error.DNSLookupError('Fake DNS failure'))))
original.getHostByName('google.com').AndReturn(
defer.succeed('1.2.3.4'))
mox.Replay(original)
cache = dnsRetry.RetryingDNS(original,
tries=2,
sleep=time.SleepManager(0, 0, 0))
result = cache.getHostByName('google.com')
self.assertEquals(result, '1.2.3.4')
mox.Verify(original)
def testOneTry(self):
"""Test a single try cache."""
original = mox.MockAnything()
original.getHostByName('google.com').AndReturn(
defer.fail(
failure.Failure(error.DNSLookupError('Fake DNS failure'))))
original.getHostByName('google.com').AndReturn(
defer.succeed('1.2.3.4'))
mox.Replay(original)
cache = dnsRetry.RetryingDNS(original, tries=1)
self.assertRaises(error.DNSLookupError, cache.getHostByName,
'google.com')
result = cache.getHostByName('google.com')
self.assertEquals(result, '1.2.3.4')
mox.Verify(original)
def testFallback(self):
"""Test a cache with fallback."""
original = mox.MockAnything()
original.getHostByName('google.com').AndReturn(
defer.succeed('1.2.3.4'))
original.getHostByName('google.com').AndReturn(
defer.fail(failure.Failure(error.DNSLookupError('Fake DNS failure'))))
original.getHostByName('google.com').AndReturn(
defer.succeed('9.8.7.6'))
mox.Replay(original)
cache = dnsCache.CachingDNS(original, timeout = 0)
result = cache.getHostByName('google.com')
self.assertEquals(result.result, '1.2.3.4')
result = cache.getHostByName('google.com')
self.assertEquals(result.result, '1.2.3.4')
result = cache.getHostByName('google.com')
self.assertEquals(result.result, '9.8.7.6')
mox.Verify(original)
def connect(self):
"""Start connection to remote server."""
self.factory.doStart()
self.factory.startedConnecting(self)
if not self.servers:
if self.domain is None:
self.connectionFailed(error.DNSLookupError("Domain is not defined."))
return
d = client.lookupService('_%s._%s.%s' % (self.service,
self.protocol,
self.domain))
d.addCallbacks(self._cbGotServers, self._ebGotServers)
d.addCallback(lambda x, self=self: self._reallyConnect())
d.addErrback(self.connectionFailed)
elif self.connector is None:
self._reallyConnect()
else:
self.connector.connect()
def _cbRecords(self, records, name, effort):
(ans, auth, add) = records
result = extractRecord(self, dns.Name(name), ans + auth + add, effort)
if not result:
raise error.DNSLookupError(name)
return result
def _fail(self, name: str, err: str) -> Failure:
lookupError = error.DNSLookupError(
f"address {name!r} not found: {err}")
return Failure(lookupError)
class SRVConnector:
"""A connector that looks up DNS SRV records. See RFC2782."""
implements(interfaces.IConnector)
stopAfterDNS = 0
def __init__(
self,
reactor,
service,
domain,
factory,
protocol='tcp',
connectFuncName='connectTCP',
connectFuncArgs=(),
connectFuncKwArgs={},
defaultPort=None,
):
"""
@param domain: The domain to connect to. If passed as a unicode
string, it will be encoded using C{idna} encoding.
@type domain: L{bytes} or L{unicode}
@param defaultPort: Optional default port number to be used when SRV
lookup fails and the service name is unknown. This should be the
port number associated with the service name as defined by the IANA
registry.
@type defaultPort: C{int}
"""
self.reactor = reactor
self.service = service
if isinstance(domain, unicode):
domain = domain.encode('idna')
self.domain = domain
self.factory = factory
self.protocol = protocol
self.connectFuncName = connectFuncName
self.connectFuncArgs = connectFuncArgs
self.connectFuncKwArgs = connectFuncKwArgs
self._defaultPort = defaultPort
self.connector = None
self.servers = None
self.orderedServers = None # list of servers already used in this round
def connect(self):
"""Start connection to remote server."""
self.factory.doStart()
self.factory.startedConnecting(self)
if not self.servers:
if self.domain is None:
self.connectionFailed(
error.DNSLookupError("Domain is not defined."))
return
d = client.lookupService(
'_%s._%s.%s' % (self.service, self.protocol, self.domain))
d.addCallbacks(self._cbGotServers, self._ebGotServers)
d.addCallback(lambda x, self=self: self._reallyConnect())
if self._defaultPort:
d.addErrback(self._ebServiceUnknown)
d.addErrback(self.connectionFailed)
elif self.connector is None:
self._reallyConnect()
else:
self.connector.connect()
def _ebGotServers(self, failure):
failure.trap(DNSNameError)
# Some DNS servers reply with NXDOMAIN when in fact there are
# just no SRV records for that domain. Act as if we just got an
# empty response and use fallback.
self.servers = []
self.orderedServers = []
def _cbGotServers(self, (answers, auth, add)):
if len(answers) == 1 and answers[0].type == dns.SRV \
and answers[0].payload \
and answers[0].payload.target == dns.Name('.'):
# decidedly not available
raise error.DNSLookupError(
"Service %s not available for domain %s." %
(repr(self.service), repr(self.domain)))
self.servers = []
self.orderedServers = []
for a in answers:
if a.type != dns.SRV or not a.payload:
continue
self.orderedServers.append((a.payload.priority, a.payload.weight,
str(a.payload.target), a.payload.port))
class SRVConnector:
"""A connector that looks up DNS SRV records. See RFC2782."""
implements(interfaces.IConnector)
stopAfterDNS=0
def __init__(self, reactor, service, domain, factory,
protocol='tcp', connectFuncName='connectTCP',
connectFuncArgs=(),
connectFuncKwArgs={},
):
self.reactor = reactor
self.service = service
self.domain = domain
self.factory = factory
self.protocol = protocol
self.connectFuncName = connectFuncName
self.connectFuncArgs = connectFuncArgs
self.connectFuncKwArgs = connectFuncKwArgs
self.connector = None
self.servers = None
self.orderedServers = None # list of servers already used in this round
def connect(self):
"""Start connection to remote server."""
self.factory.doStart()
self.factory.startedConnecting(self)
if not self.servers:
if self.domain is None:
self.connectionFailed(error.DNSLookupError("Domain is not defined."))
return
d = client.lookupService('_%s._%s.%s' % (self.service,
self.protocol,
self.domain))
d.addCallbacks(self._cbGotServers, self._ebGotServers)
d.addCallback(lambda x, self=self: self._reallyConnect())
d.addErrback(self.connectionFailed)
elif self.connector is None:
self._reallyConnect()
else:
self.connector.connect()
def _ebGotServers(self, failure):
failure.trap(DNSNameError)
# Some DNS servers reply with NXDOMAIN when in fact there are
# just no SRV records for that domain. Act as if we just got an
# empty response and use fallback.
self.servers = []
self.orderedServers = []
def _cbGotServers(self, (answers, auth, add)):
if len(answers) == 1 and answers[0].type == dns.SRV \
and answers[0].payload \
and answers[0].payload.target == dns.Name('.'):
# decidedly not available
raise error.DNSLookupError("Service %s not available for domain %s."
% (repr(self.service), repr(self.domain)))
self.servers = []
self.orderedServers = []
for a in answers:
if a.type != dns.SRV or not a.payload:
continue
self.orderedServers.append((a.payload.priority, a.payload.weight,
str(a.payload.target), a.payload.port))
def _fail(self, name, err):
err = error.DNSLookupError("address %r not found: %s" % (name, err))
return failure.Failure(err)
@see: twisted.names.client.lookupAllRecords
"""
return self._lookup(name, dns.IN, dns.ALL_RECORDS, timeout)
def getHostByName(self, name, timeout=None, effort=10):
"""
@see: twisted.names.client.getHostByName
"""
# XXX - respect timeout
return self.lookupAllRecords(name, timeout).addCallback(
self._cbRecords, name, effort)
def _cbRecords(self, (ans, auth, add), name, effort):
result = extractRecord(self, dns.Name(name), ans + auth + add, effort)
if not result:
raise error.DNSLookupError(name)
return result
def extractRecord(resolver, name, answers, level=10):
if not level:
return None
if hasattr(socket, 'inet_ntop'):
for r in answers:
if r.name == name and r.type == dns.A6:
return socket.inet_ntop(socket.AF_INET6, r.payload.address)
for r in answers:
if r.name == name and r.type == dns.AAAA:
return socket.inet_ntop(socket.AF_INET6, r.payload.address)
for r in answers:
if r.name == name and r.type == dns.A:
def resolve(name, timeout=(1, 3, 11, 45)):
if name not in MOCK_DHT_SEED_DNS:
return defer.fail(error.DNSLookupError(name))
return defer.succeed(MOCK_DHT_SEED_DNS[name])
class SRVConnector:
"""A connector that looks up DNS SRV records. See RFC2782."""
implements(interfaces.IConnector)
stopAfterDNS = 0
def __init__(
self,
reactor,
service,
domain,
factory,
protocol='tcp',
connectFuncName='connectTCP',
connectFuncArgs=(),
connectFuncKwArgs={},
):
self.reactor = reactor
self.service = service
self.domain = domain
self.factory = factory
self.protocol = protocol
self.connectFuncName = connectFuncName
self.connectFuncArgs = connectFuncArgs
self.connectFuncKwArgs = connectFuncKwArgs
self.connector = None
self.servers = None
self.orderedServers = None # list of servers already used in this round
def connect(self):
"""Start connection to remote server."""
self.factory.doStart()
self.factory.startedConnecting(self)
if not self.servers:
if self.domain is None:
self.connectionFailed(
error.DNSLookupError("Domain is not defined."))
return
d = client.lookupService(
'_%s._%s.%s' % (self.service, self.protocol, self.domain))
d.addCallback(self._cbGotServers)
d.addCallback(lambda x, self=self: self._reallyConnect())
d.addErrback(self.connectionFailed)
elif self.connector is None:
self._reallyConnect()
else:
self.connector.connect()
def _cbGotServers(self, (answers, auth, add)):
if len(answers) == 1 and answers[0].payload.target == '.':
# decidedly not available
raise error.DNSLookupError(
"Service %s not available for domain %s." %
(repr(self.service), repr(self.domain)))
self.servers = []
self.orderedServers = []
for a in answers:
self.orderedServers.append((a.payload.priority, a.payload.weight,
str(a.payload.target), a.payload.port))