class Connector(base.BaseConnector): _addressType = address.IPv4Address def __init__(self, host, port, factory, timeout, bindAddress, reactor=None): if isinstance(port, types.StringTypes): try: port = socket.getservbyname(port, 'tcp') except socket.error, e: raise error.ServiceNameUnknownError(string="%s (%r)" % (e, port)) # do a host lookup to make sure we have the correct address family try: addressInfo = socket.getaddrinfo(host, port) except socket.gaierror: raise error.DNSLookupError(host) else: assert len(addressInfo) > 0 if addressInfo[0][0] == socket.AF_INET6: self._addressType = address.IPv6Address self.host, self.port = addressInfo[0][4][:2] self.bindAddress = bindAddress base.BaseConnector.__init__(self, factory, timeout, reactor)
def connect(self): """Start connection to remote server.""" self.factory.doStart() self.factory.startedConnecting(self) if not self.servers: if self.domain is None: self.connectionFailed( error.DNSLookupError("Domain is not defined."), ) return d = client.lookupService( "_%s._%s.%s" % ( nativeString(self.service), nativeString(self.protocol), nativeString(self.domain), ), ) d.addCallbacks(self._cbGotServers, self._ebGotServers) d.addCallback(lambda x, self=self: self._reallyConnect()) if self._defaultPort: d.addErrback(self._ebServiceUnknown) d.addErrback(self.connectionFailed) elif self.connector is None: self._reallyConnect() else: self.connector.connect()
def getHostByName(self, name, timeout = (1, 3, 11, 45)): try: address = socket.gethostbyname(name) except socket.error: msg = "address %r not found" % (name,) err = error.DNSLookupError(msg) return defer.fail(err) else: return defer.succeed(address)
def test_robotstxt_immediate_error(self): self.crawler.settings.set('ROBOTSTXT_OBEY', True) err = error.DNSLookupError('Robotstxt address not found') def immediate_failure(request, spider): deferred = Deferred() deferred.errback(failure.Failure(err)) return deferred self.crawler.engine.download.side_effect = immediate_failure middleware = RobotsTxtMiddleware(self.crawler) return self.assertNotIgnored(Request('http://site.local'), middleware)
def getHostByName( self, name: str, timeout: Sequence[int] = (1, 3, 11, 45)) -> Deferred[str]: try: address = socket.gethostbyname(name) except OSError: msg = f"address {name!r} not found" err = error.DNSLookupError(msg) return defer.fail(err) else: return defer.succeed(address)
def query_complete(address, name): if address is None or address == "": from twisted.internet import error msg = "address %r not found" % (name, ) err = error.DNSLookupError(msg) from twisted.internet import defer return defer.fail(err) else: return address
def test_robotstxt_error(self): self.crawler.settings.set('ROBOTSTXT_OBEY', True) err = error.DNSLookupError('Robotstxt address not found') def return_failure(request, spider): deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(err)) return deferred self.crawler.engine.download.side_effect = return_failure middleware = RobotsTxtMiddleware(self.crawler) middleware._logerror = mock.MagicMock(side_effect=middleware._logerror) deferred = middleware.process_request(Request('http://site.local'), None) deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called)) return deferred
def testFallbackFailsFirstTime(self): """Test a cache with fallback.""" original = mox.MockAnything() original.getHostByName('google.com').AndReturn( defer.fail(failure.Failure(error.DNSLookupError('Fake DNS failure')))) original.getHostByName('google.com').AndReturn( defer.succeed('1.2.3.4')) mox.Replay(original) cache = dnsCache.CachingDNS(original, timeout = 1) result = cache.getHostByName('google.com') self.assertTrue(isinstance(result.result, failure.Failure)) result.addErrback(lambda _: None) # Consume the error. result = cache.getHostByName('google.com') self.assertEquals(result.result, '1.2.3.4') mox.Verify(original)
def _cbGotServers(self, result): answers, auth, add = result if len(answers) == 1 and answers[0].type == dns.SRV \ and answers[0].payload \ and answers[0].payload.target == dns.Name(b'.'): # decidedly not available raise error.DNSLookupError("Service %s not available for domain %s." % (repr(self.service), repr(self.domain))) self.servers = [] self.orderedServers = [] for a in answers: if a.type != dns.SRV or not a.payload: continue self.orderedServers.append(a.payload)
def testTwoTries(self): """Test a two try cache.""" original = mox.MockAnything() original.getHostByName('google.com').AndReturn( defer.fail( failure.Failure(error.DNSLookupError('Fake DNS failure')))) original.getHostByName('google.com').AndReturn( defer.succeed('1.2.3.4')) mox.Replay(original) cache = dnsRetry.RetryingDNS(original, tries=2, sleep=time.SleepManager(0, 0, 0)) result = cache.getHostByName('google.com') self.assertEquals(result, '1.2.3.4') mox.Verify(original)
def testOneTry(self): """Test a single try cache.""" original = mox.MockAnything() original.getHostByName('google.com').AndReturn( defer.fail( failure.Failure(error.DNSLookupError('Fake DNS failure')))) original.getHostByName('google.com').AndReturn( defer.succeed('1.2.3.4')) mox.Replay(original) cache = dnsRetry.RetryingDNS(original, tries=1) self.assertRaises(error.DNSLookupError, cache.getHostByName, 'google.com') result = cache.getHostByName('google.com') self.assertEquals(result, '1.2.3.4') mox.Verify(original)
def testFallback(self): """Test a cache with fallback.""" original = mox.MockAnything() original.getHostByName('google.com').AndReturn( defer.succeed('1.2.3.4')) original.getHostByName('google.com').AndReturn( defer.fail(failure.Failure(error.DNSLookupError('Fake DNS failure')))) original.getHostByName('google.com').AndReturn( defer.succeed('9.8.7.6')) mox.Replay(original) cache = dnsCache.CachingDNS(original, timeout = 0) result = cache.getHostByName('google.com') self.assertEquals(result.result, '1.2.3.4') result = cache.getHostByName('google.com') self.assertEquals(result.result, '1.2.3.4') result = cache.getHostByName('google.com') self.assertEquals(result.result, '9.8.7.6') mox.Verify(original)
def connect(self): """Start connection to remote server.""" self.factory.doStart() self.factory.startedConnecting(self) if not self.servers: if self.domain is None: self.connectionFailed(error.DNSLookupError("Domain is not defined.")) return d = client.lookupService('_%s._%s.%s' % (self.service, self.protocol, self.domain)) d.addCallbacks(self._cbGotServers, self._ebGotServers) d.addCallback(lambda x, self=self: self._reallyConnect()) d.addErrback(self.connectionFailed) elif self.connector is None: self._reallyConnect() else: self.connector.connect()
def _cbRecords(self, records, name, effort): (ans, auth, add) = records result = extractRecord(self, dns.Name(name), ans + auth + add, effort) if not result: raise error.DNSLookupError(name) return result
def _fail(self, name: str, err: str) -> Failure: lookupError = error.DNSLookupError( f"address {name!r} not found: {err}") return Failure(lookupError)
class SRVConnector: """A connector that looks up DNS SRV records. See RFC2782.""" implements(interfaces.IConnector) stopAfterDNS = 0 def __init__( self, reactor, service, domain, factory, protocol='tcp', connectFuncName='connectTCP', connectFuncArgs=(), connectFuncKwArgs={}, defaultPort=None, ): """ @param domain: The domain to connect to. If passed as a unicode string, it will be encoded using C{idna} encoding. @type domain: L{bytes} or L{unicode} @param defaultPort: Optional default port number to be used when SRV lookup fails and the service name is unknown. This should be the port number associated with the service name as defined by the IANA registry. @type defaultPort: C{int} """ self.reactor = reactor self.service = service if isinstance(domain, unicode): domain = domain.encode('idna') self.domain = domain self.factory = factory self.protocol = protocol self.connectFuncName = connectFuncName self.connectFuncArgs = connectFuncArgs self.connectFuncKwArgs = connectFuncKwArgs self._defaultPort = defaultPort self.connector = None self.servers = None self.orderedServers = None # list of servers already used in this round def connect(self): """Start connection to remote server.""" self.factory.doStart() self.factory.startedConnecting(self) if not self.servers: if self.domain is None: self.connectionFailed( error.DNSLookupError("Domain is not defined.")) return d = client.lookupService( '_%s._%s.%s' % (self.service, self.protocol, self.domain)) d.addCallbacks(self._cbGotServers, self._ebGotServers) d.addCallback(lambda x, self=self: self._reallyConnect()) if self._defaultPort: d.addErrback(self._ebServiceUnknown) d.addErrback(self.connectionFailed) elif self.connector is None: self._reallyConnect() else: self.connector.connect() def _ebGotServers(self, failure): failure.trap(DNSNameError) # Some DNS servers reply with NXDOMAIN when in fact there are # just no SRV records for that domain. Act as if we just got an # empty response and use fallback. self.servers = [] self.orderedServers = [] def _cbGotServers(self, (answers, auth, add)): if len(answers) == 1 and answers[0].type == dns.SRV \ and answers[0].payload \ and answers[0].payload.target == dns.Name('.'): # decidedly not available raise error.DNSLookupError( "Service %s not available for domain %s." % (repr(self.service), repr(self.domain))) self.servers = [] self.orderedServers = [] for a in answers: if a.type != dns.SRV or not a.payload: continue self.orderedServers.append((a.payload.priority, a.payload.weight, str(a.payload.target), a.payload.port))
class SRVConnector: """A connector that looks up DNS SRV records. See RFC2782.""" implements(interfaces.IConnector) stopAfterDNS=0 def __init__(self, reactor, service, domain, factory, protocol='tcp', connectFuncName='connectTCP', connectFuncArgs=(), connectFuncKwArgs={}, ): self.reactor = reactor self.service = service self.domain = domain self.factory = factory self.protocol = protocol self.connectFuncName = connectFuncName self.connectFuncArgs = connectFuncArgs self.connectFuncKwArgs = connectFuncKwArgs self.connector = None self.servers = None self.orderedServers = None # list of servers already used in this round def connect(self): """Start connection to remote server.""" self.factory.doStart() self.factory.startedConnecting(self) if not self.servers: if self.domain is None: self.connectionFailed(error.DNSLookupError("Domain is not defined.")) return d = client.lookupService('_%s._%s.%s' % (self.service, self.protocol, self.domain)) d.addCallbacks(self._cbGotServers, self._ebGotServers) d.addCallback(lambda x, self=self: self._reallyConnect()) d.addErrback(self.connectionFailed) elif self.connector is None: self._reallyConnect() else: self.connector.connect() def _ebGotServers(self, failure): failure.trap(DNSNameError) # Some DNS servers reply with NXDOMAIN when in fact there are # just no SRV records for that domain. Act as if we just got an # empty response and use fallback. self.servers = [] self.orderedServers = [] def _cbGotServers(self, (answers, auth, add)): if len(answers) == 1 and answers[0].type == dns.SRV \ and answers[0].payload \ and answers[0].payload.target == dns.Name('.'): # decidedly not available raise error.DNSLookupError("Service %s not available for domain %s." % (repr(self.service), repr(self.domain))) self.servers = [] self.orderedServers = [] for a in answers: if a.type != dns.SRV or not a.payload: continue self.orderedServers.append((a.payload.priority, a.payload.weight, str(a.payload.target), a.payload.port))
def _fail(self, name, err): err = error.DNSLookupError("address %r not found: %s" % (name, err)) return failure.Failure(err)
@see: twisted.names.client.lookupAllRecords """ return self._lookup(name, dns.IN, dns.ALL_RECORDS, timeout) def getHostByName(self, name, timeout=None, effort=10): """ @see: twisted.names.client.getHostByName """ # XXX - respect timeout return self.lookupAllRecords(name, timeout).addCallback( self._cbRecords, name, effort) def _cbRecords(self, (ans, auth, add), name, effort): result = extractRecord(self, dns.Name(name), ans + auth + add, effort) if not result: raise error.DNSLookupError(name) return result def extractRecord(resolver, name, answers, level=10): if not level: return None if hasattr(socket, 'inet_ntop'): for r in answers: if r.name == name and r.type == dns.A6: return socket.inet_ntop(socket.AF_INET6, r.payload.address) for r in answers: if r.name == name and r.type == dns.AAAA: return socket.inet_ntop(socket.AF_INET6, r.payload.address) for r in answers: if r.name == name and r.type == dns.A:
def resolve(name, timeout=(1, 3, 11, 45)): if name not in MOCK_DHT_SEED_DNS: return defer.fail(error.DNSLookupError(name)) return defer.succeed(MOCK_DHT_SEED_DNS[name])
class SRVConnector: """A connector that looks up DNS SRV records. See RFC2782.""" implements(interfaces.IConnector) stopAfterDNS = 0 def __init__( self, reactor, service, domain, factory, protocol='tcp', connectFuncName='connectTCP', connectFuncArgs=(), connectFuncKwArgs={}, ): self.reactor = reactor self.service = service self.domain = domain self.factory = factory self.protocol = protocol self.connectFuncName = connectFuncName self.connectFuncArgs = connectFuncArgs self.connectFuncKwArgs = connectFuncKwArgs self.connector = None self.servers = None self.orderedServers = None # list of servers already used in this round def connect(self): """Start connection to remote server.""" self.factory.doStart() self.factory.startedConnecting(self) if not self.servers: if self.domain is None: self.connectionFailed( error.DNSLookupError("Domain is not defined.")) return d = client.lookupService( '_%s._%s.%s' % (self.service, self.protocol, self.domain)) d.addCallback(self._cbGotServers) d.addCallback(lambda x, self=self: self._reallyConnect()) d.addErrback(self.connectionFailed) elif self.connector is None: self._reallyConnect() else: self.connector.connect() def _cbGotServers(self, (answers, auth, add)): if len(answers) == 1 and answers[0].payload.target == '.': # decidedly not available raise error.DNSLookupError( "Service %s not available for domain %s." % (repr(self.service), repr(self.domain))) self.servers = [] self.orderedServers = [] for a in answers: self.orderedServers.append((a.payload.priority, a.payload.weight, str(a.payload.target), a.payload.port))