def performALookup(self, hostname, dns_server):
"""
Performs an A lookup and returns an array containg all the dotted quad
IP addresses in the response.
:hostname: is the hostname to perform the A lookup on
:dns_server: is the dns_server that should be used for the lookup as a
tuple of ip port (ex. ("127.0.0.1", 53))
"""
query = [dns.Query(hostname, dns.A, dns.IN)]
def gotResponse(message):
addrs = []
answers = []
for answer in message.answers:
if answer.type is 1:
addr = answer.payload.dottedQuad()
addrs.append(addr)
answers.append(representAnswer(answer))
DNSTest.addToReport(self, query, resolver=dns_server, query_type='A',
answers=answers, addrs=addrs)
return addrs
def gotError(failure):
failure.trap(gaierror, TimeoutError)
DNSTest.addToReport(self, query, resolver=dns_server, query_type='A',
failure=failure)
return failure
resolver = Resolver(servers=[dns_server])
d = resolver.queryUDP(query, timeout=self.queryTimeout)
d.addCallback(gotResponse)
d.addErrback(gotError)
return d
def __init__(self, servers, reactor):
self.serverHosts = {}
nonHostnameServers = []
for host, ip, port in servers:
self.serverHosts[ip, port] = host
nonHostnameServers.append((ip, port))
NonrecursiveResolver.__init__(self, servers=nonHostnameServers, reactor=reactor)
def test_no_answer(self):
"""
If a request returns a L{dns.NS} response, but we can't connect to the
given server, the request fails with the error returned at connection.
"""
def query(self, *args):
# Pop from the message list, so that it blows up if more queries
# are run than expected.
return succeed(messages.pop(0))
def queryProtocol(self, *args, **kwargs):
return defer.fail(socket.gaierror("Couldn't connect"))
resolver = Resolver(servers=[('0.0.0.0', 0)])
resolver._query = query
messages = []
# Let's patch dns.DNSDatagramProtocol.query, as there is no easy way to
# customize it.
self.patch(dns.DNSDatagramProtocol, "query", queryProtocol)
records = [
dns.RRHeader(name='fooba.com', type=dns.NS, cls=dns.IN, ttl=700,
auth=False,
payload=dns.Record_NS(name='ns.twistedmatrix.com',
ttl=700))]
m = dns.Message(id=999, answer=1, opCode=0, recDes=0, recAv=1, auth=1,
rCode=0, trunc=0, maxSize=0)
m.answers = records
messages.append(m)
return self.assertFailure(
resolver.getHostByName("fooby.com"), socket.gaierror)
def test_noAnswer(self):
"""
If a request returns a L{dns.NS} response, but we can't connect to the
given server, the request fails with the error returned at connection.
"""
def query(self, *args):
# Pop from the message list, so that it blows up if more queries
# are run than expected.
return succeed(messages.pop(0))
def queryProtocol(self, *args, **kwargs):
return defer.fail(socket.gaierror("Couldn't connect"))
resolver = Resolver(servers=[('0.0.0.0', 0)])
resolver._query = query
messages = []
# Let's patch dns.DNSDatagramProtocol.query, as there is no easy way to
# customize it.
self.patch(dns.DNSDatagramProtocol, "query", queryProtocol)
records = [
dns.RRHeader(name='fooba.com', type=dns.NS, cls=dns.IN, ttl=700,
auth=False,
payload=dns.Record_NS(name='ns.twistedmatrix.com',
ttl=700))]
m = dns.Message(id=999, answer=1, opCode=0, recDes=0, recAv=1, auth=1,
rCode=0, trunc=0, maxSize=0)
m.answers = records
messages.append(m)
return self.assertFailure(
resolver.getHostByName("fooby.com"), socket.gaierror)
def __init__(self,
postgresListener,
clock=reactor,
retryOnFailure=True,
rbacRetryOnFailureDelay=10):
"""Initialise a new `RegionControllerService`.
:param postgresListener: The `PostgresListenerService` that is running
in this regiond process.
"""
super(RegionControllerService, self).__init__()
self.clock = clock
self.retryOnFailure = retryOnFailure
self.rbacRetryOnFailureDelay = rbacRetryOnFailureDelay
self.processing = LoopingCall(self.process)
self.processing.clock = self.clock
self.processingDefer = None
self.needsDNSUpdate = False
self.needsProxyUpdate = False
self.needsRBACUpdate = False
self.postgresListener = postgresListener
self.dnsResolver = Resolver(resolv=None,
servers=[('127.0.0.1', 53)],
timeout=(1, ),
reactor=clock)
self.previousSerial = None
self.rbacClient = None
self.rbacInit = False
def test_zoneTransferConnectionFails(self):
"""
A failed AXFR TCP connection errbacks the L{Deferred} returned
from L{Resolver.lookupZone}.
"""
resolver = Resolver(servers=[("nameserver.invalid", 53)])
return self.assertFailure(resolver.lookupZone("impossible.invalid"),
error.DNSLookupError)
def test_zoneTransferConnectionFails(self):
"""
A failed AXFR TCP connection errbacks the L{Deferred} returned
from L{Resolver.lookupZone}.
"""
resolver = Resolver(servers=[("nameserver.invalid", 53)])
return self.assertFailure(resolver.lookupZone("impossible.invalid"),
error.DNSLookupError)
def __init__(self, flag=False, servers=[('8.8.8.8', 53)], timeout=(2, 3, 5, 10)):
Resolver.__init__(self, servers=servers, timeout=timeout)
self.flag = flag
self.servers = servers
# load TLD nameservers
if self.flag:
from pickle import load
self.tld_servers = load(open('/usr/local/share/tld_nameservers.pkl'))
def __init__(self, servers, reactor):
self.serverHosts = {}
nonHostnameServers = []
for host, ip, port in servers:
self.serverHosts[ip, port] = host
nonHostnameServers.append((ip, port))
NonrecursiveResolver.__init__(self,
servers=nonHostnameServers,
reactor=reactor)
def test_a_lookup_ooni_query(self):
def done_query(message, *arg):
answer = message.answers[0]
self.assertEqual(answer.type, 1)
dns_query = [dns.Query('ooni.nu', type=dns.A)]
resolver = Resolver(servers=[('8.8.8.8', 53)])
d = resolver.queryUDP(dns_query)
d.addCallback(done_query)
return d
def dnsLookup(self, hostname, dns_type, dns_server = None):
"""
Performs a DNS lookup and returns the response.
:hostname: is the hostname to perform the DNS lookup on
:dns_type: type of lookup 'NS'/'A'/'SOA'
:dns_server: is the dns_server that should be used for the lookup as a
tuple of ip port (ex. ("127.0.0.1", 53))
"""
types={'NS':dns.NS,'A':dns.A,'SOA':dns.SOA,'PTR':dns.PTR}
dnsType=types[dns_type]
query = [dns.Query(hostname, dnsType, dns.IN)]
def gotResponse(message):
log.debug(dns_type+" Lookup successful")
log.debug(str(message))
addrs = []
answers = []
if dns_server:
msg = message.answers
else:
msg = message[0]
for answer in msg:
if answer.type is dnsType:
if dnsType is dns.SOA:
addr = (answer.name.name,answer.payload.serial)
elif dnsType in [dns.NS,dns.PTR]:
addr = answer.payload.name.name
elif dnsType is dns.A:
addr = answer.payload.dottedQuad()
else:
addr = None
addrs.append(addr)
answers.append(representAnswer(answer))
DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type,
answers=answers, addrs=addrs)
return addrs
def gotError(failure):
failure.trap(gaierror, TimeoutError)
DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type,
failure=failure)
return failure
if dns_server:
resolver = Resolver(servers=[dns_server])
d = resolver.queryUDP(query, timeout=self.queryTimeout)
else:
lookupFunction={'NS':client.lookupNameservers, 'SOA':client.lookupAuthority, 'A':client.lookupAddress, 'PTR':client.lookupPointer}
d = lookupFunction[dns_type](hostname)
d.addCallback(gotResponse)
d.addErrback(gotError)
return d
def __init__(self, flag=False, servers=[('8.8.8.8', 53)], timeout=(2, 3, 5, 10)):
Resolver.__init__(self, servers=servers, timeout=timeout)
self.flag = flag
self.servers = servers
# load TLD nameservers
if self.flag:
if pkg_resources.resource_exists('hydra_resolver', 'data/tld_nameservers.pkl'):
data = pkg_resources.resource_stream('hydra_resolver', 'data/tld_nameservers.pkl')
self.tld_servers = load(data)
else:
raise EnvironmentError('Unable to load tld_nameservers.pkl')
def __init__(self,
flag=False,
servers=[('8.8.8.8', 53)],
timeout=(2, 3, 5, 10)):
Resolver.__init__(self, servers=servers, timeout=timeout)
self.flag = flag
self.servers = servers
# load TLD nameservers
if self.flag:
from pickle import load
self.tld_servers = load(
open('/usr/local/share/tld_nameservers.pkl'))
def test_lookup(self):
"""
We perform an A lookup on the DNS test servers for the domains to be
tested and an A lookup on the known good DNS server.
We then compare the results from test_resolvers and that from
control_resolver and see if the match up.
If they match up then no censorship is happening (tampering: false).
If they do not we do a reverse lookup (PTR) on the test_resolvers and
the control resolver for every IP address we got back and check to see
if anyone of them matches the control ones.
If they do then we take not of the fact that censorship is probably not
happening (tampering: reverse-match).
If they do not match then censorship is probably going on (tampering:
true).
"""
log.msg("Doing the test lookups on %s" % self.input)
list_of_ds = []
hostname = self.input
dns_query = [dns.Query(hostname, dns.IN, dns.A)]
dns_server = [(self.localOptions['backend'],
self.localOptions['backendport'])]
resolver = Resolver(servers=dns_server)
control_d = resolver.queryUDP(dns_query, timeout=self.lookupTimeout)
control_d.addCallback(self.process_a_answers, 'control')
control_d.addErrback(self.a_lookup_error, 'control')
for test_resolver in self.test_resolvers:
log.msg("Going for %s" % test_resolver)
dns_server = [(test_resolver, 53)]
resolver = Resolver(servers=dns_server)
d = resolver.queryUDP(dns_query, timeout=self.lookupTimeout)
d.addCallback(self.process_a_answers, test_resolver)
d.addErrback(self.a_lookup_error, test_resolver)
# This is required to cancel the delayed calls of the
# twisted.names.client resolver
list_of_ds.append(d)
list_of_ds.append(control_d)
dl = defer.DeferredList(list_of_ds)
dl.addCallback(self.do_reverse_lookups)
dl.addBoth(self.compare_results)
return dl
def createResolver(servers=None, resolvconf=None, hosts=None):
if hosts is None:
hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts'
theResolver = Resolver(resolvconf, servers)
hostResolver = hostsModule.Resolver(hosts)
chain = [hostResolver, cache.CacheResolver(), theResolver]
return resolve.ResolverChain(chain)
def performPTRLookup(self, address, dns_server):
"""
Does a reverse DNS lookup on the input ip address
:address: the IP Address as a dotted quad to do a reverse lookup on.
:dns_server: is the dns_server that should be used for the lookup as a
tuple of ip port (ex. ("127.0.0.1", 53))
"""
ptr = '.'.join(address.split('.')[::-1]) + '.in-addr.arpa'
query = [dns.Query(ptr, dns.PTR, dns.IN)]
def gotResponse(message):
log.debug("Lookup successful")
log.debug(message)
answers = []
name = ''
for answer in message.answers:
if answer.type is 12:
name = str(answer.payload.name)
answers.append(representAnswer(answer))
DNSTest.addToReport(self,
query,
resolver=dns_server,
query_type='PTR',
answers=answers,
name=name)
return name
def gotError(failure):
log.err("Failed to perform lookup")
log.exception(failure)
failure.trap(gaierror, TimeoutError)
DNSTest.addToReport(self,
query,
resolver=dns_server,
query_type='PTR',
failure=failure)
return None
resolver = Resolver(servers=[dns_server])
d = resolver.queryUDP(query, timeout=self.queryTimeout)
d.addCallback(gotResponse)
d.addErrback(gotError)
return d
def performALookup(self, hostname, dns_server):
"""
Performs an A lookup and returns an array containg all the dotted quad
IP addresses in the response.
:hostname: is the hostname to perform the A lookup on
:dns_server: is the dns_server that should be used for the lookup as a
tuple of ip port (ex. ("127.0.0.1", 53))
"""
query = [dns.Query(hostname, dns.IN, dns.A)]
def gotResponse(message):
addrs = []
answers = []
for answer in message.answers:
if answer.type is 1:
addr = answer.payload.dottedQuad()
addrs.append(addr)
# We store the resource record and the answer payload in a
# tuple
r = (repr(answer), repr(answer.payload))
answers.append(r)
result = {}
result['resolver'] = dns_server
result['query_type'] = 'A'
result['query'] = repr(query)
result['answers'] = answers
result['addrs'] = addrs
self.report['queries'].append(result)
return addrs
def gotError(failure):
log.exception(failure)
result = {}
result['resolver'] = dns_server
result['query_type'] = 'A'
result['query'] = repr(query)
result['error'] = str(failure)
return None
resolver = Resolver(servers=[dns_server])
d = resolver.queryUDP(query, timeout=self.queryTimeout)
d.addCallback(gotResponse)
d.addErrback(gotError)
return d
def performALookup(self, hostname, dns_server):
"""
Performs an A lookup and returns an array containg all the dotted quad
IP addresses in the response.
:hostname: is the hostname to perform the A lookup on
:dns_server: is the dns_server that should be used for the lookup as a
tuple of ip port (ex. ("127.0.0.1", 53))
"""
query = [dns.Query(hostname, dns.IN, dns.A)]
def gotResponse(message):
addrs = []
answers = []
for answer in message.answers:
if answer.type is 1:
addr = answer.payload.dottedQuad()
addrs.append(addr)
# We store the resource record and the answer payload in a
# tuple
r = (repr(answer), repr(answer.payload))
answers.append(r)
result = {}
result['resolver'] = dns_server
result['query_type'] = 'A'
result['query'] = repr(query)
result['answers'] = answers
result['addrs'] = addrs
self.report['queries'].append(result)
return addrs
def gotError(failure):
log.exception(failure)
result = {}
result['resolver'] = dns_server
result['query_type'] = 'A'
result['query'] = repr(query)
result['error'] = str(failure)
return None
resolver = Resolver(servers=[dns_server])
d = resolver.queryUDP(query, timeout=self.queryTimeout)
d.addCallback(gotResponse)
d.addErrback(gotError)
return d
def performALookup(self, hostname, dns_server):
"""
Performs an A lookup and returns an array containg all the dotted quad
IP addresses in the response.
:hostname: is the hostname to perform the A lookup on
:dns_server: is the dns_server that should be used for the lookup as a
tuple of ip port (ex. ("127.0.0.1", 53))
"""
query = [dns.Query(hostname, dns.A, dns.IN)]
def gotResponse(message):
addrs = []
answers = []
for answer in message.answers:
if answer.type is 1:
addr = answer.payload.dottedQuad()
addrs.append(addr)
answers.append(representAnswer(answer))
DNSTest.addToReport(self,
query,
resolver=dns_server,
query_type='A',
answers=answers,
addrs=addrs)
return addrs
def gotError(failure):
failure.trap(gaierror, TimeoutError)
DNSTest.addToReport(self,
query,
resolver=dns_server,
query_type='A',
failure=failure)
return failure
resolver = Resolver(servers=[dns_server])
d = resolver.queryUDP(query, timeout=self.queryTimeout)
d.addCallback(gotResponse)
d.addErrback(gotError)
return d
def performPTRLookup(self, address, dns_server):
"""
Does a reverse DNS lookup on the input ip address
:address: the IP Address as a dotted quad to do a reverse lookup on.
:dns_server: is the dns_server that should be used for the lookup as a
tuple of ip port (ex. ("127.0.0.1", 53))
"""
ptr = '.'.join(address.split('.')[::-1]) + '.in-addr.arpa'
query = [dns.Query(ptr, dns.IN, dns.PTR)]
def gotResponse(message):
answers = []
name = None
for answer in message.answers:
if answer.type is 12:
name = answer.payload.name
result = {}
result['resolver'] = dns_server
result['query_type'] = 'PTR'
result['query'] = repr(query)
result['answers'] = answers
result['name'] = name
self.report['queries'].append(result)
return name
def gotError(failure):
log.exception(failure)
result = {}
result['resolver'] = dns_server
result['query_type'] = 'PTR'
result['query'] = repr(query)
result['error'] = str(failure)
return None
resolver = Resolver(servers=[dns_server])
d = resolver.queryUDP(query, timeout=self.queryTimeout)
d.addCallback(gotResponse)
d.addErrback(gotError)
return d
def performPTRLookup(self, address, dns_server):
"""
Does a reverse DNS lookup on the input ip address
:address: the IP Address as a dotted quad to do a reverse lookup on.
:dns_server: is the dns_server that should be used for the lookup as a
tuple of ip port (ex. ("127.0.0.1", 53))
"""
ptr = '.'.join(address.split('.')[::-1]) + '.in-addr.arpa'
query = [dns.Query(ptr, dns.IN, dns.PTR)]
def gotResponse(message):
answers = []
name = None
for answer in message.answers:
if answer.type is 12:
name = answer.payload.name
result = {}
result['resolver'] = dns_server
result['query_type'] = 'PTR'
result['query'] = repr(query)
result['answers'] = answers
result['name'] = name
self.report['queries'].append(result)
return name
def gotError(failure):
log.exception(failure)
result = {}
result['resolver'] = dns_server
result['query_type'] = 'PTR'
result['query'] = repr(query)
result['error'] = str(failure)
return None
resolver = Resolver(servers=[dns_server])
d = resolver.queryUDP(query, timeout=self.queryTimeout)
d.addCallback(gotResponse)
d.addErrback(gotError)
return d
def performPTRLookup(self, address, dns_server):
"""
Does a reverse DNS lookup on the input ip address
:address: the IP Address as a dotted quad to do a reverse lookup on.
:dns_server: is the dns_server that should be used for the lookup as a
tuple of ip port (ex. ("127.0.0.1", 53))
"""
ptr = '.'.join(address.split('.')[::-1]) + '.in-addr.arpa'
query = [dns.Query(ptr, dns.PTR, dns.IN)]
def gotResponse(message):
log.debug("Lookup successful")
log.debug(message)
answers = []
name = ''
for answer in message.answers:
if answer.type is 12:
name = str(answer.payload.name)
answers.append(representAnswer(answer))
DNSTest.addToReport(self, query, resolver=dns_server,
query_type = 'PTR', answers=answers, name=name)
return name
def gotError(failure):
log.err("Failed to perform lookup")
log.exception(failure)
failure.trap(gaierror, TimeoutError)
DNSTest.addToReport(self, query, resolver=dns_server,
query_type = 'PTR', failure=failure)
return None
resolver = Resolver(servers=[dns_server])
d = resolver.queryUDP(query, timeout=self.queryTimeout)
d.addCallback(gotResponse)
d.addErrback(gotError)
return d
def default(cls, reactor):
"""
Create a consistency checker with resolvers from Google, Cloudflare,
OpenDNS, and Level3.
"""
return cls(
[
Resolver(servers=[(addr, 53)], reactor=reactor) for addr in [
# Google
"8.8.8.8",
"8.8.4.4",
# Level3
"4.2.2.2",
"4.2.2.1",
# OpenDNS
"208.67.222.222",
"208.67.220.220",
# Cloudflare
"1.1.1.1",
"1.0.0.1",
]
],
reactor,
)
class RegionControllerService(Service):
"""
A service that controllers external services that are in MAAS's control on
a region controller. This service is ran only on the master regiond process
for a region controller.
See module documentation for more details.
"""
def __init__(self,
postgresListener,
clock=reactor,
retryOnFailure=True,
rbacRetryOnFailureDelay=10):
"""Initialise a new `RegionControllerService`.
:param postgresListener: The `PostgresListenerService` that is running
in this regiond process.
"""
super(RegionControllerService, self).__init__()
self.clock = clock
self.retryOnFailure = retryOnFailure
self.rbacRetryOnFailureDelay = rbacRetryOnFailureDelay
self.processing = LoopingCall(self.process)
self.processing.clock = self.clock
self.processingDefer = None
self.needsDNSUpdate = False
self.needsProxyUpdate = False
self.needsRBACUpdate = False
self.postgresListener = postgresListener
self.dnsResolver = Resolver(resolv=None,
servers=[('127.0.0.1', 53)],
timeout=(1, ),
reactor=clock)
self.previousSerial = None
self.rbacClient = None
self.rbacInit = False
@asynchronous(timeout=FOREVER)
def startService(self):
"""Start listening for messages."""
super(RegionControllerService, self).startService()
self.postgresListener.register("sys_dns", self.markDNSForUpdate)
self.postgresListener.register("sys_proxy", self.markProxyForUpdate)
self.postgresListener.register("sys_rbac", self.markRBACForUpdate)
self.postgresListener.events.connected.registerHandler(
self.markAllForUpdate)
@asynchronous(timeout=FOREVER)
def stopService(self):
"""Close the controller."""
super(RegionControllerService, self).stopService()
self.postgresListener.events.connected.unregisterHandler(
self.markAllForUpdate)
self.postgresListener.unregister("sys_dns", self.markDNSForUpdate)
self.postgresListener.unregister("sys_proxy", self.markProxyForUpdate)
self.postgresListener.unregister("sys_rbac", self.markRBACForUpdate)
if self.processingDefer is not None:
self.processingDefer, d = None, self.processingDefer
self.processing.stop()
return d
def markAllForUpdate(self):
self.markDNSForUpdate(None, None)
self.markProxyForUpdate(None, None)
self.markRBACForUpdate(None, None)
def markDNSForUpdate(self, channel, message):
"""Called when the `sys_dns` message is received."""
self.needsDNSUpdate = True
self.startProcessing()
def markProxyForUpdate(self, channel, message):
"""Called when the `sys_proxy` message is received."""
self.needsProxyUpdate = True
self.startProcessing()
def markRBACForUpdate(self, channel, message):
"""Called when the `sys_rbac` message is received."""
self.needsRBACUpdate = True
self.startProcessing()
def startProcessing(self):
"""Start the process looping call."""
if not self.processing.running:
self.processingDefer = self.processing.start(0.1, now=False)
def process(self):
"""Process the DNS and/or proxy update."""
def _onFailureRetry(failure, attr):
"""Retry update on failure.
Doesn't mask the failure, the failure is still raised.
"""
if self.retryOnFailure:
setattr(self, attr, True)
return failure
def _rbacInit(result):
"""Mark initialization took place."""
if result is not None:
# A sync occurred.
self.rbacInit = True
return result
def _rbacFailure(failure, delay):
log.err(failure, "Failed syncing resources to RBAC.")
if delay:
return pause(delay)
defers = []
if self.needsDNSUpdate:
self.needsDNSUpdate = False
d = deferToDatabase(transactional(dns_update_all_zones))
d.addCallback(self._checkSerial)
d.addCallback(self._logDNSReload)
# Order here matters, first needsDNSUpdate is set then pass the
# failure onto `_onDNSReloadFailure` to do the correct thing
# with the DNS server.
d.addErrback(_onFailureRetry, 'needsDNSUpdate')
d.addErrback(self._onDNSReloadFailure)
d.addErrback(log.err, "Failed configuring DNS.")
defers.append(d)
if self.needsProxyUpdate:
self.needsProxyUpdate = False
d = proxy_update_config(reload_proxy=True)
d.addCallback(lambda _: log.msg("Successfully configured proxy."))
d.addErrback(_onFailureRetry, 'needsProxyUpdate')
d.addErrback(log.err, "Failed configuring proxy.")
defers.append(d)
if self.needsRBACUpdate:
self.needsRBACUpdate = False
d = deferToDatabase(self._rbacSync)
d.addCallback(_rbacInit)
d.addCallback(self._logRBACSync)
d.addErrback(_onFailureRetry, 'needsRBACUpdate')
d.addErrback(
_rbacFailure,
self.rbacRetryOnFailureDelay if self.retryOnFailure else None)
defers.append(d)
if len(defers) == 0:
# Nothing more to do.
self.processing.stop()
self.processingDefer = None
else:
return DeferredList(defers)
@inlineCallbacks
def _checkSerial(self, result):
"""Check that the serial of the domain is updated."""
if result is None:
return None
serial, reloaded, domain_names = result
if not reloaded:
raise DNSReloadError(
"Failed to reload DNS; timeout or rdnc command failed.")
not_matching_domains = set(domain_names)
loop = 0
while len(not_matching_domains) > 0 and loop != 30:
for domain in list(not_matching_domains):
try:
answers, _, _ = yield self.dnsResolver.lookupAuthority(
domain)
except (ValueError, TimeoutError):
answers = []
if len(answers) > 0:
if int(answers[0].payload.serial) == int(serial):
not_matching_domains.remove(domain)
loop += 1
yield pause(2)
# 30 retries with 2 second pauses (aka. 60 seconds) has passed and
# there still is a domain that has the wrong serial. For now just
# raise the error, in the future we should take action and force
# restart bind.
if len(not_matching_domains) > 0:
raise DNSReloadError("Failed to reload DNS; serial mismatch "
"on domains %s" %
', '.join(not_matching_domains))
return result
def _logDNSReload(self, result):
"""Log the reason DNS was reloaded."""
if result is None:
return None
serial, _, domain_names = result
if self.previousSerial is None:
# This was the first load for starting the service.
self.previousSerial = serial
log.msg("Reloaded DNS configuration; regiond started.")
else:
# This is a reload since the region has been running. Get the
# reason for the reload.
def _logReason(reasons):
if len(reasons) == 0:
msg = (
"Reloaded DNS configuration; previous failure (retry)")
elif len(reasons) == 1:
msg = "Reloaded DNS configuration; %s" % reasons[0]
else:
msg = 'Reloaded DNS configuration: \n' + '\n'.join(
' * %s' % reason for reason in reasons)
log.msg(msg)
d = deferToDatabase(self._getReloadReasons, self.previousSerial,
serial)
d.addCallback(_logReason)
d.addErrback(log.err, "Failed to log reason for DNS reload")
self.previousSerial = serial
return d
def _onDNSReloadFailure(self, failure):
"""Force kill and restart bind9."""
failure.trap(DNSReloadError)
if not self.retryOnFailure:
return failure
log.err(failure, "Failed configuring DNS; killing and restarting")
d = service_monitor.killService('bind9')
d.addErrback(log.err, "Failed to kill and restart DNS.")
return d
@transactional
def _getReloadReasons(self, previousSerial, currentSerial):
return [
publication.source
for publication in DNSPublication.objects.filter(
serial__gt=previousSerial, serial__lte=currentSerial).order_by(
'-id')
]
def _getRBACClient(self):
"""Return the `RBACClient`.
This tries to use an already held client when initialized because the
cookiejar will be updated with the already authenticated macaroon.
"""
url = Config.objects.get_config('rbac_url')
if not url:
# RBAC is not enabled (or no longer enabled).
self.rbacClient = None
return None
auth_info = get_auth_info()
if (self.rbacClient is None or self.rbacClient._url != url
or self.rbacClient._auth_info != auth_info):
self.rbacClient = RBACClient(url, auth_info)
return self.rbacClient
@with_connection # Needed by the following lock.
@synchronised(locks.rbac_sync)
@transactional
def _rbacSync(self):
"""Sync the RBAC information."""
# Currently this whole method is scoped to dealing with
# 'resource-pool'. As more items are synced to RBAC this
# will need to be adjusted to handle multiple.
changes = RBACSync.objects.changes('resource-pool')
if not changes and self.rbacInit:
# Nothing has changed, meaning another region already took care
# of performing the update.
return None
client = self._getRBACClient()
if client is None:
# RBAC is disabled, do nothing.
RBACSync.objects.clear('resource-pool') # Changes not needed.
return None
# Push the resource information based on the last sync.
new_sync_id = None
try:
last_sync = RBACLastSync.objects.get(resource_type='resource-pool')
except RBACLastSync.DoesNotExist:
last_sync = None
if last_sync is None or self._rbacNeedsFull(changes):
# First sync or requires a full sync.
resources = [
Resource(identifier=rpool.id, name=rpool.name)
for rpool in ResourcePool.objects.order_by('id')
]
new_sync_id = client.update_resources('resource-pool',
updates=resources)
else:
# Send only the difference of what has been changed.
updates, removals = self._rbacDifference(changes)
if updates or removals:
try:
new_sync_id = client.update_resources(
'resource-pool',
updates=updates,
removals=removals,
last_sync_id=last_sync.sync_id)
except SyncConflictError:
# Issue occurred syncing, push all information.
resources = [
Resource(identifier=rpool.id, name=rpool.name)
for rpool in ResourcePool.objects.order_by('id')
]
new_sync_id = client.update_resources('resource-pool',
updates=resources)
if new_sync_id:
RBACLastSync.objects.update_or_create(
resource_type='resource-pool',
defaults={'sync_id': new_sync_id})
if not self.rbacInit:
# This was initial sync on start-up.
RBACSync.objects.clear('resource-pool')
return []
# Return the changes and clear the table, so new changes will be
# tracked. Being inside a transaction allows us not to worry about
# a new change already existing with the clear.
changes = [change.source for change in changes]
RBACSync.objects.clear('resource-pool')
return changes
def _logRBACSync(self, changes):
"""Log the reason RBAC was synced."""
if changes is None:
return None
if len(changes) == 0:
# This was the first load for starting the service.
log.msg("Synced RBAC service; regiond started.")
else:
# This is a sync since the region has been running. Get the
# reason for the reload.
if len(changes) == 1:
msg = "Synced RBAC service; %s" % changes[0]
else:
msg = 'Synced RBAC service: \n' + '\n'.join(
' * %s' % reason for reason in changes)
log.msg(msg)
def _rbacNeedsFull(self, changes):
"""Return True if any changes are marked requiring full sync."""
return any(change.action == RBAC_ACTION.FULL for change in changes)
def _rbacDifference(self, changes):
"""Return the only the changes that need to be pushed to RBAC."""
# Removals are calculated first. A removal is never followed by an
# update and `resource_id` is never re-used.
removals = set(change.resource_id for change in changes
if change.action == RBAC_ACTION.REMOVE)
# Changes are ordered from oldest to lates. The latest change will
# be the last item of that `resource_id` in the dictionary.
updates = {
change.resource_id: change.resource_name
for change in changes if (change.resource_id not in removals
and change.action != RBAC_ACTION.REMOVE)
}
# Any additions with also a removal is not sync to RBAC.
for change in changes:
if change.action == RBAC_ACTION.ADD:
if change.resource_id in removals:
removals.remove(change.resource_id)
return sorted([
Resource(identifier=res_id, name=res_name)
for res_id, res_name in updates.items()
],
key=attrgetter('identifier')), removals
import twisted, sys, codecs, traceback
from twisted.words.protocols import irc
from twisted.internet import reactor, protocol, abstract
from twisted.web import resource, server
from twisted.protocols import basic
from twisted.names.client import Resolver
import hmac, time, config, random, qwebirc.config_options as config_options
from config import HMACTEMPORAL
import database
if config.get("CONNECTION_RESOLVER"):
CONNECTION_RESOLVER = Resolver(servers=config.get("CONNECTION_RESOLVER"))
else:
CONNECTION_RESOLVER = None
if hasattr(config, "WEBIRC_MODE") and config.WEBIRC_MODE == "hmac":
HMACKEY = hmac.HMAC(key=config.HMACKEY)
def hmacfn(*args):
h = HMACKEY.copy()
h.update("%d %s" % (int(time.time() / HMACTEMPORAL), " ".join(args)))
return h.hexdigest()
def utf8_iso8859_1(data,
table=dict((x, x.decode("iso-8859-1"))
for x in map(chr, range(0, 256)))):
return (table.get(data.object[data.start]), data.start + 1)
class RegionControllerService(Service):
"""
A service that controllers external services that are in MAAS's control on
a region controller. This service is ran only on the master regiond process
for a region controller.
See module documentation for more details.
"""
def __init__(self, postgresListener, clock=reactor):
"""Initialise a new `RegionControllerService`.
:param postgresListener: The `PostgresListenerService` that is running
in this regiond process.
"""
super(RegionControllerService, self).__init__()
self.clock = clock
self.processing = LoopingCall(self.process)
self.processing.clock = self.clock
self.processingDefer = None
self.needsDNSUpdate = False
self.needsProxyUpdate = False
self.postgresListener = postgresListener
self.dnsResolver = Resolver(resolv=None,
servers=[('127.0.0.1', 53)],
timeout=(1, ),
reactor=clock)
self.previousSerial = None
@asynchronous(timeout=FOREVER)
def startService(self):
"""Start listening for messages."""
super(RegionControllerService, self).startService()
self.postgresListener.register("sys_dns", self.markDNSForUpdate)
self.postgresListener.register("sys_proxy", self.markProxyForUpdate)
# Update DNS and proxy on first start.
self.markDNSForUpdate(None, None)
self.markProxyForUpdate(None, None)
@asynchronous(timeout=FOREVER)
def stopService(self):
"""Close the controller."""
super(RegionControllerService, self).stopService()
self.postgresListener.unregister("sys_dns", self.markDNSForUpdate)
self.postgresListener.unregister("sys_proxy", self.markProxyForUpdate)
if self.processingDefer is not None:
self.processingDefer, d = None, self.processingDefer
self.processing.stop()
return d
def markDNSForUpdate(self, channel, message):
"""Called when the `sys_dns` message is received."""
self.needsDNSUpdate = True
self.startProcessing()
def markProxyForUpdate(self, channel, message):
"""Called when the `sys_proxy` message is received."""
self.needsProxyUpdate = True
self.startProcessing()
def startProcessing(self):
"""Start the process looping call."""
if not self.processing.running:
self.processingDefer = self.processing.start(0.1, now=False)
def process(self):
"""Process the DNS and/or proxy update."""
defers = []
if self.needsDNSUpdate:
self.needsDNSUpdate = False
d = deferToDatabase(transactional(dns_update_all_zones))
d.addCallback(self._checkSerial)
d.addCallback(self._logDNSReload)
d.addErrback(log.err, "Failed configuring DNS.")
defers.append(d)
if self.needsProxyUpdate:
self.needsProxyUpdate = False
d = proxy_update_config(reload_proxy=True)
d.addCallback(lambda _: log.msg("Successfully configured proxy."))
d.addErrback(log.err, "Failed configuring proxy.")
defers.append(d)
if len(defers) == 0:
# Nothing more to do.
self.processing.stop()
self.processingDefer = None
else:
return DeferredList(defers)
@inlineCallbacks
def _checkSerial(self, result):
"""Check that the serial of the domain is updated."""
if result is None:
return None
serial, domain_names = result
not_matching_domains = set(domain_names)
loop = 0
while len(not_matching_domains) > 0 and loop != 30:
for domain in list(not_matching_domains):
try:
answers, _, _ = yield self.dnsResolver.lookupAuthority(
domain)
except (ValueError, TimeoutError):
answers = []
if len(answers) > 0:
if int(answers[0].payload.serial) == int(serial):
not_matching_domains.remove(domain)
loop += 1
yield pause(2)
# 30 retries with 2 second pauses (aka. 60 seconds) has passed and
# there still is a domain that has the wrong serial. For now just
# raise the error, in the future we should take action and force
# restart bind.
if len(not_matching_domains) > 0:
raise DNSReloadError("Failed to reload DNS; serial mismatch "
"on domains %s" %
', '.join(not_matching_domains))
return serial, domain_names
def _logDNSReload(self, result):
"""Log the reason DNS was reloaded."""
if result is None:
return None
serial, domain_names = result
if self.previousSerial is None:
# This was the first load for starting the service.
self.previousSerial = serial
log.msg("Reloaded DNS configuration; regiond started.")
else:
# This is a reload since the region has been running. Get the
# reason for the reload.
def _logReason(reasons):
if len(reasons) == 1:
msg = "Reloaded DNS configuration; %s" % reasons[0]
else:
msg = 'Reloaded DNS configuration: \n' + '\n'.join(
' * %s' % reason for reason in reasons)
log.msg(msg)
d = deferToDatabase(self._getReloadReasons, self.previousSerial,
serial)
d.addCallback(_logReason)
d.addErrback(log.err, "Failed to log reason for DNS reload")
self.previousSerial = serial
return d
@transactional
def _getReloadReasons(self, previousSerial, currentSerial):
return [
publication.source
for publication in DNSPublication.objects.filter(
serial__gt=previousSerial, serial__lte=currentSerial).order_by(
'-id')
]
class RegionControllerService(Service):
"""
A service that controllers external services that are in MAAS's control on
a region controller. This service is ran only on the master regiond process
for a region controller.
See module documentation for more details.
"""
def __init__(self, postgresListener, clock=reactor, retryOnFailure=True):
"""Initialise a new `RegionControllerService`.
:param postgresListener: The `PostgresListenerService` that is running
in this regiond process.
"""
super(RegionControllerService, self).__init__()
self.clock = clock
self.retryOnFailure = retryOnFailure
self.processing = LoopingCall(self.process)
self.processing.clock = self.clock
self.processingDefer = None
self.needsDNSUpdate = False
self.needsProxyUpdate = False
self.needsRBACUpdate = False
self.postgresListener = postgresListener
self.dnsResolver = Resolver(resolv=None,
servers=[('127.0.0.1', 53)],
timeout=(1, ),
reactor=clock)
self.previousSerial = None
self.rbacClient = None
self.rbacInit = False
@asynchronous(timeout=FOREVER)
def startService(self):
"""Start listening for messages."""
super(RegionControllerService, self).startService()
self.postgresListener.register("sys_dns", self.markDNSForUpdate)
self.postgresListener.register("sys_proxy", self.markProxyForUpdate)
self.postgresListener.register("sys_rbac", self.markRBACForUpdate)
# Update DNS and proxy on first start.
self.markDNSForUpdate(None, None)
self.markProxyForUpdate(None, None)
self.markRBACForUpdate(None, None)
@asynchronous(timeout=FOREVER)
def stopService(self):
"""Close the controller."""
super(RegionControllerService, self).stopService()
self.postgresListener.unregister("sys_dns", self.markDNSForUpdate)
self.postgresListener.unregister("sys_proxy", self.markProxyForUpdate)
self.postgresListener.unregister("sys_rbac", self.markRBACForUpdate)
if self.processingDefer is not None:
self.processingDefer, d = None, self.processingDefer
self.processing.stop()
return d
def markDNSForUpdate(self, channel, message):
"""Called when the `sys_dns` message is received."""
self.needsDNSUpdate = True
self.startProcessing()
def markProxyForUpdate(self, channel, message):
"""Called when the `sys_proxy` message is received."""
self.needsProxyUpdate = True
self.startProcessing()
def markRBACForUpdate(self, channel, message):
"""Called when the `sys_rbac` message is received."""
self.needsRBACUpdate = True
self.startProcessing()
def startProcessing(self):
"""Start the process looping call."""
if not self.processing.running:
self.processingDefer = self.processing.start(0.1, now=False)
def process(self):
"""Process the DNS and/or proxy update."""
def _onFailureRetry(failure, attr):
"""Retry update on failure.
Doesn't mask the failure, the failure is still raised.
"""
if self.retryOnFailure:
setattr(self, attr, True)
return failure
def _rbacInit(result):
"""Mark initialization took place."""
if result is not None:
# A sync occurred.
self.rbacInit = True
return result
defers = []
if self.needsDNSUpdate:
self.needsDNSUpdate = False
d = deferToDatabase(transactional(dns_update_all_zones))
d.addCallback(self._checkSerial)
d.addCallback(self._logDNSReload)
d.addErrback(_onFailureRetry, 'needsDNSUpdate')
d.addErrback(log.err, "Failed configuring DNS.")
defers.append(d)
if self.needsProxyUpdate:
self.needsProxyUpdate = False
d = proxy_update_config(reload_proxy=True)
d.addCallback(lambda _: log.msg("Successfully configured proxy."))
d.addErrback(_onFailureRetry, 'needsProxyUpdate')
d.addErrback(log.err, "Failed configuring proxy.")
defers.append(d)
if self.needsRBACUpdate:
self.needsRBACUpdate = False
d = deferToDatabase(self._rbacSync)
d.addCallback(_rbacInit)
d.addCallback(self._logRBACSync)
d.addErrback(_onFailureRetry, 'needsRBACUpdate')
d.addErrback(log.err, "Failed syncing resources to RBAC.")
defers.append(d)
if len(defers) == 0:
# Nothing more to do.
self.processing.stop()
self.processingDefer = None
else:
return DeferredList(defers)
@inlineCallbacks
def _checkSerial(self, result):
"""Check that the serial of the domain is updated."""
if result is None:
return None
serial, domain_names = result
not_matching_domains = set(domain_names)
loop = 0
while len(not_matching_domains) > 0 and loop != 30:
for domain in list(not_matching_domains):
try:
answers, _, _ = yield self.dnsResolver.lookupAuthority(
domain)
except (ValueError, TimeoutError):
answers = []
if len(answers) > 0:
if int(answers[0].payload.serial) == int(serial):
not_matching_domains.remove(domain)
loop += 1
yield pause(2)
# 30 retries with 2 second pauses (aka. 60 seconds) has passed and
# there still is a domain that has the wrong serial. For now just
# raise the error, in the future we should take action and force
# restart bind.
if len(not_matching_domains) > 0:
raise DNSReloadError("Failed to reload DNS; serial mismatch "
"on domains %s" %
', '.join(not_matching_domains))
return serial, domain_names
def _logDNSReload(self, result):
"""Log the reason DNS was reloaded."""
if result is None:
return None
serial, domain_names = result
if self.previousSerial is None:
# This was the first load for starting the service.
self.previousSerial = serial
log.msg("Reloaded DNS configuration; regiond started.")
else:
# This is a reload since the region has been running. Get the
# reason for the reload.
def _logReason(reasons):
if len(reasons) == 1:
msg = "Reloaded DNS configuration; %s" % reasons[0]
else:
msg = 'Reloaded DNS configuration: \n' + '\n'.join(
' * %s' % reason for reason in reasons)
log.msg(msg)
d = deferToDatabase(self._getReloadReasons, self.previousSerial,
serial)
d.addCallback(_logReason)
d.addErrback(log.err, "Failed to log reason for DNS reload")
self.previousSerial = serial
return d
@transactional
def _getReloadReasons(self, previousSerial, currentSerial):
return [
publication.source
for publication in DNSPublication.objects.filter(
serial__gt=previousSerial, serial__lte=currentSerial).order_by(
'-id')
]
def _getRBACClient(self):
"""Return the `RBACClient`.
This tries to use an already held client when initialized because the
cookiejar will be updated with the already authenticated macaroon.
"""
url = Config.objects.get_config('rbac_url')
if not url:
# RBAC is not enabled (or no longer enabled).
self.rbacClient = None
return None
auth_info = get_auth_info()
if (self.rbacClient is None or self.rbacClient._url != url
or self.rbacClient._auth_info != auth_info):
self.rbacClient = RBACClient(url, auth_info)
return self.rbacClient
@with_connection # Needed by the following lock.
@synchronised(locks.rbac_sync)
@transactional
def _rbacSync(self):
"""Sync the RBAC information."""
changes = RBACSync.objects.changes()
if not changes and self.rbacInit:
# Nothing has changed, meaning another region already took care
# of performing the update.
return None
client = self._getRBACClient()
if client is None:
# RBAC is disabled, do nothing.
RBACSync.objects.clear() # Changes not needed, RBAC disabled.
return None
# XXX blake_r 2018-10-04 - This can be smarter by looking at the action
# and resource_type information in the changes and include the
# last_sync_id in the request, to only update what changed.
resources = [
Resource(identifier=rpool.id, name=rpool.name)
for rpool in ResourcePool.objects.all()
]
new_sync_id = client.update_resources('resource-pool',
updates=resources)
RBACLastSync.objects.update_or_create(
resource_type='resource-pool', defaults={'sync_id': new_sync_id})
if not self.rbacInit:
# This was initial sync on start-up.
RBACSync.objects.clear()
return []
# Return the changes and clear the table, so new changes will be
# tracked. Being inside a transaction allows us not to worry about
# a new change already existing with the clear.
changes = [change.source for change in changes]
RBACSync.objects.clear()
return changes
def _logRBACSync(self, changes):
"""Log the reason RBAC was synced."""
if changes is None:
return None
if len(changes) == 0:
# This was the first load for starting the service.
log.msg("Synced RBAC service; regiond started.")
else:
# This is a sync since the region has been running. Get the
# reason for the reload.
if len(changes) == 1:
msg = "Synced RBAC service; %s" % changes[0]
else:
msg = 'Synced RBAC service: \n' + '\n'.join(
' * %s' % reason for reason in changes)
log.msg(msg)
def __init__(self, *args, **kwargs):
Resolver.__init__(self, *args, **kwargs)
self.clear_cache()
def dnsLookup(self, hostname, dns_type, dns_server = None):
"""
Performs a DNS lookup and returns the response.
:hostname: is the hostname to perform the DNS lookup on
:dns_type: type of lookup 'NS'/'A'/'SOA'
:dns_server: is the dns_server that should be used for the lookup as a
tuple of ip port (ex. ("127.0.0.1", 53))
"""
types = {
'NS': dns.NS,
'A': dns.A,
'SOA': dns.SOA,
'PTR': dns.PTR
}
dnsType = types[dns_type]
query = [dns.Query(hostname, dnsType, dns.IN)]
def gotResponse(message):
log.debug(dns_type + " Lookup successful")
log.debug(str(message))
if dns_server:
msg = message.answers
else:
msg = message[0]
answers = []
addrs = []
for answer in msg:
addr = None
if answer.type is dns.SOA:
addr = (answer.name.name,answer.payload.serial)
elif answer.type in [dns.NS, dns.PTR, dns.CNAME]:
addr = answer.payload.name.name
elif answer.type is dns.A:
addr = answer.payload.dottedQuad()
else:
log.debug("Unidentified answer %s" % answer)
addrs.append(addr)
answers.append(representAnswer(answer))
if dns_type == 'SOA':
for authority in message.authority:
answers.append(representAnswer(authority))
DNSTest.addToReport(self, query, resolver=dns_server,
query_type=dns_type, answers=answers)
return addrs
def gotError(failure):
failure.trap(gaierror, TimeoutError)
DNSTest.addToReport(self, query, resolver=dns_server,
query_type=dns_type, failure=failure)
return failure
if dns_server:
resolver = Resolver(servers=[dns_server])
d = resolver.queryUDP(query, timeout=self.queryTimeout)
else:
lookupFunction = {
'NS': client.lookupNameservers,
'SOA': client.lookupAuthority,
'A': client.lookupAddress,
'PTR': client.lookupPointer
}
d = lookupFunction[dns_type](hostname)
d.addCallback(gotResponse)
d.addErrback(gotError)
return d
def setUp(self):
# Create a resolver pointed at an invalid server - we won't be hitting
# the network in any of these tests.
self.resolver = Resolver(servers=[('0.0.0.0', 0)])
class FilterAnswersTests(unittest.TestCase):
"""
Test L{twisted.names.client.Resolver.filterAnswers}'s handling of various
error conditions it might encounter.
"""
def setUp(self):
# Create a resolver pointed at an invalid server - we won't be hitting
# the network in any of these tests.
self.resolver = Resolver(servers=[('0.0.0.0', 0)])
def test_truncatedMessage(self):
"""
Test that a truncated message results in an equivalent request made via
TCP.
"""
m = Message(trunc=True)
m.addQuery('example.com')
def queryTCP(queries):
self.assertEqual(queries, m.queries)
response = Message()
response.answers = ['answer']
response.authority = ['authority']
response.additional = ['additional']
return succeed(response)
self.resolver.queryTCP = queryTCP
d = self.resolver.filterAnswers(m)
d.addCallback(
self.assertEqual, (['answer'], ['authority'], ['additional']))
return d
def _rcodeTest(self, rcode, exc):
m = Message(rCode=rcode)
err = self.resolver.filterAnswers(m)
err.trap(exc)
def test_formatError(self):
"""
Test that a message with a result code of C{EFORMAT} results in a
failure wrapped around L{DNSFormatError}.
"""
return self._rcodeTest(EFORMAT, DNSFormatError)
def test_serverError(self):
"""
Like L{test_formatError} but for C{ESERVER}/L{DNSServerError}.
"""
return self._rcodeTest(ESERVER, DNSServerError)
def test_nameError(self):
"""
Like L{test_formatError} but for C{ENAME}/L{DNSNameError}.
"""
return self._rcodeTest(ENAME, DNSNameError)
def test_notImplementedError(self):
"""
Like L{test_formatError} but for C{ENOTIMP}/L{DNSNotImplementedError}.
"""
return self._rcodeTest(ENOTIMP, DNSNotImplementedError)
def test_refusedError(self):
"""
Like L{test_formatError} but for C{EREFUSED}/L{DNSQueryRefusedError}.
"""
return self._rcodeTest(EREFUSED, DNSQueryRefusedError)
def test_refusedErrorUnknown(self):
"""
Like L{test_formatError} but for an unrecognized error code and
L{DNSUnknownError}.
"""
return self._rcodeTest(EREFUSED + 1, DNSUnknownError)
