def _start_download(self):
client = None
# Retry a few times, since this service usually comes up before
# the RPC service.
for elapsed, remaining, wait in retries(15, 5, self.clock):
try:
client = yield self.client_service.getClientNow()
break
except NoConnectionsAvailable:
yield pause(wait, self.clock)
else:
maaslog.error(
"Can't initiate image download, no RPC connection to region.")
return
# Get sources from region
sources = yield self._get_boot_sources(client)
# Get http proxy from region
proxies = yield client(GetProxies)
def get_proxy_url(scheme):
url = proxies.get(scheme) # url is a ParsedResult.
return None if url is None else url.geturl()
yield import_boot_images(
sources.get("sources"), get_proxy_url("http"),
get_proxy_url("https"))
def commission_node(system_id, user):
"""Commission a Node on the region.
:param system_id: system_id of node to commission.
:param user: user for the node.
"""
# Avoid circular dependencies.
from provisioningserver.rpc.region import CommissionNode
for elapsed, remaining, wait in retries(15, 5, reactor):
try:
client = getRegionClient()
break
except NoConnectionsAvailable:
yield pause(wait, reactor)
else:
maaslog.error("Can't commission node, no RPC connection to region.")
return
try:
yield client(CommissionNode, system_id=system_id, user=user)
except CommissionNodeFailed as e:
# The node cannot be commissioned, give up.
maaslog.error("Could not commission with system_id %s because %s.",
system_id, e.args[0])
except UnhandledCommand:
# The region hasn't been upgraded to support this method
# yet, so give up.
maaslog.error("Unable to commission node on region: Region does not "
"support the CommissionNode RPC method.")
finally:
returnValue(None)
def test_notification_gets_added_to_notifications(self):
socket_path = self.patch_socket_path()
service = LeaseSocketService(sentinel.service, reactor)
service.startService()
self.addCleanup(service.stopService)
# Stop the looping call to check that the notification gets added
# to notifications.
process_done = service.done
service.processor.stop()
yield process_done
service.processor = MagicMock()
# Create test payload to send.
packet = {"test": factory.make_name("test")}
# Send notification to the socket should appear in notifications.
yield deferToThread(self.send_notification, socket_path, packet)
# Loop until the notifications has a notification.
for elapsed, remaining, wait in retries(5, 0.1, reactor):
if len(service.notifications) > 0:
break
else:
yield pause(wait, reactor)
# Should have one notitication.
self.assertEquals([packet], list(service.notifications))
def _tryGetClient(self):
client = None
for elapsed, remaining, wait in retries(15, 5, self.clock):
try:
client = yield self.client_service.getClientNow()
break
except NoConnectionsAvailable:
yield pause(wait, self.clock)
return client
def dns_wait_soa(self, fqdn, removing=False):
# Get the serial number for the zone containing the FQDN by asking DNS
# nicely for the SOA for the FQDN. If it's top-of-zone, we get an
# answer, if it's not, we get the SOA in authority.
if not fqdn.endswith("."):
fqdn = fqdn + "."
for elapsed, remaining, wait in retries(15, 0.02):
query_name = fqdn
# Loop until we have a value for serial, be that numeric or None.
serial = undefined = object()
while serial is undefined:
try:
ans = self.resolver.query(query_name,
"SOA",
raise_on_no_answer=False)
except dns.resolver.NXDOMAIN:
if removing:
# The zone has gone; we're done.
return
elif "." in query_name:
# Query the parent domain for the SOA record.
# For most things, this will be the correct DNS zone.
# In the case of SRV records, we'll actually need to
# strip more, hence the loop.
query_name = query_name.split(".", 1)[1]
else:
# We've hit the root zone; no SOA found.
serial = None
except dns.resolver.NoNameservers:
# No DNS service as yet.
serial = None
else:
# If we got here, then we either have (1) a situation where
# the LHS exists in the DNS, but no SOA RR exists for that
# LHS (because it's a node with an A or AAAA RR, and not
# the domain...) or (2) an answer to our SOA query.
# Either way, we get exactly one SOA in the reply: in the
# first case, it's in the Authority section, in the second,
# it's in the Answer section.
if ans.rrset is None:
serial = ans.response.authority[0].items[0].serial
else:
serial = ans.rrset.items[0].serial
if serial == DNSPublication.objects.get_most_recent().serial:
# The zone is up-to-date; we're done.
return
else:
time.sleep(wait)
self.fail("Timed-out waiting for %s to update." % fqdn)
def _getConnection(self):
"""Get a connection to the region."""
client = None
for elapsed, remaining, wait in retries(30, 10, self.clock):
try:
client = yield self.client_service.getClientNow()
break
except NoConnectionsAvailable:
yield pause(wait, self.clock)
else:
maaslog.error("Can't update service statuses, no RPC "
"connection to region.")
return client
def start_services():
rpc_service = ClusterClientService(reactor)
rpc_service.setName("rpc")
rpc_service.setServiceParent(services)
yield services.startService()
for elapsed, remaining, wait in retries(15, 1, reactor):
try:
yield getRegionClient()
except NoConnectionsAvailable:
yield pause(wait, reactor)
else:
break
else:
print("Can't connect to the region.", file=stderr)
raise SystemExit(1)
def _updateRegion(self, services):
"""Update region about services status."""
client = None
for elapsed, remaining, wait in retries(30, 10, self.clock):
try:
client = yield self.client_service.getClientNow()
break
except NoConnectionsAvailable:
yield pause(wait, self.clock)
else:
maaslog.error("Can't update service statuses, no RPC "
"connection to region.")
return
services = yield self._buildServices(services)
yield client(UpdateServices,
system_id=client.localIdent,
services=services)
def processNotification(self, notification, clock=reactor):
"""Send a notification to the region."""
client = None
for elapsed, remaining, wait in retries(30, 10, clock):
try:
client = yield self.client_service.getClientNow()
break
except NoConnectionsAvailable:
yield pause(wait, self.clock)
else:
maaslog.error("Can't send DHCP lease information, no RPC "
"connection to region.")
return
# Notification contains all the required data except for the cluster
# UUID. Add that into the notification and send the information to
# the region for processing.
notification["cluster_uuid"] = client.localIdent
yield client(UpdateLease, **notification)
def test_YUI3_unit_tests(self):
# Load the page and then wait for #suite to contain
# 'done'. Read the results in '#test_results'.
self.browser.get(self.test_url)
for elapsed, remaining, wait in retries(intervals=0.2):
suite = self.browser.find_element_by_id("suite")
if suite.text == "done":
results = self.browser.find_element_by_id("test_results")
results = json.loads(results.text)
break
else:
sleep(wait)
else:
self.fail("Timed-out after %ds" % elapsed)
if results['failed'] != 0:
message = '%d test(s) failed.\n\n%s' % (
results['failed'], yui3.get_failed_tests_message(results))
self.fail(message)
def wait(self, timeout=86400):
"""Wait for the lock to become available.
:param timeout: The number of seconds to wait. By default it will wait
up to 1 day.
"""
interval = max(0.1, min(1.0, float(timeout) / 10.0))
for _, _, wait in retries(timeout, interval, self.reactor):
with self.PROCESS_LOCK:
if self._fslock.lock():
break
if wait > 0:
sleep(wait)
else:
raise self.NotAvailable(self._fslock.name)
try:
yield
finally:
with self.PROCESS_LOCK:
self._fslock.unlock()
def _updateRegion(self, services):
"""Update region about services status."""
services = yield self._buildServices(services)
if self._services is not None and self._services == services:
# The updated status to the region hasn't changed no reason
# to update the region controller.
return None
self._services = services
client = None
for elapsed, remaining, wait in retries(30, 10, self.clock):
try:
client = yield self.client_service.getClientNow()
break
except NoConnectionsAvailable:
yield pause(wait, self.clock)
else:
maaslog.error("Can't update service statuses, no RPC "
"connection to region.")
return
yield client(UpdateServices,
system_id=client.localIdent,
services=services)
def makeService(self, options, clock=reactor, sleep=sleep):
"""Construct the MAAS Cluster service."""
register_sigusr1_toggle_cprofile("rackd")
register_sigusr2_thread_dump_handler()
clean_prometheus_dir()
add_patches_to_txtftp()
add_patches_to_twisted()
self._loadSettings()
self._configureCrochet()
if settings.DEBUG:
# Always log at debug level in debug mode.
self._configureLogging(3)
else:
self._configureLogging(options["verbosity"])
with ClusterConfiguration.open() as config:
tftp_root = config.tftp_root
tftp_port = config.tftp_port
from provisioningserver import services
secret = None
for elapsed, remaining, wait in retries(timeout=5 * 60, clock=clock):
secret = get_shared_secret_from_filesystem()
if secret is not None:
break
sleep(wait)
if secret is not None:
# only setup services if the shared secret is configured
for service in self._makeServices(tftp_root,
tftp_port,
clock=clock):
service.setServiceParent(services)
reactor.callInThread(generate_certificate_if_needed)
return services
def create_node(macs,
arch,
power_type,
power_parameters,
domain=None,
hostname=None):
"""Create a Node on the region and return its system_id.
:param macs: A list of MAC addresses belonging to the node.
:param arch: The node's architecture, in the form 'arch/subarch'.
:param power_type: The node's power type as a string.
:param power_parameters: The power parameters for the node, as a
dict.
:param domain: The domain the node should join.
"""
if hostname is not None:
hostname = coerce_to_valid_hostname(hostname)
for elapsed, remaining, wait in retries(15, 5, reactor):
try:
client = getRegionClient()
break
except NoConnectionsAvailable:
yield pause(wait, reactor)
else:
maaslog.error("Can't create node, no RPC connection to region.")
return
# De-dupe the MAC addresses we pass. We sort here to avoid test
# failures.
macs = sorted(set(macs))
try:
response = yield client(CreateNode,
architecture=arch,
power_type=power_type,
power_parameters=json.dumps(power_parameters),
mac_addresses=macs,
hostname=hostname,
domain=domain)
except NodeAlreadyExists:
# The node already exists on the region, so we log the error and
# give up.
maaslog.error(
"A node with one of the mac addresses in %s already exists.", macs)
returnValue(None)
except UnhandledCommand:
# The region hasn't been upgraded to support this method
# yet, so give up.
maaslog.error("Unable to create node on region: Region does not "
"support the CreateNode RPC method.")
returnValue(None)
except UnknownRemoteError as e:
# This happens, for example, if a ValidationError occurs on the region.
# (In particular, we see this if the hostname is a duplicate.)
# We should probably create specific exceptions for these, so we can
# act on them appropriately.
maaslog.error(
"Unknown error while creating node %s: %s (see regiond.log)", macs,
e.description)
returnValue(None)
else:
returnValue(response['system_id'])
def get_response(self, request):
"""Override `BaseHandler.get_response`.
Wrap Django's default get_response(). Middleware and templates will
thus also run within the same transaction, but streaming responses
will *not* run within the same transaction, or any transaction at all
by default.
"""
django_get_response = super(WebApplicationHandler, self).get_response
def get_response(request):
# Up-call to Django's get_response() in a transaction. This
# transaction may fail because of a retryable conflict, so
# pass errors to handle_uncaught_exception().
try:
with post_commit_hooks:
with transaction.atomic():
response = django_get_response(request)
if response.status_code == 500:
raise InternalErrorResponse(response)
return response
except SystemExit:
# Allow sys.exit() to actually exit, reproducing behaviour
# found in Django's BaseHandler.
raise
except InternalErrorResponse as exc:
# Response is good, but the transaction needed to be rolled
# back because the response was a 500 error.
return exc.response
except:
# Catch *everything* else, also reproducing behaviour found in
# Django's BaseHandler. In practice, we should only really see
# transaction failures here from the outermost atomic block as
# all other exceptions are handled by django_get_response. The
# setting DEBUG_PROPAGATE_EXCEPTIONS upsets this, so be on
# your guard when tempted to use it.
signals.got_request_exception.send(sender=self.__class__,
request=request)
return self.handle_uncaught_exception(request,
get_resolver(None),
sys.exc_info(),
reraise=False)
# Attempt to start new transactions for up to `__retry_timeout`
# seconds, at intervals defined by `gen_retry_intervals`, but don't
# try more than `__retry_attempts` times.
retry_intervals = gen_retry_intervals()
retry_details = retries(self.__retry_timeout, retry_intervals, clock)
retry_attempts = self.__retry_attempts
retry_set = self.__retry
with retry_context:
for attempt in count(1):
retry_context.prepare()
response = get_response(request)
if response in retry_set:
elapsed, remaining, wait = next(retry_details)
if attempt == retry_attempts or wait == 0:
# Time's up: this was the final attempt.
log_final_failed_attempt(request, attempt, elapsed)
conflict_response = HttpResponseConflict(response)
conflict_response.render()
return conflict_response
else:
# We'll retry after a brief interlude.
log_failed_attempt(request, attempt, elapsed,
remaining, wait)
delete_oauth_nonce(request)
request = reset_request(request)
sleep(wait)
else:
return response
