def regenerate(self, storage=True, network=True):
"""Regenerate any ScriptResult which has a storage parameter.
Deletes and recreates ScriptResults for any ScriptResult which has a
storage parameter. Used after commissioning has completed when there
are tests to be run.
"""
# Avoid circular dependencies.
from metadataserver.models import ScriptResult
regenerate_scripts = {}
for script_result in self.scriptresult_set.filter(
status=SCRIPT_STATUS.PENDING).exclude(parameters={}).defer(
'stdout', 'stderr', 'output', 'result'):
# If there are multiple storage devices or interface on the system
# for every script which contains a storage or interface type
# parameter there will be one ScriptResult per device. If we
# already know a script must be regenearted it can be deleted as
# the device the ScriptResult is for may no longer exist.
# Regeneratation below will generate ScriptResults for each
# existing storage or interface device.
if script_result.script in regenerate_scripts:
script_result.delete()
continue
# Check if the ScriptResult contains any storage or interface type
# parameter. If so remove the value of the storage or interface
# parameter only and add it to the list of Scripts which must be
# regenearted.
for param_name, param in script_result.parameters.items():
if ((storage and param['type'] == 'storage')
or (network and param['type'] == 'interface')):
# Remove the storage or interface parameter as the storage
# device or interface may no longer exist. The
# ParametersForm will set the default value(all).
script_result.parameters.pop(param_name)
# Only preserve the value of the parameter as that is what
# the form will validate.
regenerate_scripts[script_result.script] = {
key: value['value']
for key, value in script_result.parameters.items()
}
script_result.delete()
break
for script, params in regenerate_scripts.items():
form = ParametersForm(data=params, script=script, node=self.node)
if not form.is_valid():
err_msg = (
"Removing Script %s from ScriptSet due to regeneration "
"error - %s" % (script.name, dict(form.errors)))
logger.error(err_msg)
Event.objects.create_node_event(
system_id=self.node.system_id,
event_type=EVENT_TYPES.SCRIPT_RESULT_ERROR,
event_description=err_msg)
continue
for i in form.cleaned_data['input']:
ScriptResult.objects.create(script_set=self,
status=SCRIPT_STATUS.PENDING,
script=script,
script_name=script.name,
parameters=i)
def select_for_hardware_scripts(self, modaliases=None):
"""Select for_hardware scripts for the given node and user input.
Goes through an existing ScriptSet and adds any for_hardware tagged
Script and removes those that were autoselected but the hardware has
been removed.
"""
# Only the builtin commissioning scripts run on controllers.
if self.node.is_controller:
return
if modaliases is None:
modaliases = self.node.modaliases
regexes = []
for nmd in self.node.nodemetadata_set.all():
if nmd.key in [
'system_vendor', 'system_product', 'system_version',
'mainboard_vendor', 'mainboard_product'
]:
regexes.append('%s:%s' %
(nmd.key, fnmatch.translate(nmd.value)))
if len(regexes) > 0:
node_hw_regex = re.compile('^%s$' % '|'.join(regexes), re.I)
else:
node_hw_regex = None
# Remove scripts autoselected at the start of commissioning but updated
# commissioning data shows the Script is no longer applicable.
script_results = self.scriptresult_set.exclude(script=None)
script_results = script_results.filter(status=SCRIPT_STATUS.PENDING)
script_results = script_results.exclude(script__for_hardware=[])
script_results = script_results.prefetch_related('script')
script_results = script_results.only('status', 'script__for_hardware',
'script_set_id')
for script_result in script_results:
matches = filter_modaliases(modaliases,
*script_result.script.ForHardware)
found_hw_match = False
if node_hw_regex is not None:
for hardware in script_result.script.for_hardware:
if node_hw_regex.search(hardware) is not None:
found_hw_match = True
break
matches = filter_modaliases(modaliases,
*script_result.script.ForHardware)
if len(matches) == 0 and not found_hw_match:
script_result.delete()
# Add Scripts which match the node with current commissioning data.
scripts = Script.objects.all()
if self.result_type == RESULT_TYPE.COMMISSIONING:
scripts = scripts.filter(script_type=SCRIPT_TYPE.COMMISSIONING)
else:
scripts = scripts.filter(script_type=SCRIPT_TYPE.TESTING)
scripts = scripts.filter(tags__overlap=self.requested_scripts)
scripts = scripts.exclude(for_hardware=[])
scripts = scripts.exclude(name__in=[s.name for s in self])
for script in scripts:
found_hw_match = False
if node_hw_regex is not None:
for hardware in script.for_hardware:
if node_hw_regex.search(hardware) is not None:
found_hw_match = True
break
matches = filter_modaliases(modaliases, *script.ForHardware)
if len(matches) != 0 or found_hw_match:
try:
self.add_pending_script(script)
except ValidationError as e:
err_msg = (
"Error adding for_hardware Script %s due to error - %s"
% (script.name, str(e)))
logger.error(err_msg)
Event.objects.create_node_event(
system_id=self.node.system_id,
event_type=EVENT_TYPES.SCRIPT_RESULT_ERROR,
event_description=err_msg)
def store_result(
self,
exit_status=None,
output=None,
stdout=None,
stderr=None,
result=None,
script_version_id=None,
timedout=False,
runtime=None,
):
# Controllers and Pods are allowed to overwrite their results during any status
# to prevent new ScriptSets being created everytime a controller
# starts. This also allows us to avoid creating an RPC call for the
# rack controller to create a new ScriptSet.
if (not self.script_set.node.is_controller
and not self.script_set.node.is_pod):
# Allow PENDING, APPLYING_NETCONF, INSTALLING, and RUNNING scripts
# incase the node didn't inform MAAS the Script was being run, it
# just uploaded results.
assert self.status in SCRIPT_STATUS_RUNNING_OR_PENDING
if timedout:
self.status = SCRIPT_STATUS.TIMEDOUT
elif exit_status is not None:
self.exit_status = exit_status
if exit_status == 0:
self.status = SCRIPT_STATUS.PASSED
elif self.status == SCRIPT_STATUS.INSTALLING:
self.status = SCRIPT_STATUS.FAILED_INSTALLING
elif self.status == SCRIPT_STATUS.APPLYING_NETCONF:
self.status = SCRIPT_STATUS.FAILED_APPLYING_NETCONF
else:
self.status = SCRIPT_STATUS.FAILED
if output is not None:
self.output = Bin(output)
if stdout is not None:
self.stdout = Bin(stdout)
if stderr is not None:
self.stderr = Bin(stderr)
if result is not None:
self.result = Bin(result)
try:
parsed_yaml = self.read_results()
except ValidationError as err:
err_msg = (
"%s(%s) sent a script result with invalid YAML: %s" % (
self.script_set.node.fqdn,
self.script_set.node.system_id,
err.message,
))
logger.error(err_msg)
Event.objects.create_node_event(
system_id=self.script_set.node.system_id,
event_type=EVENT_TYPES.SCRIPT_RESULT_ERROR,
event_description=err_msg,
)
else:
status = parsed_yaml.get("status")
if status == "passed":
self.status = SCRIPT_STATUS.PASSED
elif status == "failed":
self.status = SCRIPT_STATUS.FAILED
elif status == "degraded":
self.status = SCRIPT_STATUS.DEGRADED
elif status == "timedout":
self.status = SCRIPT_STATUS.TIMEDOUT
elif status == "skipped":
self.status = SCRIPT_STATUS.SKIPPED
link_connected = parsed_yaml.get("link_connected")
if self.interface and isinstance(link_connected, bool):
self.interface.link_connected = link_connected
self.interface.save(update_fields=["link_connected"])
if self.script:
if script_version_id is not None:
for script in self.script.script.previous_versions():
if script.id == script_version_id:
self.script_version = script
break
if self.script_version is None:
err_msg = (
"%s(%s) sent a script result for %s(%d) with an "
"unknown script version(%d)." % (
self.script_set.node.fqdn,
self.script_set.node.system_id,
self.script.name,
self.script.id,
script_version_id,
))
logger.error(err_msg)
Event.objects.create_node_event(
system_id=self.script_set.node.system_id,
event_type=EVENT_TYPES.SCRIPT_RESULT_ERROR,
event_description=err_msg,
)
else:
# If no script version was given assume the latest version
# was run.
self.script_version = self.script.script
# If commissioning result check if its a builtin script, if so run its
# hook before committing to the database.
if (self.script_set.result_type == RESULT_TYPE.COMMISSIONING
and self.name in NODE_INFO_SCRIPTS and stdout is not None):
post_process_hook = NODE_INFO_SCRIPTS[self.name]["hook"]
err = ("%s(%s): commissioning script '%s' failed during "
"post-processing." % (
self.script_set.node.fqdn,
self.script_set.node.system_id,
self.name,
))
# Circular imports.
from metadataserver.api import try_or_log_event
signal_status = try_or_log_event(
self.script_set.node,
None,
err,
post_process_hook,
node=self.script_set.node,
output=self.stdout,
exit_status=self.exit_status,
)
# If the script failed to process mark the script as failed to
# prevent testing from running and help users identify where
# the error came from. This can happen when a commissioning
# script generated invalid output.
if signal_status is not None:
self.status = SCRIPT_STATUS.FAILED
if (self.status == SCRIPT_STATUS.PASSED and self.script
and self.script.script_type == SCRIPT_TYPE.COMMISSIONING
and self.script.recommission):
self.script_set.scriptresult_set.filter(
script_name__in=NODE_INFO_SCRIPTS).update(
status=SCRIPT_STATUS.PENDING,
started=None,
ended=None,
updated=now(),
)
self.save(runtime=runtime)
def render_POST(self, request):
"""Receive and process a status message from a node, usually cloud-init
A node can call this to report progress of its booting or deployment.
Calling this from a node that is not Allocated, Commissioning, Ready,
or Failed Tests will update the substatus_message node attribute.
Signaling completion more than once is not an error; all but the first
successful call are ignored.
This method accepts a single JSON-encoded object payload, described as
follows.
{
"event_type": "finish",
"origin": "curtin",
"description": "Finished XYZ",
"name": "cmd-install",
"result": "SUCCESS",
"files": [
{
"name": "logs.tgz",
"encoding": "base64",
"content": "QXVnIDI1IDA3OjE3OjAxIG1hYXMtZGV2...
},
{
"name": "results.log",
"compression": "bzip2"
"encoding": "base64",
"content": "AAAAAAAAAAAAAAAAAAAAAAA...
}
]
}
`event_type` can be "start", "progress" or "finish".
`origin` tells us the program that originated the call.
`description` is a human-readable, operator-friendly string that
conveys what is being done to the node and that can be presented on the
web UI.
`name` is the name of the activity that's being executed. It's
meaningful to the calling program and is a slash-separated path. We are
mainly concerned with top-level events (no slashes), which are used to
change the status of the node.
`result` can be "SUCCESS" or "FAILURE" indicating whether the activity
was successful or not.
`files`, when present, contains one or more files. The attribute `path`
tells us the name of the file, `compression` tells the compression we
used before applying the `encoding` and content is the encoded data
from the file. If the file being sent is the result of the execution of
a script, the `result` key will hold its value. If `result` is not
sent, it is interpreted as zero.
`script_result_id`, when present, MAAS will search for an existing
ScriptResult with the given id to store files present.
"""
# Extract the authorization from request. This only does a basic
# check that its provided. The status worker will do the authorization,
# the negative to this is that the calling client will no know. To
# them the message was accepted. This overall is okay since they are
# just status messages.
authorization = request.getHeader(b"authorization")
if not authorization:
request.setResponseCode(401)
return b""
authorization = extract_oauth_key_from_auth_header(
authorization.decode("utf-8"))
if authorization is None:
request.setResponseCode(401)
return b""
# Load the content to ensure that its atleast correct before placing
# it into the status worker.
payload = request.content.read()
try:
payload = payload.decode("ascii")
except UnicodeDecodeError as error:
request.setResponseCode(400)
error_msg = "Status payload must be ASCII-only: %s" % error
logger.error(error_msg)
return error_msg.encode("ascii")
try:
message = json.loads(payload)
except ValueError:
request.setResponseCode(400)
error_msg = "Status payload is not valid JSON:\n%s\n\n" % payload
logger.error(error_msg)
return error_msg.encode("ascii")
# Ensure the other required keys exist.
missing_keys = [
key for key in self.requiredMessageKeys if key not in message
]
if len(missing_keys) > 0:
request.setResponseCode(400)
error_msg = ("Missing parameter(s) %s in "
"status message." % ", ".join(missing_keys))
logger.error(error_msg)
return error_msg.encode("ascii")
# Queue the message with its authorization in the status worker.
d = self.worker.queueMessage(authorization, message)
# Finish the request after defer finishes.
def _finish(result, request):
request.setResponseCode(204)
request.finish()
d.addCallback(_finish, request)
return NOT_DONE_YET
def store_result(self,
exit_status=None,
output=None,
stdout=None,
stderr=None,
result=None,
script_version_id=None,
timedout=False):
# Controllers are allowed to overwrite their results during any status
# to prevent new ScriptSets being created everytime a controller
# starts. This also allows us to avoid creating an RPC call for the
# rack controller to create a new ScriptSet.
if not self.script_set.node.is_controller:
# Allow PENDING, INSTALLING, and RUNNING scripts incase the node
# didn't inform MAAS the Script was being run, it just uploaded
# results.
assert self.status in (SCRIPT_STATUS.PENDING,
SCRIPT_STATUS.INSTALLING,
SCRIPT_STATUS.RUNNING)
if timedout:
self.status = SCRIPT_STATUS.TIMEDOUT
elif exit_status is not None:
self.exit_status = exit_status
if exit_status == 0:
self.status = SCRIPT_STATUS.PASSED
elif self.status == SCRIPT_STATUS.INSTALLING:
self.status = SCRIPT_STATUS.FAILED_INSTALLING
else:
self.status = SCRIPT_STATUS.FAILED
if output is not None:
self.output = Bin(output)
if stdout is not None:
self.stdout = Bin(stdout)
if stderr is not None:
self.stderr = Bin(stderr)
if result is not None:
self.result = Bin(result)
try:
parsed_yaml = self.read_results()
except ValidationError as err:
err_msg = (
"%s(%s) sent a script result with invalid YAML: %s" %
(self.script_set.node.fqdn, self.script_set.node.system_id,
err.message))
logger.error(err_msg)
Event.objects.create_node_event(
system_id=self.script_set.node.system_id,
event_type=EVENT_TYPES.SCRIPT_RESULT_ERROR,
event_description=err_msg)
else:
status = parsed_yaml.get('status')
if status == 'passed':
self.status = SCRIPT_STATUS.PASSED
elif status == 'failed':
self.status = SCRIPT_STATUS.FAILED
elif status == 'degraded':
self.status = SCRIPT_STATUS.DEGRADED
elif status == 'timedout':
self.status = SCRIPT_STATUS.TIMEDOUT
if self.script:
if script_version_id is not None:
for script in self.script.script.previous_versions():
if script.id == script_version_id:
self.script_version = script
break
if self.script_version is None:
err_msg = (
"%s(%s) sent a script result for %s(%d) with an "
"unknown script version(%d)." %
(self.script_set.node.fqdn,
self.script_set.node.system_id, self.script.name,
self.script.id, script_version_id))
logger.error(err_msg)
Event.objects.create_node_event(
system_id=self.script_set.node.system_id,
event_type=EVENT_TYPES.SCRIPT_RESULT_ERROR,
event_description=err_msg)
else:
# If no script version was given assume the latest version
# was run.
self.script_version = self.script.script
# If commissioning result check if its a builtin script, if so run its
# hook before committing to the database.
if (self.script_set.result_type == RESULT_TYPE.COMMISSIONING
and self.name in NODE_INFO_SCRIPTS and stdout is not None):
post_process_hook = NODE_INFO_SCRIPTS[self.name]['hook']
err = ("%s(%s): commissioning script '%s' failed during "
"post-processing." %
(self.script_set.node.fqdn, self.script_set.node.system_id,
self.name))
# Circular imports.
from metadataserver.api import try_or_log_event
try_or_log_event(self.script_set.node,
None,
err,
post_process_hook,
node=self.script_set.node,
output=self.stdout,
exit_status=self.exit_status)
if (self.status == SCRIPT_STATUS.PASSED and self.script
and self.script.script_type == SCRIPT_TYPE.COMMISSIONING
and self.script.recommission):
self.script_set.scriptresult_set.filter(
script_name__in=NODE_INFO_SCRIPTS).update(
status=SCRIPT_STATUS.PENDING,
started=None,
ended=None,
updated=now())
self.save()