def run_and_check(cmd,
scripts,
status,
send_result=True,
sudo=False,
failure_hook=None):
if sudo:
cmd = ['sudo', '-En'] + cmd
proc = Popen(cmd, stdin=DEVNULL, stdout=PIPE, stderr=PIPE)
capture_script_output(proc, scripts[0]['combined_path'],
scripts[0]['stdout_path'], scripts[0]['stderr_path'])
if proc.returncode != 0 and send_result:
if failure_hook is not None:
failure_hook()
for script in scripts:
args = copy.deepcopy(script['args'])
script['exit_status'] = args['exit_status'] = proc.returncode
args['status'] = status
args['files'] = {
scripts[0]['combined_name']:
open(scripts[0]['combined_path'], 'rb').read(),
scripts[0]['stdout_name']:
open(scripts[0]['stdout_path'], 'rb').read(),
scripts[0]['stderr_name']:
open(scripts[0]['stderr_path'], 'rb').read(),
}
output_and_send(
'Failed installing package(s) for %s' % script['msg_name'],
**args)
return False
else:
return True
def run_and_check(cmd,
combined_path,
stdout_path,
stderr_path,
script_name,
args,
ignore_error=False):
proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
capture_script_output(proc, combined_path, stdout_path, stderr_path)
if proc.returncode != 0 and not ignore_error:
args['exit_status'] = proc.returncode
args['files'] = {
script_name: open(combined_path, 'rb').read(),
'%s.out' % script_name: open(stdout_path, 'rb').read(),
'%s.err' % script_name: open(stderr_path, 'rb').read(),
}
signal_wrapper(error='Failed installing package(s) for %s' %
(script_name),
**args)
return False
else:
return True
def run_and_check(cmd,
scripts,
status,
send_result=True,
sudo=False,
failure_hook=None):
if sudo:
cmd = ["sudo", "-En"] + cmd
proc = Popen(cmd, stdin=DEVNULL, stdout=PIPE, stderr=PIPE)
capture_script_output(
proc,
scripts[0]["combined_path"],
scripts[0]["stdout_path"],
scripts[0]["stderr_path"],
)
if proc.returncode != 0 and send_result:
if failure_hook is not None:
failure_hook()
for script in scripts:
args = copy.deepcopy(script["args"])
script["exit_status"] = args["exit_status"] = proc.returncode
args["status"] = status
args["files"] = {
scripts[0]["combined_name"]:
open(scripts[0]["combined_path"], "rb").read(),
scripts[0]["stdout_name"]:
open(scripts[0]["stdout_path"], "rb").read(),
scripts[0]["stderr_name"]:
open(scripts[0]["stderr_path"], "rb").read(),
}
output_and_send(
"Failed installing package(s) for %s" % script["msg_name"],
**args)
return False
else:
return True
def run_and_check(cmd, scripts, send_result=True):
proc = Popen(cmd, stdin=DEVNULL, stdout=PIPE, stderr=PIPE)
capture_script_output(
proc, scripts[0]['combined_path'], scripts[0]['stdout_path'],
scripts[0]['stderr_path'])
if proc.returncode != 0 and send_result:
for script in scripts:
args = copy.deepcopy(script['args'])
script['exit_status'] = args['exit_status'] = proc.returncode
args['status'] = 'INSTALLING'
args['files'] = {
scripts[0]['combined_name']: open(
scripts[0]['combined_path'], 'rb').read(),
scripts[0]['stdout_name']: open(
scripts[0]['stdout_path'], 'rb').read(),
scripts[0]['stderr_name']: open(
scripts[0]['stderr_path'], 'rb').read(),
}
output_and_send(
'Failed installing package(s) for %s' % script['msg_name'],
**args)
return False
else:
return True
def run_scripts(url, creds, scripts_dir, out_dir, scripts):
"""Run and report results for the given scripts."""
total_scripts = len(scripts)
fail_count = 0
base_args = {
'url': url,
'creds': creds,
'status': 'WORKING',
}
for i, script in enumerate(scripts):
i += 1
args = copy.deepcopy(base_args)
args['script_result_id'] = script['script_result_id']
script_version_id = script.get('script_version_id')
if script_version_id is not None:
args['script_version_id'] = script_version_id
timeout_seconds = script.get('timeout_seconds')
signal_wrapper(error='Starting %s [%d/%d]' %
(script['name'], i, len(scripts)),
**args)
script_path = os.path.join(scripts_dir, script['path'])
combined_path = os.path.join(out_dir, script['name'])
stdout_name = '%s.out' % script['name']
stdout_path = os.path.join(out_dir, stdout_name)
stderr_name = '%s.err' % script['name']
stderr_path = os.path.join(out_dir, stderr_name)
try:
# This script sets its own niceness value to the highest(-20) below
# to help ensure the heartbeat keeps running. When launching the
# script we need to lower the nice value as a child process
# inherits the parent processes niceness value. preexec_fn is
# executed in the child process before the command is run. When
# setting the nice value the kernel adds the current nice value
# to the provided value. Since the runner uses a nice value of -20
# setting it to 40 gives the actual nice value of 20.
proc = Popen(script_path,
stdout=PIPE,
stderr=PIPE,
preexec_fn=lambda: os.nice(40))
capture_script_output(proc, combined_path, stdout_path,
stderr_path, timeout_seconds)
except OSError as e:
fail_count += 1
if isinstance(e.errno, int) and e.errno != 0:
args['exit_status'] = e.errno
else:
# 2 is the return code bash gives when it can't execute.
args['exit_status'] = 2
result = str(e).encode()
if result == b'':
result = b'Unable to execute script'
args['files'] = {
script['name']: result,
stderr_name: result,
}
signal_wrapper(
error='Failed to execute %s [%d/%d]: %d' %
(script['name'], i, total_scripts, args['exit_status']),
**args)
except TimeoutExpired:
fail_count += 1
args['status'] = 'TIMEDOUT'
args['files'] = {
script['name']: open(combined_path, 'rb').read(),
stdout_name: open(stdout_path, 'rb').read(),
stderr_name: open(stderr_path, 'rb').read(),
}
signal_wrapper(error='Timeout(%s) expired on %s [%d/%d]' %
(str(timedelta(seconds=timeout_seconds)),
script['name'], i, total_scripts),
**args)
else:
if proc.returncode != 0:
fail_count += 1
args['exit_status'] = proc.returncode
args['files'] = {
script['name']: open(combined_path, 'rb').read(),
stdout_name: open(stdout_path, 'rb').read(),
stderr_name: open(stderr_path, 'rb').read(),
}
signal_wrapper(
error='Finished %s [%d/%d]: %d' %
(script['name'], i, len(scripts), args['exit_status']),
**args)
# Signal failure after running commissioning or testing scripts so MAAS
# transisitions the node into FAILED_COMMISSIONING or FAILED_TESTING.
if fail_count != 0:
signal_wrapper(url, creds, 'FAILED',
'%d scripts failed to run' % fail_count)
return fail_count
def run_script(script, scripts_dir, send_result=True):
args = copy.deepcopy(script["args"])
args["status"] = "WORKING"
args["send_result"] = send_result
timeout_seconds = script.get("timeout_seconds")
for param in script.get("parameters", {}).values():
if param.get("type") == "runtime":
timeout_seconds = param["value"]
break
output_and_send("Starting %s" % script["msg_name"], **args)
env = copy.deepcopy(os.environ)
env["OUTPUT_COMBINED_PATH"] = script["combined_path"]
env["OUTPUT_STDOUT_PATH"] = script["stdout_path"]
env["OUTPUT_STDERR_PATH"] = script["stderr_path"]
env["RESULT_PATH"] = script["result_path"]
env["DOWNLOAD_PATH"] = script["download_path"]
env["RUNTIME"] = str(timeout_seconds)
env["HAS_STARTED"] = str(script.get("has_started", False))
try:
script_arguments = parse_parameters(script, scripts_dir)
except KeyError as e:
# 2 is the return code bash gives when it can't execute.
script["exit_status"] = args["exit_status"] = 2
output = "Unable to run '%s': %s\n\n" % (
script["name"],
str(e).replace('"', "").replace("\\n", "\n"),
)
output += "Given parameters:\n%s\n\n" % str(
script.get("parameters", {}))
try:
output += "Discovered storage devices:\n%s\n" % str(
get_block_devices())
except KeyError:
pass
output += "Discovered interfaces:\n%s\n" % str(get_interfaces())
output = output.encode()
args["files"] = {
script["combined_name"]: output,
script["stderr_name"]: output,
}
output_and_send(
"Failed to execute %s: %d" %
(script["msg_name"], args["exit_status"]), **args)
return False
try:
# This script sets its own niceness value to the highest(-20) below
# to help ensure the heartbeat keeps running. When launching the
# script we need to lower the nice value as a child process
# inherits the parent processes niceness value. preexec_fn is
# executed in the child process before the command is run. When
# setting the nice value the kernel adds the current nice value
# to the provided value. Since the runner uses a nice value of -20
# setting it to 40 gives the actual nice value of 20.
proc = Popen(
script_arguments,
stdout=PIPE,
stderr=PIPE,
env=env,
preexec_fn=lambda: os.nice(40),
)
capture_script_output(
proc,
script["combined_path"],
script["stdout_path"],
script["stderr_path"],
timeout_seconds,
)
except OSError as e:
if isinstance(e.errno, int) and e.errno != 0:
script["exit_status"] = args["exit_status"] = e.errno
else:
# 2 is the return code bash gives when it can't execute.
script["exit_status"] = args["exit_status"] = 2
_check_link_connected(script)
stderr = str(e).encode()
if stderr == b"":
stderr = b"Unable to execute script"
args["files"] = {
script["combined_name"]: stderr,
script["stderr_name"]: stderr,
}
if os.path.exists(script["result_path"]):
args["files"][script["result_name"]] = open(
script["result_path"], "rb").read()
output_and_send(
"Failed to execute %s: %d" %
(script["msg_name"], args["exit_status"]), **args)
sys.stdout.write("%s\n" % stderr)
sys.stdout.flush()
return False
except TimeoutExpired:
# 124 is the exit status from the timeout command.
script["exit_status"] = args["exit_status"] = 124
args["status"] = "TIMEDOUT"
_check_link_connected(script)
args["files"] = {
script["combined_name"]: open(script["combined_path"],
"rb").read(),
script["stdout_name"]: open(script["stdout_path"], "rb").read(),
script["stderr_name"]: open(script["stderr_path"], "rb").read(),
}
if os.path.exists(script["result_path"]):
args["files"][script["result_name"]] = open(
script["result_path"], "rb").read()
output_and_send(
"Timeout(%s) expired on %s" %
(str(timedelta(seconds=timeout_seconds)), script["msg_name"]),
**args)
return False
else:
script["exit_status"] = args["exit_status"] = proc.returncode
_check_link_connected(script)
args["files"] = {
script["combined_name"]: open(script["combined_path"],
"rb").read(),
script["stdout_name"]: open(script["stdout_path"], "rb").read(),
script["stderr_name"]: open(script["stderr_path"], "rb").read(),
}
if os.path.exists(script["result_path"]):
args["files"][script["result_name"]] = open(
script["result_path"], "rb").read()
output_and_send(
"Finished %s: %s" % (script["msg_name"], args["exit_status"]),
**args)
if proc.returncode != 0:
return False
else:
return True
def run_scripts(url, creds, scripts_dir, out_dir, scripts):
"""Run and report results for the given scripts."""
total_scripts = len(scripts)
fail_count = 0
base_args = {
'url': url,
'creds': creds,
'status': 'WORKING',
}
for i, script in enumerate(scripts):
i += 1
args = copy.deepcopy(base_args)
args['script_result_id'] = script['script_result_id']
script_version_id = script.get('script_version_id')
if script_version_id is not None:
args['script_version_id'] = script_version_id
timeout_seconds = script.get('timeout_seconds')
for param in script.get('parameters', {}).values():
if param.get('type') == 'runtime':
timeout_seconds = param['value']
break
# Create a seperate output directory for each script being run as
# multiple scripts with the same name may be run.
script_out_dir = os.path.join(
out_dir, '%s.%s' % (script['name'], script['script_result_id']))
os.makedirs(script_out_dir, exist_ok=True)
combined_path = os.path.join(script_out_dir, script['name'])
stdout_name = '%s.out' % script['name']
stdout_path = os.path.join(script_out_dir, stdout_name)
stderr_name = '%s.err' % script['name']
stderr_path = os.path.join(script_out_dir, stderr_name)
result_name = '%s.yaml' % script['name']
result_path = os.path.join(script_out_dir, result_name)
download_path = os.path.join(scripts_dir, 'downloads', script['name'])
if not install_dependencies(args, script, combined_path, stdout_path,
stderr_path, download_path):
fail_count += 1
continue
signal_wrapper(error='Starting %s [%d/%d]' %
(script['name'], i, len(scripts)),
**args)
env = copy.deepcopy(os.environ)
env['OUTPUT_COMBINED_PATH'] = combined_path
env['OUTPUT_STDOUT_PATH'] = stdout_path
env['OUTPUT_STDERR_PATH'] = stderr_path
env['RESULT_PATH'] = result_path
env['DOWNLOAD_PATH'] = download_path
try:
script_arguments = parse_parameters(script, scripts_dir)
except KeyError:
# 2 is the return code bash gives when it can't execute.
args['exit_status'] = 2
args['files'] = {
script['name']: b'Unable to map parameters',
stderr_name: b'Unable to map parameters',
}
signal_wrapper(
error='Failed to execute %s [%d/%d]: %d' %
(script['name'], i, total_scripts, args['exit_status']),
**args)
continue
try:
# This script sets its own niceness value to the highest(-20) below
# to help ensure the heartbeat keeps running. When launching the
# script we need to lower the nice value as a child process
# inherits the parent processes niceness value. preexec_fn is
# executed in the child process before the command is run. When
# setting the nice value the kernel adds the current nice value
# to the provided value. Since the runner uses a nice value of -20
# setting it to 40 gives the actual nice value of 20.
proc = Popen(script_arguments,
stdout=PIPE,
stderr=PIPE,
env=env,
preexec_fn=lambda: os.nice(40))
capture_script_output(proc, combined_path, stdout_path,
stderr_path, timeout_seconds)
except OSError as e:
fail_count += 1
if isinstance(e.errno, int) and e.errno != 0:
args['exit_status'] = e.errno
else:
# 2 is the return code bash gives when it can't execute.
args['exit_status'] = 2
result = str(e).encode()
if result == b'':
result = b'Unable to execute script'
args['files'] = {
script['name']: result,
stderr_name: result,
}
signal_wrapper(
error='Failed to execute %s [%d/%d]: %d' %
(script['name'], i, total_scripts, args['exit_status']),
**args)
except TimeoutExpired:
fail_count += 1
args['status'] = 'TIMEDOUT'
args['files'] = {
script['name']: open(combined_path, 'rb').read(),
stdout_name: open(stdout_path, 'rb').read(),
stderr_name: open(stderr_path, 'rb').read(),
}
if os.path.exists(result_path):
args['files'][result_name] = open(result_path, 'rb').read()
signal_wrapper(error='Timeout(%s) expired on %s [%d/%d]' %
(str(timedelta(seconds=timeout_seconds)),
script['name'], i, total_scripts),
**args)
else:
if proc.returncode != 0:
fail_count += 1
args['exit_status'] = proc.returncode
args['files'] = {
script['name']: open(combined_path, 'rb').read(),
stdout_name: open(stdout_path, 'rb').read(),
stderr_name: open(stderr_path, 'rb').read(),
}
if os.path.exists(result_path):
args['files'][result_name] = open(result_path, 'rb').read()
signal_wrapper(
error='Finished %s [%d/%d]: %d' %
(script['name'], i, len(scripts), args['exit_status']),
**args)
# Signal failure after running commissioning or testing scripts so MAAS
# transisitions the node into FAILED_COMMISSIONING or FAILED_TESTING.
if fail_count != 0:
signal_wrapper(url, creds, 'FAILED',
'%d scripts failed to run' % fail_count)
return fail_count
def run_script(script, scripts_dir, send_result=True):
args = copy.deepcopy(script['args'])
args['status'] = 'WORKING'
args['send_result'] = send_result
timeout_seconds = script.get('timeout_seconds')
for param in script.get('parameters', {}).values():
if param.get('type') == 'runtime':
timeout_seconds = param['value']
break
output_and_send('Starting %s' % script['msg_name'], **args)
env = copy.deepcopy(os.environ)
env['OUTPUT_COMBINED_PATH'] = script['combined_path']
env['OUTPUT_STDOUT_PATH'] = script['stdout_path']
env['OUTPUT_STDERR_PATH'] = script['stderr_path']
env['RESULT_PATH'] = script['result_path']
env['DOWNLOAD_PATH'] = script['download_path']
env['RUNTIME'] = str(timeout_seconds)
try:
script_arguments = parse_parameters(script, scripts_dir)
except KeyError as e:
# 2 is the return code bash gives when it can't execute.
script['exit_status'] = args['exit_status'] = 2
output = (
"Unable to run '%s': %s\n\n"
'Given parameters:\n%s\n\n'
'Discovered storage devices:\n%s\n' % (
script['name'], str(e).replace('"', '').replace('\\n', '\n'),
str(script.get('parameters', {})),
str(get_block_devices()),
)
)
output = output.encode()
args['files'] = {
script['combined_name']: output,
script['stderr_name']: output,
}
output_and_send(
'Failed to execute %s: %d' % (
script['msg_name'], args['exit_status']), **args)
return False
try:
# This script sets its own niceness value to the highest(-20) below
# to help ensure the heartbeat keeps running. When launching the
# script we need to lower the nice value as a child process
# inherits the parent processes niceness value. preexec_fn is
# executed in the child process before the command is run. When
# setting the nice value the kernel adds the current nice value
# to the provided value. Since the runner uses a nice value of -20
# setting it to 40 gives the actual nice value of 20.
proc = Popen(
script_arguments, stdout=PIPE, stderr=PIPE, env=env,
preexec_fn=lambda: os.nice(40))
capture_script_output(
proc, script['combined_path'], script['stdout_path'],
script['stderr_path'], timeout_seconds)
except OSError as e:
if isinstance(e.errno, int) and e.errno != 0:
script['exit_status'] = args['exit_status'] = e.errno
else:
# 2 is the return code bash gives when it can't execute.
script['exit_status'] = args['exit_status'] = 2
stderr = str(e).encode()
if stderr == b'':
stderr = b'Unable to execute script'
args['files'] = {
script['combined_name']: stderr,
script['stderr_name']: stderr,
}
output_and_send(
'Failed to execute %s: %d' % (
script['msg_name'], args['exit_status']), **args)
sys.stdout.write('%s\n' % stderr)
return False
except TimeoutExpired:
args['status'] = 'TIMEDOUT'
args['files'] = {
script['combined_name']: open(
script['combined_path'], 'rb').read(),
script['stdout_name']: open(script['stdout_path'], 'rb').read(),
script['stderr_name']: open(script['stderr_path'], 'rb').read(),
}
if os.path.exists(script['result_path']):
args['files'][script['result_name']] = open(
script['result_path'], 'rb').read()
output_and_send(
'Timeout(%s) expired on %s' % (
str(timedelta(seconds=timeout_seconds)), script['msg_name']),
**args)
return False
else:
script['exit_status'] = args['exit_status'] = proc.returncode
args['files'] = {
script['combined_name']: open(
script['combined_path'], 'rb').read(),
script['stdout_name']: open(script['stdout_path'], 'rb').read(),
script['stderr_name']: open(script['stderr_path'], 'rb').read(),
}
if os.path.exists(script['result_path']):
args['files'][script['result_name']] = open(
script['result_path'], 'rb').read()
output_and_send('Finished %s: %s' % (
script['msg_name'], args['exit_status']), **args)
if proc.returncode != 0:
return False
else:
return True