def dig_call(port=53, server="127.0.0.1", commands=None):
"""Call `dig` with the given command.
Note that calling dig without a command will perform an NS
query for "." (the root) which is useful to check if there
is a running server.
:param port: Port of the queried DNS server (defaults to 53).
:param server: IP address of the queried DNS server (defaults
to '127.0.0.1').
:param commands: List of dig commands to run (defaults to None
which will perform an NS query for "." (the root)).
:return: The output as a string.
:rtype: str
"""
# The time and tries below are high so that tests pass in environments
# that are much slower than the average developer's machine, so beware
# before lowering. Many Bothans died to discover these parameters.
cmd = ["dig", "+time=10", "+tries=5", "@%s" % server, "-p", "%d" % port]
if commands is not None:
if not isinstance(commands, list):
commands = (commands, )
cmd.extend(commands)
output = check_output(cmd, env=get_env_with_locale())
return output.decode("utf-8").strip()
def run_nmap(args: NmapParameters) -> dict:
"""Runs `nmap` to scan the target using the specified arguments.
Note: This function is required to take a single parameter, due to the way
Pool.imap(...) works.
:param args: NmapParameters namedtuple
:return: dict representing the event that occurred.
"""
clock = time.monotonic()
nmap = subprocess.Popen(
get_nmap_arguments(args),
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
env=get_env_with_locale(),
# This pre-exec function prevents `nmap` from writing to the pty, which
# misbehaves when it runs in parallel and can require a `reset`.
preexec_fn=os.setsid,
)
nmap.wait()
clock_diff = time.monotonic() - clock
event = {
"scan_type": "nmap",
"interface": args.interface,
"cidr": args.cidr,
"returncode": nmap.returncode,
"seconds": int(clock_diff),
"slow": args.slow,
}
return event
def _issue_ipmi_chassis_config_command(command, power_change,
power_address):
env = shell.get_env_with_locale()
with NamedTemporaryFile("w+", encoding="utf-8") as tmp_config:
# Write out the chassis configuration.
tmp_config.write(IPMI_CONFIG)
tmp_config.flush()
# Use it when running the chassis config command.
# XXX: Not using call_and_check here because we
# need to check stderr.
command = tuple(command) + ("--filename", tmp_config.name)
process = Popen(command, stdout=PIPE, stderr=PIPE, env=env)
_, stderr = process.communicate()
stderr = stderr.decode("utf-8").strip()
# XXX newell 2016-11-21 bug=1516065: Some IPMI hardware have timeout
# issues when trying to set the boot order to PXE. We want to
# continue and not raise an error here.
ipmi_errors = {
key: IPMI_ERRORS[key]
for key in IPMI_ERRORS
if IPMI_ERRORS[key]['exception'] == PowerAuthError
}
for error, error_info in ipmi_errors.items():
if error in stderr:
raise error_info.get('exception')(error_info.get('message'))
if process.returncode != 0:
maaslog.warning("Failed to change the boot order to PXE %s: %s" %
(power_address, stderr))
def test__issue_ipmi_command_issues_power_off_soft_mode(self):
context = make_context()
context['power_off_mode'] = 'soft'
ipmi_chassis_config_command = make_ipmi_chassis_config_command(
**context, tmp_config_name=ANY)
ipmipower_command = make_ipmipower_command(**context)
ipmipower_command += ('--soft', )
ipmi_power_driver = IPMIPowerDriver()
env = get_env_with_locale()
popen_mock = self.patch(ipmi_module, 'Popen')
process = popen_mock.return_value
process.communicate.side_effect = [(b'', b''), (b'off', b'')]
process.returncode = 0
result = ipmi_power_driver._issue_ipmi_command('off', **context)
self.expectThat(
popen_mock,
MockCallsMatch(
call(ipmi_chassis_config_command,
stdout=PIPE,
stderr=PIPE,
env=env),
call(ipmipower_command, stdout=PIPE, stderr=PIPE, env=env)))
self.expectThat(result, Equals('off'))
def _issue_fence_cdu_command(self,
command,
power_address=None,
power_id=None,
power_user=None,
power_pass=None,
**extra):
"""Issue fence_cdu command for the given power change."""
try:
stdout = call_and_check([
'fence_cdu', '-a', power_address, '-n', power_id, '-l',
power_user, '-p', power_pass, '-o', command
],
env=get_env_with_locale())
except ExternalProcessError as e:
# XXX 2016-01-08 newell-jensen, bug=1532310:
# fence-agents fence_action method returns an exit code
# of 2, by default, for querying power status while machine
# is OFF.
if e.returncode == 2 and command == 'status':
return "Status: OFF\n"
else:
raise PowerError(
"Fence CDU failed issuing command %s for Power ID %s: %s" %
(command, power_id, e.output_as_unicode))
else:
return stdout.decode("utf-8")
def test_set_outlet_state_calls_wget(self):
driver = dli_module.DLIPowerDriver()
env = get_env_with_locale()
power_change = factory.make_name("power_change")
outlet_id = choice(["1", "2", "3", "4", "5", "6", "7", "8"])
power_user = factory.make_name("power_user")
power_pass = factory.make_name("power_pass")
power_address = factory.make_name("power_address")
url = "http://%s:%s@%s/outlet?%s=%s" % (
power_user,
power_pass,
power_address,
outlet_id,
power_change,
)
call_and_check_mock = self.patch(dli_module, "call_and_check")
driver._set_outlet_state(power_change, outlet_id, power_user,
power_pass, power_address)
self.assertThat(
call_and_check_mock,
MockCalledOnceWith(
["wget", "--auth-no-challenge", "-O", "/dev/null", url],
env=env,
),
)
def test__issue_fence_cdu_command(self):
driver = fence_cdu_module.FenceCDUPowerDriver()
mock = self.patch(fence_cdu_module, "call_and_check")
mock.return_value = b"test"
stdout = driver._issue_fence_cdu_command(
sentinel.command,
sentinel.power_address,
sentinel.power_id,
sentinel.power_user,
sentinel.power_pass,
)
self.expectThat(
mock,
MockCalledOnceWith(
[
"fence_cdu",
"-a",
sentinel.power_address,
"-n",
sentinel.power_id,
"-l",
sentinel.power_user,
"-p",
sentinel.power_pass,
"-o",
sentinel.command,
],
env=get_env_with_locale(),
),
)
self.expectThat(stdout, Equals("test"))
def test__issue_ipmi_command_issues_power_on(self):
context = make_context()
ipmi_chassis_config_command = make_ipmi_chassis_config_command(
**context, tmp_config_name=ANY)
ipmipower_command = make_ipmipower_command(**context)
ipmipower_command += ("--cycle", "--on-if-off")
ipmi_power_driver = IPMIPowerDriver()
env = get_env_with_locale()
popen_mock = self.patch(ipmi_module, "Popen")
process = popen_mock.return_value
process.communicate.side_effect = [(b"", b""), (b"on", b"")]
process.returncode = 0
result = ipmi_power_driver._issue_ipmi_command("on", **context)
self.expectThat(
popen_mock,
MockCallsMatch(
call(
ipmi_chassis_config_command,
stdout=PIPE,
stderr=PIPE,
env=env,
),
call(ipmipower_command, stdout=PIPE, stderr=PIPE, env=env),
),
)
self.expectThat(result, Equals("on"))
def test_sets_LANG_and_LC_ALL(self):
self.assertThat(
get_env_with_locale({}),
Equals(
{"LANG": "C.UTF-8", "LANGUAGE": "C.UTF-8", "LC_ALL": "C.UTF-8"}
),
)
def test_overwrites_LANGUAGE(self):
self.assertThat(
get_env_with_locale({"LANGUAGE": factory.make_name("LANGUAGE")}),
Equals(
{"LANG": "C.UTF-8", "LANGUAGE": "C.UTF-8", "LC_ALL": "C.UTF-8"}
),
)
def test_defaults_to_process_environment(self):
name = factory.make_name("name")
value = factory.make_name("value")
with EnvironmentVariable(name, value):
self.assertThat(
get_env_with_locale(), ContainsDict({name: Equals(value)})
)
def _issue_ipmitool_command(self,
power_change,
power_address=None,
power_user=None,
power_pass=None,
power_hwaddress=None,
**extra):
"""Issue ipmitool command for HP Moonshot cartridge."""
command = ('ipmitool', '-I', 'lanplus', '-H', power_address, '-U',
power_user, '-P', power_pass) + tuple(
power_hwaddress.split())
if power_change == 'pxe':
command += ('chassis', 'bootdev', 'pxe')
else:
command += ('power', power_change)
try:
stdout = call_and_check(command, env=get_env_with_locale())
stdout = stdout.decode('utf-8')
except ExternalProcessError as e:
raise PowerActionError(
"Failed to execute %s for cartridge %s at %s: %s" %
(command, power_hwaddress, power_address, e.output_as_unicode))
else:
# Return output if power query
if power_change == 'status':
match = re.search(r'\b(on|off)\b$', stdout)
return stdout if match is None else match.group(0)
def _set_outlet_state(self,
power_change,
outlet_id=None,
power_user=None,
power_pass=None,
power_address=None,
**extra):
"""Power DLI outlet ON/OFF."""
try:
url = "http://%s:%s@%s/outlet?%s=%s" % (
power_user,
power_pass,
power_address,
outlet_id,
power_change,
)
# --auth-no-challenge: send Basic HTTP authentication
# information without first waiting for the server's challenge.
call_and_check(
["wget", "--auth-no-challenge", "-O", "/dev/null", url],
env=get_env_with_locale(),
)
except ExternalProcessError as e:
raise PowerActionError(
"Failed to power %s outlet %s: %s" %
(power_change, outlet_id, e.output_as_unicode))
def run_ping(args: PingParameters) -> dict:
"""Runs `ping` and returns the resulting event.
Note: This function is required to take a single parameter, due to the way
Pool.imap(...) works.
:param args: PingParameters namedtuple
:return: dict representing the event that occurred.
"""
ping = subprocess.Popen(
get_ping_arguments(args),
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
env=get_env_with_locale(),
)
ping.wait()
result = bool(ping.returncode == 0)
event = {
"scan_type": "ping",
"interface": args.interface,
"ip": args.ip,
"result": result,
}
return event
def validate_dhcpd_configuration(test, configuration, ipv6):
"""Validate `configuration` using `dhcpd` itself.
:param test: An instance of `maastesting.testcase.TestCase`.
:param configuration: The contents of the configuration file as a string.
:param ipv6: When true validate as DHCPv6, otherwise validate as DHCPv4.
"""
with tempfile.NamedTemporaryFile(
"w", encoding="ascii", prefix="dhcpd.",
suffix=".conf") as conffile, tempfile.NamedTemporaryFile(
"w", encoding="ascii", prefix="dhcpd.",
suffix=".leases") as leasesfile:
# Write the configuration to the temporary file.
conffile.write(configuration)
conffile.flush()
# Add line numbers to configuration and add as a detail. This will
# make it much easier to debug; `dhcpd -t` prints line numbers for any
# errors it finds.
test.addDetail(
conffile.name,
Content(
UTF8_TEXT,
lambda: map(
str.encode,
("> %3d %s" % entry for entry in zip(
count(1), configuration.splitlines(keepends=True))),
),
),
)
# Call `dhcpd` via `aa-exec --profile unconfined`. The latter is
# needed so that `dhcpd` can open the configuration file from /tmp.
cmd = (
"aa-exec",
"--profile",
"unconfined",
"dhcpd",
("-6" if ipv6 else "-4"),
"-t",
"-cf",
conffile.name,
"-lf",
leasesfile.name,
)
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=get_env_with_locale(),
)
command = " ".join(map(pipes.quote, process.args))
output, _ = process.communicate()
# Record the output from `dhcpd -t` as a detail.
test.addDetail(
"stdout/err from `%s`" % command,
text_content(output.decode("utf-8")),
)
# Check that it completed successfully.
test.assertThat(process.returncode, Equals(0),
"`%s` failed." % command)
def _query_outlet_state(
self,
outlet_id=None,
power_user=None,
power_pass=None,
power_address=None,
**extra
):
"""Query DLI outlet power state.
Sample snippet of query output from DLI:
...
<!--
function reg() {
window.open('http://www.digital-loggers.com/reg.html?SN=LPC751740');
}
//-->
</script>
</head>
<!-- state=02 lock=00 -->
<body alink="#0000FF" vlink="#0000FF">
<FONT FACE="Arial, Helvetica, Sans-Serif">
...
"""
try:
url = "http://%s:%s@%s/index.htm" % (
power_user,
power_pass,
power_address,
)
# --auth-no-challenge: send Basic HTTP authentication
# information without first waiting for the server's challenge.
wget_output = call_and_check(
["wget", "--auth-no-challenge", "-qO-", url],
env=get_env_with_locale(),
)
wget_output = wget_output.decode("utf-8")
match = re.search("<!-- state=([0-9a-fA-F]+)", wget_output)
if match is None:
raise PowerError(
"Unable to extract power state for outlet %s from "
"wget output: %s" % (outlet_id, wget_output)
)
else:
state = match.group(1)
# state is a bitmap of the DLI's oulet states, where bit 0
# corresponds to oulet 1's power state, bit 1 corresponds to
# outlet 2's power state, etc., encoded as hexadecimal.
if (int(state, 16) & (1 << int(outlet_id) - 1)) > 0:
return "on"
else:
return "off"
except ExternalProcessError as e:
raise PowerActionError(
"Failed to power query outlet %s: %s"
% (outlet_id, e.output_as_unicode)
)
def test_removes_other_LC_variables(self):
self.assertThat(
get_env_with_locale(
{name: factory.make_name(name) for name in LC_VAR_NAMES}
),
Equals(
{"LANG": "C.UTF-8", "LANGUAGE": "C.UTF-8", "LC_ALL": "C.UTF-8"}
),
)
def test__runs_ping_e2e(self):
ip = factory.make_ip_address(ipv6=False)
# Force the use of `ping` even if `nmap` is installed.
self.run_command('--ping', 'eth0', '%s/32' % ip)
expected_params = PingParameters(interface='eth0', ip=ip)
self.assertThat(self.popen, MockCalledOnceWith(
get_ping_arguments(expected_params),
stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL, env=get_env_with_locale()))
def test__passes_other_variables_through(self):
basis = {
factory.make_name("name"): factory.make_name("value")
for _ in range(5)
}
expected = basis.copy()
expected["LANG"] = expected["LC_ALL"] = expected["LANGUAGE"] = (
"C.UTF-8")
observed = get_env_with_locale(basis)
self.assertThat(observed, Equals(expected))
def test__issue_ipmitool_command_sets_pxe_boot(self):
context = make_context()
env = get_env_with_locale()
pxe_command = make_pxe_command(context)
moonshot_driver = MoonshotIPMIPowerDriver()
call_and_check_mock = self.patch(moonshot_module, "call_and_check")
moonshot_driver._issue_ipmitool_command("pxe", **context)
self.assertThat(call_and_check_mock,
MockCalledOnceWith(pxe_command, env=env))
def __init__(self, timeout=30, maxread=2000, dom_prefix=None):
super(VirshSSH, self).__init__(
None, timeout=timeout, maxread=maxread,
env=get_env_with_locale())
self.name = '<virssh>'
if dom_prefix is None:
self.dom_prefix = ''
else:
self.dom_prefix = dom_prefix
# Store a mapping of { machine_name: xml }.
self.xml = {}
def test_resets_locale(self):
"""
VirshSSH resets the locale to ensure we only ever get English strings.
"""
c_utf8_environment = get_env_with_locale()
mock_spawn = self.patch(pexpect.spawn, "__init__")
self.configure_virshssh('')
self.assertThat(
mock_spawn,
MockCalledOnceWith(None,
timeout=30,
maxread=2000,
env=c_utf8_environment))
def test__runs_popen_with_expected_parameters(self):
popen = self.patch(scan_network_module.subprocess, 'Popen')
popen.return_value.poll = Mock()
popen.return_value.poll.return_value = None
popen.return_value.returncode = 0
interface = factory.make_name('eth')
ip = factory.make_ip_address(ipv6=False)
params = PingParameters(interface, ip)
run_ping(params)
self.assertThat(popen, MockCalledOnceWith(
get_ping_arguments(params),
stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL, env=get_env_with_locale()))
def test__runs_popen_with_expected_parameters(self):
popen = self.patch(scan_network_module.subprocess, 'Popen')
popen.return_value.poll = Mock()
popen.return_value.poll.return_value = None
interface = factory.make_name('eth')
cidr = "192.168.0.0/24"
params = NmapParameters(interface, cidr, slow=False)
run_nmap(params)
self.assertThat(popen, MockCalledOnceWith(
get_nmap_arguments(params),
stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL, env=get_env_with_locale(),
preexec_fn=os.setsid))
def test__issue_ipmitool_command_returns_stdout_if_no_match(self):
context = make_context()
env = get_env_with_locale()
ipmitool_command = make_ipmitool_command("status", context)
moonshot_driver = MoonshotIPMIPowerDriver()
call_and_check_mock = self.patch(moonshot_module, "call_and_check")
call_and_check_mock.return_value = b"other"
result = moonshot_driver._issue_ipmitool_command("status", **context)
self.expectThat(call_and_check_mock,
MockCalledOnceWith(ipmitool_command, env=env))
self.expectThat(result, Equals("other"))
def run_script(self):
script = self.bindir.joinpath("virtuality")
script.write_text(VIRTUALITY_SCRIPT, "ascii")
script.chmod(0o700)
env = get_env_with_locale()
env["PATH"] = str(self.bindir)
try:
return check_output((str(script), ), stderr=STDOUT, env=env)
except CalledProcessError as error:
self.addDetail(
"output", text_content(error.output.decode("utf-8",
"replace")))
raise
def get_command_output(*command_line):
"""Execute a command line, and return its output.
Raises an exception if return value is nonzero.
:param *command_line: Words for the command line. No shell expansions
are performed.
:type *command_line: Sequence of unicode.
:return: Output from the command.
:rtype: List of unicode, one per line.
"""
env = get_env_with_locale()
output = check_output(command_line, env=env)
return output.decode("utf-8").splitlines()
def _issue_ipmipower_command(command, power_change, power_address):
env = shell.get_env_with_locale()
command = tuple(command) # For consistency when testing.
process = Popen(command, stdout=PIPE, stderr=PIPE, env=env)
stdout, _ = process.communicate()
stdout = stdout.decode("utf-8").strip()
for error, error_info in IPMI_ERRORS.items():
# ipmipower dumps errors to stdout
if error in stdout:
raise error_info.get('exception')(error_info.get('message'))
if process.returncode != 0:
raise PowerError("Failed to power %s %s: %s" %
(power_change, power_address, stdout))
match = re.search(":\s*(on|off)", stdout)
return stdout if match is None else match.group(1)
def test_runs_ping_single_threaded(self):
ip = factory.make_ip_address(ipv6=False)
# Force the use of `ping` even if `nmap` is installed.
self.run_command("--threads", "1", "--ping", "eth0", "%s/32" % ip)
expected_params = PingParameters(interface="eth0", ip=ip)
self.assertThat(
self.popen,
MockCalledOnceWith(
get_ping_arguments(expected_params),
stderr=subprocess.DEVNULL,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
env=get_env_with_locale(),
),
)
def log_something(name: str,
*,
verbosity: int,
set_verbosity: int = None,
mode: LoggingMode):
env = dict(get_env_with_locale(), PYTHONPATH=":".join(sys.path))
script = here.parent.joinpath("testing", "logsomething.py")
args = [
"--name", name, "--verbosity",
"%d" % verbosity, "--mode", mode.name
]
if set_verbosity is not None:
args.extend(["--set-verbosity", "%d" % set_verbosity])
cmd = [sys.executable, str(script)] + args
output = subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT)
return output.decode("utf-8")