def nm_pid():
pid = None
out, _, code = nmci.run("systemctl show -pMainPID NetworkManager.service")
if code == 0:
pid = int(out.split('=')[-1])
if not pid:
out, _, code = nmci.run("pgrep NetworkManager")
if code == 0:
pid = int(out)
return pid
def teardown_libreswan(context):
context.run("sh prepare/libreswan.sh teardown")
print("Attach Libreswan logs")
nmci.run(
"sudo journalctl -t pluto --no-pager -o cat %s > /tmp/journal-pluto.log"
% context.log_cursor)
journal_log = utf_only_open_read("/tmp/journal-pluto.log")
conf = utf_only_open_read("/opt/ipsec/connection.conf")
context.embed("text/plain", journal_log, caption="Libreswan Pluto Journal")
context.embed("text/plain", conf, caption="Libreswan Config")
def reinitialize_devices():
if nmci.command_code('systemctl is-active ModemManager') != 0:
nmci.run('systemctl restart ModemManager')
timer = 40
while nmci.command_code("nmcli device |grep gsm") != 0:
time.sleep(1)
timer -= 1
if timer == 0:
break
if nmci.command_code('nmcli d |grep gsm') != 0:
print("reinitialize devices")
reset_usb_devices()
nmci.run(
'for i in $(ls /sys/bus/usb/devices/usb*/authorized); do echo 0 > $i; done'
)
nmci.run(
'for i in $(ls /sys/bus/usb/devices/usb*/authorized); do echo 1 > $i; done'
)
nmci.run('systemctl restart ModemManager')
timer = 80
while nmci.command_code("nmcli device |grep gsm") != 0:
time.sleep(1)
timer -= 1
if timer == 0:
assert False, "Cannot initialize modem"
time.sleep(60)
return True
def _command_output_err(command, *a, **kw):
out, err, code = nmci.run(command, *a, **kw)
assert code == 0, "command '%s' exited with code %d\noutput:\n%s\nstderr:\n%s" \
% (command, code, out, err)
command_calls = getattr(context, "command_calls", [])
command_calls.append((command, code, out, err))
return out, err
def dump_status(context, when, fail_only=False):
nm_running = nmci.command_code('systemctl status NetworkManager') == 0
msg = ""
cmds = ['date "+%Y%m%d-%H%M%S.%N"']
if nm_running:
cmds += ['NetworkManager --version']
cmds += ['ip addr', 'ip -4 route', 'ip -6 route']
if nm_running:
cmds += [
'nmcli g', 'nmcli c', 'nmcli d', 'nmcli d w l', 'hostnamectl',
'NetworkManager --print-config', 'cat /etc/resolv.conf',
'ps aux | grep dhclient'
]
for cmd in cmds:
msg += "\n--- %s ---\n" % cmd
cmd_out, _, _ = nmci.run(cmd)
msg += cmd_out
if nm_running:
if os.path.isfile('/tmp/nm_newveth_configured'):
msg += "\nVeth setup network namespace and DHCP server state:\n"
for cmd in [
'ip netns exec vethsetup ip addr',
'ip netns exec vethsetup ip -4 route',
'ip netns exec vethsetup ip -6 route',
'ps aux | grep dnsmasq'
]:
msg += "\n--- %s ---\n" % cmd
cmd_out, _, _ = nmci.run(cmd)
msg += cmd_out
context.embed("text/plain", msg, "Status " + when, fail_only=fail_only)
# Always include memory stats
if context.nm_pid is not None:
msg = "Daemon memory consumption: %d KiB\n" % nm_size_kb()
if os.path.isfile("/etc/systemd/system/NetworkManager.service") \
and nmci.command_code(
"grep -q valgrind /etc/systemd/system/NetworkManager.service") == 0:
cmd_out, _, _ = nmci.run(
"LOGNAME=root HOSTNAME=localhost gdb /usr/sbin/NetworkManager "
" -ex 'target remote | vgdb' -ex 'monitor leak_check summary' -batch"
)
msg += cmd_out
context.embed("text/plain", msg, "Memory use " + when, fail_only=False)
def project_git_commit(self):
r = getattr(self, "_project_git_commit", None)
if r is None:
r, _, rc = nmci.run("git rev-parse HEAD")
r = r.strip("\n")
if rc != 0:
r = False
self._project_git_commit = r
return r
def project_git_url(self):
r = getattr(self, "_project_git_url", None)
if r is None:
r, _, rc = nmci.run("git config --get remote.origin.url")
r = r.strip("\n")[:-4]
if rc != 0:
r = False
elif r.startswith("git@"):
r = r.replace(":", "/").replace("git@", "https://")
self._project_git_url = r
return r
def dump_status_nmtui(context, when, fail_only=False):
msg = ""
cmds = [
'date "+%Y%m%d-%H%M%S.%N"', 'ip link', 'ip addr', 'ip -4 route',
'ip -6 route', 'nmcli g', 'nmcli c', 'nmcli d', 'nmcli d w l'
]
for cmd in cmds:
msg += "\n--- %s ---\n" % cmd
cmd_out, _, _ = nmci.run(cmd)
msg += cmd_out
context.embed("text/plain", msg, "Status " + when, fail_only=fail_only)
def setup_hostapd(context):
wait_for_testeth0(context)
arch = nmci.command_output("uname -p").strip()
if arch != "s390x":
# Install under RHEL7 only
if nmci.command_code("grep -q Maipo /etc/redhat-release") == 0:
nmci.run(
"[ -f /etc/yum.repos.d/epel.repo ] || sudo rpm -i http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm"
)
nmci.run(
"[ -x /usr/sbin/hostapd ] || (yum -y install hostapd; sleep 10)")
if nmci.command_code("sh prepare/hostapd_wired.sh tmp/8021x/certs") != 0:
nmci.run("sh prepare/hostapd_wired.sh teardown")
assert False, "hostapd setup failed"
def nm_size_kb():
memsize = 0
pid = nm_pid()
if not pid:
print(
"Warning: unable to get mem usage, NetworkManager is not running!")
return 0
out, _, rc = nmci.run("sudo cat /proc/%d/smaps" % pid)
if rc != 0:
print(
"Warning: unable to get mem usage, smaps file missing for NetworkManager process!"
)
return 0
smaps = out.strip("\n").split("\n")
for line in smaps:
fields = line.split()
if not fields[0] in ('Private_Dirty:', 'Swap:'):
continue
memsize += int(fields[1])
return memsize
def _command_code(command, *a, **kw):
out, err, code = nmci.run(command, *a, **kw)
command_calls = getattr(context, "command_calls", [])
command_calls.append((command, code, out, err))
return code
def _run(command, *a, **kw):
out, err, code = nmci.run(command, *a, **kw)
context._command_calls.append((command, code, out, err))
return out, err, code
def after_scenario(context, scenario):
nm_pid_after = nmci.lib.nm_pid()
if not nm_pid_after:
print("Starting NM as it was found stopped")
nmci.lib.restart_NM_service()
if IS_NMTUI:
# record the network status after the test
if os.path.isfile('/tmp/tui-screen.log'):
fd = open('/tmp/tui-screen.log', 'a+')
nmci.lib.dump_status_nmtui(fd, 'after')
fd.flush()
fd.close()
if os.path.isfile('/tmp/tui-screen.log'):
context.embed("text/plain",
nmci.lib.utf_only_open_read('/tmp/tui-screen.log'),
caption="TUI")
# Stop TUI
nmci.run("sudo killall nmtui &> /dev/null")
os.remove('/tmp/nmtui.out')
# Attach journalctl logs if failed
if scenario.status == 'failed' and hasattr(context, "embed"):
logs = nmci.lib.NM_log(context.log_cursor) or "NM log is empty!"
context.embed('text/plain', logs, caption="NM")
else:
print(("NetworkManager process id after: %s (was %s)" %
(nm_pid_after, context.nm_pid)))
if scenario.status == 'failed':
nmci.lib.dump_status_nmcli(context, 'after %s' % scenario.name)
# run after_scenario tags (in reverse order)
excepts = []
tag_registry = list(nmci.tags.tag_registry)
tag_registry.reverse()
for tag in tag_registry:
if tag.tag_name in scenario.tags and tag.after_scenario is not None:
try:
tag.after_scenario(context, scenario)
except Exception:
excepts.append(traceback.format_exc())
if not IS_NMTUI:
# check for crash reports and embed them
# sets crash_embeded and crashed_step, if crash found
nmci.lib.check_coredump(context, 'no_abrt' not in scenario.tags)
nmci.lib.check_faf(context, 'no_abrt' not in scenario.tags)
if scenario.status == 'failed' or context.crashed_step:
nmci.lib.dump_status_nmcli(context,
'after cleanup %s' % scenario.name)
# Attach journalctl logs
print("Attaching NM log")
log = "~~~~~~~~~~~~~~~~~~~~~~~~~~ NM LOG ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
log += nmci.lib.NM_log(
context.log_cursor)[:20000001] or "NM log is empty!"
context.embed('text/plain', log, caption="NM")
if nm_pid_after is not None and context.nm_pid == nm_pid_after:
context.log.write(
"NetworkManager memory consumption after: %d KiB\n" %
nmci.lib.nm_size_kb())
if os.path.isfile("/etc/systemd/system/NetworkManager.service") \
and nmci.command_code(
"grep -q valgrind /etc/systemd/system/NetworkManager.service") == 0:
time.sleep(3)
nmci.run(
"LOGNAME=root HOSTNAME=localhost gdb /usr/sbin/NetworkManager "
" -ex 'target remote | vgdb' -ex 'monitor leak_check summary' -batch",
stdout=context.log,
stderr=context.log)
context.log.close()
print("Attaching MAIN log")
log = nmci.lib.utf_only_open_read("/tmp/log_%s.html" % scenario.name)
context.embed('text/plain', log, caption="MAIN")
if context.crashed_step:
print("\n\n" + ("!" * 80))
print(
"!! NM CRASHED. NEEDS INSPECTION. FAILING THE TEST !!"
)
print("!! %-74s !!" % ("CRASHING STEP: " + context.crashed_step))
print(("!" * 80) + "\n\n")
context.embed('text/plain',
context.crashed_step,
caption="CRASHED_STEP_NAME")
if not context.crash_embeded:
msg = "!!! no crash report detected, but NM PID changed !!!"
context.embed('text/plain', msg, caption="NO_COREDUMP/NO_FAF")
assert not excepts, "Exceptions in after_scenario(): \n " + "\n\n".join(
excepts)
def before_all(context):
def on_signal(signum, frame):
assert False, "killed externally (timeout)"
signal.signal(signal.SIGTERM, on_signal)
signal.signal(signal.SIGINT, on_signal)
context.no_step = True
def embed_data(mime_type, data, caption):
# If data is empty we want to finish html tag by at least one character
non_empty_data = " " if not data else data
if context.no_step:
print(
f"Embed in before_scenario ({mime_type}):\n== {caption} ==\n\n{data}\n\n"
)
for formatter in context._runner.formatters:
if "html" in formatter.name and getattr(formatter, "embedding",
None) is not None:
formatter.embedding(mime_type=mime_type,
data=non_empty_data,
caption=caption)
return True
return False
def _set_title(title, append=False, tag="span", **kwargs):
for formatter in context._runner.formatters:
if "html" in formatter.name and getattr(formatter, "set_title",
None) is not None:
formatter.set_title(title=title,
append=append,
tag=tag,
**kwargs)
return True
return False
def _run(command, *a, **kw):
out, err, code = nmci.run(command, *a, **kw)
command_calls = getattr(context, "command_calls", [])
command_calls.append((command, code, out, err))
return out, err, code
def _command_output(command, *a, **kw):
out, err, code = nmci.run(command, *a, **kw)
assert code == 0, "command '%s' exited with code %d\noutput:\n%s\nstderr:\n%s" \
% (command, code, out, err)
command_calls = getattr(context, "command_calls", [])
command_calls.append((command, code, out, err))
return out
def _command_output_err(command, *a, **kw):
out, err, code = nmci.run(command, *a, **kw)
assert code == 0, "command '%s' exited with code %d\noutput:\n%s\nstderr:\n%s" \
% (command, code, out, err)
command_calls = getattr(context, "command_calls", [])
command_calls.append((command, code, out, err))
return out, err
def _command_code(command, *a, **kw):
out, err, code = nmci.run(command, *a, **kw)
command_calls = getattr(context, "command_calls", [])
command_calls.append((command, code, out, err))
return code
context.embed = embed_data
context.set_title = _set_title
context.command_code = _command_code
context.run = _run
context.command_output = _command_output
context.command_output_err = _command_output_err
if IS_NMTUI:
"""
Being executed before all features
"""
# Kill initial setup
nmci.run("sudo pkill nmtui")
# Store scenario start cursor for session logs
context.log_cursor = nmci.lib.new_log_cursor()
def before_scenario(context, scenario):
# set important context attributes
context.nm_restarted = False
context.nm_pid = nmci.lib.nm_pid()
context.crashed_step = False
context.log_cursor = ""
context.arch = nmci.command_output("uname -p").strip()
context.IS_NMTUI = IS_NMTUI
context.rh_release = nmci.command_output("cat /etc/redhat-release")
if IS_NMTUI:
os.environ['TERM'] = 'dumb'
# Do the cleanup
if os.path.isfile('/tmp/tui-screen.log'):
os.remove('/tmp/tui-screen.log')
fd = open('/tmp/tui-screen.log', 'a+')
nmci.lib.dump_status_nmtui(fd, 'before')
fd.write('Screen recordings after each step:' +
'\n----------------------------------\n')
fd.flush()
fd.close()
context.log = None
else:
if not os.path.isfile('/tmp/nm_wifi_configured') \
and not os.path.isfile('/tmp/nm_dcb_inf_wol_sriov_configured'):
if nmci.command_code(
"nmcli device |grep testeth0 |grep ' connected'") != 0:
nmci.run(
"sudo nmcli connection modify testeth0 ipv4.may-fail no")
nmci.run("sudo nmcli connection up id testeth0")
for attempt in range(0, 10):
if nmci.command_code(
"nmcli device |grep testeth0 |grep ' connected'"
) == 0:
break
time.sleep(1)
os.environ['TERM'] = 'dumb'
context.log = open('/tmp/log_%s.html' % scenario.name, 'w')
# dump status before the test preparation starts
nmci.lib.dump_status_nmcli(context, 'before %s' % scenario.name)
context.start_timestamp = int(time.time())
excepts = []
if 'eth0' in scenario.tags \
or 'delete_testeth0' in scenario.tags \
or 'connect_testeth0' in scenario.tags \
or 'restart' in scenario.tags \
or 'dummy' in scenario.tags:
try:
nmci.tags.skip_restarts_bs(context, scenario)
except Exception as e:
excepts.append(str(e))
for tag in nmci.tags.tag_registry:
if tag.tag_name in scenario.tags and tag.before_scenario is not None:
try:
tag.before_scenario(context, scenario)
except Exception:
excepts.append(traceback.format_exc())
assert not excepts, "Exceptions in before_scenario():\n" + "\n\n".join(
excepts)
context.nm_pid = nmci.lib.nm_pid()
context.crashed_step = False
print(("NetworkManager process id before: %s" % context.nm_pid))
if context.nm_pid is not None and context.log is not None:
context.log.write(
"NetworkManager memory consumption before: %d KiB\n" %
nmci.lib.nm_size_kb())
if os.path.isfile("/etc/systemd/system/NetworkManager.service") \
and nmci.command_code(
"grep -q valgrind /etc/systemd/system/NetworkManager.service") == 0:
nmci.run(
"LOGNAME=root HOSTNAME=localhost gdb /usr/sbin/NetworkManager "
" -ex 'target remote | vgdb' -ex 'monitor leak_check summary' -batch",
stdout=context.log,
stderr=context.log)
context.log_cursor = nmci.lib.new_log_cursor()
def _before_scenario(context, scenario):
time_begin = time.time()
context.before_scenario_step_el = ET.Element("li", {
"class": "step passed",
"style": "margin-bottom:1rem;"
})
ET.SubElement(context.before_scenario_step_el,
"b").text = "Before scenario"
duration_el = ET.SubElement(context.before_scenario_step_el, "small",
{"class": "step_duration"})
embed_el = ET.SubElement(context.before_scenario_step_el, "div")
context.html_formatter.actual["act_step_embed_span"] = embed_el
# set important context attributes
context.nm_restarted = False
context.nm_pid = nmci.lib.nm_pid()
context.crashed_step = False
context.log_cursor = ""
context.log_cursor_before_tags = nmci.lib.new_log_cursor()
context.arch = nmci.command_output("uname -p").strip()
context.IS_NMTUI = "nmtui" in scenario.effective_tags
context.rh_release = nmci.command_output("cat /etc/redhat-release")
release_i = context.rh_release.find("release ")
if release_i >= 0:
context.rh_release_num = float(
context.rh_release[release_i:].split(" ")[1])
else:
context.rh_release_num = 0
context.hypervisor = nmci.run("systemd-detect-virt")[0].strip()
os.environ['TERM'] = 'dumb'
# dump status before the test preparation starts
nmci.lib.dump_status(context, 'Before Scenario', fail_only=True)
if context.IS_NMTUI:
nmci.run("sudo pkill nmtui")
# Do the cleanup
if os.path.isfile('/tmp/tui-screen.log'):
os.remove('/tmp/tui-screen.log')
fd = open('/tmp/tui-screen.log', 'a+')
fd.write('Screen recordings after each step:' +
'\n----------------------------------\n')
fd.flush()
fd.close()
else:
if not os.path.isfile('/tmp/nm_wifi_configured') \
and not os.path.isfile('/tmp/nm_dcb_inf_wol_sriov_configured'):
if nmci.command_code(
"nmcli device |grep testeth0 |grep ' connected'") != 0:
nmci.run(
"sudo nmcli connection modify testeth0 ipv4.may-fail no")
nmci.run("sudo nmcli connection up id testeth0")
for attempt in range(0, 10):
if nmci.command_code(
"nmcli device |grep testeth0 |grep ' connected'"
) == 0:
break
time.sleep(1)
context.start_timestamp = int(time.time())
excepts = []
if 'eth0' in scenario.tags \
or 'delete_testeth0' in scenario.tags \
or 'connect_testeth0' in scenario.tags \
or 'restart' in scenario.tags \
or 'dummy' in scenario.tags:
try:
nmci.tags.skip_restarts_bs(context, scenario)
except Exception as e:
excepts.append(str(e))
for tag_name in scenario.tags:
tag = nmci.tags.tag_registry.get(tag_name, None)
if tag is not None and tag.before_scenario is not None:
print("Executing @" + tag_name)
t_start = time.time()
t_status = "passed"
try:
tag.before_scenario(context, scenario)
except Exception:
t_status = "failed"
excepts.append(traceback.format_exc())
print(
f" @{tag_name} ... {t_status} in {time.time() - t_start:.3f}s"
)
context.nm_pid = nmci.lib.nm_pid()
context.crashed_step = False
print(("NetworkManager process id before: %s" % context.nm_pid))
context.log_cursor = nmci.lib.new_log_cursor()
nmci.lib.process_commands(context, "before_scenario")
duration = time.time() - time_begin
status = "failed" if excepts else "passed"
print(f"before_scenario ... {status} in {duration:.3f}s")
duration_el.text = f"({duration:.3f}s)"
if excepts:
context.before_scenario_step_el.set("class", "step failed")
context.embed("text/plain", "\n\n".join(excepts),
"Exception in before scenario tags")
assert False, "Exception in before scenario tags"
def _after_scenario(context, scenario):
time_begin = time.time()
context.after_scenario_step_el = ET.Element("li", {
"class": "step passed",
"style": "margin-top:1rem;"
})
ET.SubElement(context.after_scenario_step_el, "b").text = "After scenario"
duration_el = ET.SubElement(context.after_scenario_step_el, "small",
{"class": "step_duration"})
embed_el = ET.SubElement(context.after_scenario_step_el, "div")
context.html_formatter.actual["act_step_embed_span"] = embed_el
nmci.misc.html_report_tag_links(context.html_formatter.scenario_el)
nmci.misc.html_report_file_links(context.html_formatter.scenario_el)
nm_pid_after = nmci.lib.nm_pid()
if not nm_pid_after:
print("Starting NM as it was found stopped")
nmci.lib.restart_NM_service(context)
if context.IS_NMTUI:
if os.path.isfile('/tmp/tui-screen.log'):
context.embed("text/plain",
nmci.lib.utf_only_open_read('/tmp/tui-screen.log'),
caption="TUI")
# Stop TUI
nmci.run("sudo killall nmtui &> /dev/null")
if os.path.isfile('/tmp/nmtui.out'):
os.remove('/tmp/nmtui.out')
print(("NetworkManager process id after: %s (was %s)" %
(nm_pid_after, context.nm_pid)))
if scenario.status == 'failed' or DEBUG:
nmci.lib.dump_status(context, 'After Scenario', fail_only=True)
# run after_scenario tags (in reverse order)
excepts = []
scenario_tags = list(scenario.tags)
scenario_tags.reverse()
for tag_name in scenario_tags:
tag = nmci.tags.tag_registry.get(tag_name, None)
if tag is not None and tag.after_scenario is not None:
print("Executing @" + tag_name)
t_start = time.time()
t_status = "passed"
try:
tag.after_scenario(context, scenario)
except Exception:
t_status = "failed"
excepts.append(traceback.format_exc())
print(
f" @{tag_name} ... {t_status} in {time.time() - t_start:.3f}s"
)
# check for crash reports and embed them
# sets crash_embeded and crashed_step, if crash found
nmci.lib.check_coredump(context, 'no_abrt' not in scenario.tags)
nmci.lib.check_faf(context, 'no_abrt' not in scenario.tags)
nmci.lib.process_commands(context, "after_scenario")
scenario_fail = scenario.status == 'failed' or context.crashed_step or DEBUG or len(
excepts) > 0
# Attach postponed or "fail_only" embeds
# !!! all embed calls with "fail_only" after this are ignored !!!
nmci.lib.process_embeds(context, scenario_fail)
nmci.lib.dump_status(context, 'After Clean', fail_only=False)
if scenario_fail:
# Attach journalctl logs
print("Attaching NM log")
log = "~~~~~~~~~~~~~~~~~~~~~~~~~~ NM LOG ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
log += nmci.lib.NM_log(
context.log_cursor)[:20000001] or "NM log is empty!"
context.embed('text/plain', log, caption="NM")
if context.crashed_step:
print("\n\n" + ("!" * 80))
print(
"!! NM CRASHED. NEEDS INSPECTION. FAILING THE TEST !!"
)
print("!! %-74s !!" % ("CRASHING STEP: " + context.crashed_step))
print(("!" * 80) + "\n\n")
context.embed('text/plain',
context.crashed_step,
caption="CRASHED_STEP_NAME")
if not context.crash_embeded:
msg = "!!! no crash report detected, but NM PID changed !!!"
context.embed('text/plain', msg, caption="NO_COREDUMP/NO_FAF")
nmci.lib.after_crash_reset(context)
if excepts or context.crashed_step:
context.after_scenario_step_el.set("class", "step failed")
if excepts:
context.embed("text/plain", "\n\n".join(excepts),
"Exception in after scenario tags")
# add Before/After scenario steps to HTML
context.html_formatter.steps.insert(0, context.before_scenario_step_el)
context.html_formatter.steps.append(context.after_scenario_step_el)
duration = time.time() - time_begin
status = "failed" if excepts else "passed"
print(f"after_scenario ... {status} in {duration:.3f}s")
duration_el.text = f"({duration:.3f}s)"
# we need to keep state "passed" here, as '@crash' test is expected to fail
if 'crash' in scenario.effective_tags and not context.crash_embeded:
print("No crashdump found")
return
if context.crashed_step:
assert False, "Crash happened"
assert not excepts, "Exception in after scenario tags"
def list_dumps(dumps_search):
out, err, code = nmci.run("ls -d %s" % (dumps_search))
if code != 0:
return []
return out.strip('\n').split('\n')
def get_service_log(service, journal_arg):
return nmci.run("journalctl --all --no-pager %s | grep ' %s\\['" %
(journal_arg, service))[0]
