def reinitialize_devices():
if nmci.command_code('systemctl is-active ModemManager') != 0:
nmci.run('systemctl restart ModemManager')
timer = 40
while nmci.command_code("nmcli device |grep gsm") != 0:
time.sleep(1)
timer -= 1
if timer == 0:
break
if nmci.command_code('nmcli d |grep gsm') != 0:
print("reinitialize devices")
reset_usb_devices()
nmci.run(
'for i in $(ls /sys/bus/usb/devices/usb*/authorized); do echo 0 > $i; done'
)
nmci.run(
'for i in $(ls /sys/bus/usb/devices/usb*/authorized); do echo 1 > $i; done'
)
nmci.run('systemctl restart ModemManager')
timer = 80
while nmci.command_code("nmcli device |grep gsm") != 0:
time.sleep(1)
timer -= 1
if timer == 0:
assert False, "Cannot initialize modem"
time.sleep(60)
return True
def setup_hostapd(context):
wait_for_testeth0(context)
arch = nmci.command_output("uname -p").strip()
if arch != "s390x":
# Install under RHEL7 only
if nmci.command_code("grep -q Maipo /etc/redhat-release") == 0:
nmci.run(
"[ -f /etc/yum.repos.d/epel.repo ] || sudo rpm -i http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm"
)
nmci.run(
"[ -x /usr/sbin/hostapd ] || (yum -y install hostapd; sleep 10)")
if nmci.command_code("sh prepare/hostapd_wired.sh tmp/8021x/certs") != 0:
nmci.run("sh prepare/hostapd_wired.sh teardown")
assert False, "hostapd setup failed"
def set_libreswan_connection(context, user, password, group, secret, gateway, name):
if nmci.command_code("rpm -qa | grep -q libreswan-4") == 0 and 'release 8' not in context.rh_release:
username_option = "leftusername"
else:
username_option = "leftxauthusername"
vpn_data = {
username_option: user,
"right": gateway,
"xauthpasswordinputmodes": "ask" if password == "ask" else "save",
"xauthpassword-flags": "2" if password == "ask" else "0",
"pskinputmodes": "ask" if secret == "ask" else "save",
"pskvalue-flags": "2" if secret == "ask" else "0",
"vendor": "Cisco",
}
if group != "Main":
vpn_data["leftid"] = group
vpn_secrets = {}
if password != "ask":
vpn_secrets["xauthpassword"] = password
if secret != "ask":
vpn_secrets["pskvalue"] = secret
context.execute_steps('''
* Modify connection "%s" changing options "%s"
''' % (name, vpn_options_str(vpn_data, vpn_secrets)))
def dump_status(context, when, fail_only=False):
nm_running = nmci.command_code('systemctl status NetworkManager') == 0
msg = ""
cmds = ['date "+%Y%m%d-%H%M%S.%N"']
if nm_running:
cmds += ['NetworkManager --version']
cmds += ['ip addr', 'ip -4 route', 'ip -6 route']
if nm_running:
cmds += [
'nmcli g', 'nmcli c', 'nmcli d', 'nmcli d w l', 'hostnamectl',
'NetworkManager --print-config', 'cat /etc/resolv.conf',
'ps aux | grep dhclient'
]
for cmd in cmds:
msg += "\n--- %s ---\n" % cmd
cmd_out, _, _ = nmci.run(cmd)
msg += cmd_out
if nm_running:
if os.path.isfile('/tmp/nm_newveth_configured'):
msg += "\nVeth setup network namespace and DHCP server state:\n"
for cmd in [
'ip netns exec vethsetup ip addr',
'ip netns exec vethsetup ip -4 route',
'ip netns exec vethsetup ip -6 route',
'ps aux | grep dnsmasq'
]:
msg += "\n--- %s ---\n" % cmd
cmd_out, _, _ = nmci.run(cmd)
msg += cmd_out
context.embed("text/plain", msg, "Status " + when, fail_only=fail_only)
# Always include memory stats
if context.nm_pid is not None:
msg = "Daemon memory consumption: %d KiB\n" % nm_size_kb()
if os.path.isfile("/etc/systemd/system/NetworkManager.service") \
and nmci.command_code(
"grep -q valgrind /etc/systemd/system/NetworkManager.service") == 0:
cmd_out, _, _ = nmci.run(
"LOGNAME=root HOSTNAME=localhost gdb /usr/sbin/NetworkManager "
" -ex 'target remote | vgdb' -ex 'monitor leak_check summary' -batch"
)
msg += cmd_out
context.embed("text/plain", msg, "Memory use " + when, fail_only=False)
def connect_to_vpn(context, vpn, password, secret=None, time_out=None):
cli = context.pexpect_spawn('nmcli -a connect up %s' % (vpn), timeout=180)
if not time_out:
time.sleep(1)
else:
time.sleep(int(time_out))
cli.sendline(password)
if secret is not None:
time.sleep(1)
cli.sendline(secret)
if nmci.command_code("systemctl -q is-active polkit") == 0:
r = cli.expect(['Error', pexpect.TIMEOUT, pexpect.EOF])
assert r != 0, 'Got an Error while connecting to network %s\n%s%s' % (vpn, cli.after, cli.buffer)
assert r != 1, 'nmcli vpn connect ... timed out (180s)'
else:
# Remove me when 1756441 is fixed
r = cli.expect(['Connection successfully activated', pexpect.TIMEOUT])
assert r == 0, 'Got an Error while connecting to network %s\n%s%s' % (vpn, cli.after, cli.buffer)
def after_scenario(context, scenario):
nm_pid_after = nmci.lib.nm_pid()
if not nm_pid_after:
print("Starting NM as it was found stopped")
nmci.lib.restart_NM_service()
if IS_NMTUI:
# record the network status after the test
if os.path.isfile('/tmp/tui-screen.log'):
fd = open('/tmp/tui-screen.log', 'a+')
nmci.lib.dump_status_nmtui(fd, 'after')
fd.flush()
fd.close()
if os.path.isfile('/tmp/tui-screen.log'):
context.embed("text/plain",
nmci.lib.utf_only_open_read('/tmp/tui-screen.log'),
caption="TUI")
# Stop TUI
nmci.run("sudo killall nmtui &> /dev/null")
os.remove('/tmp/nmtui.out')
# Attach journalctl logs if failed
if scenario.status == 'failed' and hasattr(context, "embed"):
logs = nmci.lib.NM_log(context.log_cursor) or "NM log is empty!"
context.embed('text/plain', logs, caption="NM")
else:
print(("NetworkManager process id after: %s (was %s)" %
(nm_pid_after, context.nm_pid)))
if scenario.status == 'failed':
nmci.lib.dump_status_nmcli(context, 'after %s' % scenario.name)
# run after_scenario tags (in reverse order)
excepts = []
tag_registry = list(nmci.tags.tag_registry)
tag_registry.reverse()
for tag in tag_registry:
if tag.tag_name in scenario.tags and tag.after_scenario is not None:
try:
tag.after_scenario(context, scenario)
except Exception:
excepts.append(traceback.format_exc())
if not IS_NMTUI:
# check for crash reports and embed them
# sets crash_embeded and crashed_step, if crash found
nmci.lib.check_coredump(context, 'no_abrt' not in scenario.tags)
nmci.lib.check_faf(context, 'no_abrt' not in scenario.tags)
if scenario.status == 'failed' or context.crashed_step:
nmci.lib.dump_status_nmcli(context,
'after cleanup %s' % scenario.name)
# Attach journalctl logs
print("Attaching NM log")
log = "~~~~~~~~~~~~~~~~~~~~~~~~~~ NM LOG ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
log += nmci.lib.NM_log(
context.log_cursor)[:20000001] or "NM log is empty!"
context.embed('text/plain', log, caption="NM")
if nm_pid_after is not None and context.nm_pid == nm_pid_after:
context.log.write(
"NetworkManager memory consumption after: %d KiB\n" %
nmci.lib.nm_size_kb())
if os.path.isfile("/etc/systemd/system/NetworkManager.service") \
and nmci.command_code(
"grep -q valgrind /etc/systemd/system/NetworkManager.service") == 0:
time.sleep(3)
nmci.run(
"LOGNAME=root HOSTNAME=localhost gdb /usr/sbin/NetworkManager "
" -ex 'target remote | vgdb' -ex 'monitor leak_check summary' -batch",
stdout=context.log,
stderr=context.log)
context.log.close()
print("Attaching MAIN log")
log = nmci.lib.utf_only_open_read("/tmp/log_%s.html" % scenario.name)
context.embed('text/plain', log, caption="MAIN")
if context.crashed_step:
print("\n\n" + ("!" * 80))
print(
"!! NM CRASHED. NEEDS INSPECTION. FAILING THE TEST !!"
)
print("!! %-74s !!" % ("CRASHING STEP: " + context.crashed_step))
print(("!" * 80) + "\n\n")
context.embed('text/plain',
context.crashed_step,
caption="CRASHED_STEP_NAME")
if not context.crash_embeded:
msg = "!!! no crash report detected, but NM PID changed !!!"
context.embed('text/plain', msg, caption="NO_COREDUMP/NO_FAF")
assert not excepts, "Exceptions in after_scenario(): \n " + "\n\n".join(
excepts)
def before_scenario(context, scenario):
# set important context attributes
context.nm_restarted = False
context.nm_pid = nmci.lib.nm_pid()
context.crashed_step = False
context.log_cursor = ""
context.arch = nmci.command_output("uname -p").strip()
context.IS_NMTUI = IS_NMTUI
context.rh_release = nmci.command_output("cat /etc/redhat-release")
if IS_NMTUI:
os.environ['TERM'] = 'dumb'
# Do the cleanup
if os.path.isfile('/tmp/tui-screen.log'):
os.remove('/tmp/tui-screen.log')
fd = open('/tmp/tui-screen.log', 'a+')
nmci.lib.dump_status_nmtui(fd, 'before')
fd.write('Screen recordings after each step:' +
'\n----------------------------------\n')
fd.flush()
fd.close()
context.log = None
else:
if not os.path.isfile('/tmp/nm_wifi_configured') \
and not os.path.isfile('/tmp/nm_dcb_inf_wol_sriov_configured'):
if nmci.command_code(
"nmcli device |grep testeth0 |grep ' connected'") != 0:
nmci.run(
"sudo nmcli connection modify testeth0 ipv4.may-fail no")
nmci.run("sudo nmcli connection up id testeth0")
for attempt in range(0, 10):
if nmci.command_code(
"nmcli device |grep testeth0 |grep ' connected'"
) == 0:
break
time.sleep(1)
os.environ['TERM'] = 'dumb'
context.log = open('/tmp/log_%s.html' % scenario.name, 'w')
# dump status before the test preparation starts
nmci.lib.dump_status_nmcli(context, 'before %s' % scenario.name)
context.start_timestamp = int(time.time())
excepts = []
if 'eth0' in scenario.tags \
or 'delete_testeth0' in scenario.tags \
or 'connect_testeth0' in scenario.tags \
or 'restart' in scenario.tags \
or 'dummy' in scenario.tags:
try:
nmci.tags.skip_restarts_bs(context, scenario)
except Exception as e:
excepts.append(str(e))
for tag in nmci.tags.tag_registry:
if tag.tag_name in scenario.tags and tag.before_scenario is not None:
try:
tag.before_scenario(context, scenario)
except Exception:
excepts.append(traceback.format_exc())
assert not excepts, "Exceptions in before_scenario():\n" + "\n\n".join(
excepts)
context.nm_pid = nmci.lib.nm_pid()
context.crashed_step = False
print(("NetworkManager process id before: %s" % context.nm_pid))
if context.nm_pid is not None and context.log is not None:
context.log.write(
"NetworkManager memory consumption before: %d KiB\n" %
nmci.lib.nm_size_kb())
if os.path.isfile("/etc/systemd/system/NetworkManager.service") \
and nmci.command_code(
"grep -q valgrind /etc/systemd/system/NetworkManager.service") == 0:
nmci.run(
"LOGNAME=root HOSTNAME=localhost gdb /usr/sbin/NetworkManager "
" -ex 'target remote | vgdb' -ex 'monitor leak_check summary' -batch",
stdout=context.log,
stderr=context.log)
context.log_cursor = nmci.lib.new_log_cursor()
def _before_scenario(context, scenario):
time_begin = time.time()
context.before_scenario_step_el = ET.Element("li", {
"class": "step passed",
"style": "margin-bottom:1rem;"
})
ET.SubElement(context.before_scenario_step_el,
"b").text = "Before scenario"
duration_el = ET.SubElement(context.before_scenario_step_el, "small",
{"class": "step_duration"})
embed_el = ET.SubElement(context.before_scenario_step_el, "div")
context.html_formatter.actual["act_step_embed_span"] = embed_el
# set important context attributes
context.nm_restarted = False
context.nm_pid = nmci.lib.nm_pid()
context.crashed_step = False
context.log_cursor = ""
context.log_cursor_before_tags = nmci.lib.new_log_cursor()
context.arch = nmci.command_output("uname -p").strip()
context.IS_NMTUI = "nmtui" in scenario.effective_tags
context.rh_release = nmci.command_output("cat /etc/redhat-release")
release_i = context.rh_release.find("release ")
if release_i >= 0:
context.rh_release_num = float(
context.rh_release[release_i:].split(" ")[1])
else:
context.rh_release_num = 0
context.hypervisor = nmci.run("systemd-detect-virt")[0].strip()
os.environ['TERM'] = 'dumb'
# dump status before the test preparation starts
nmci.lib.dump_status(context, 'Before Scenario', fail_only=True)
if context.IS_NMTUI:
nmci.run("sudo pkill nmtui")
# Do the cleanup
if os.path.isfile('/tmp/tui-screen.log'):
os.remove('/tmp/tui-screen.log')
fd = open('/tmp/tui-screen.log', 'a+')
fd.write('Screen recordings after each step:' +
'\n----------------------------------\n')
fd.flush()
fd.close()
else:
if not os.path.isfile('/tmp/nm_wifi_configured') \
and not os.path.isfile('/tmp/nm_dcb_inf_wol_sriov_configured'):
if nmci.command_code(
"nmcli device |grep testeth0 |grep ' connected'") != 0:
nmci.run(
"sudo nmcli connection modify testeth0 ipv4.may-fail no")
nmci.run("sudo nmcli connection up id testeth0")
for attempt in range(0, 10):
if nmci.command_code(
"nmcli device |grep testeth0 |grep ' connected'"
) == 0:
break
time.sleep(1)
context.start_timestamp = int(time.time())
excepts = []
if 'eth0' in scenario.tags \
or 'delete_testeth0' in scenario.tags \
or 'connect_testeth0' in scenario.tags \
or 'restart' in scenario.tags \
or 'dummy' in scenario.tags:
try:
nmci.tags.skip_restarts_bs(context, scenario)
except Exception as e:
excepts.append(str(e))
for tag_name in scenario.tags:
tag = nmci.tags.tag_registry.get(tag_name, None)
if tag is not None and tag.before_scenario is not None:
print("Executing @" + tag_name)
t_start = time.time()
t_status = "passed"
try:
tag.before_scenario(context, scenario)
except Exception:
t_status = "failed"
excepts.append(traceback.format_exc())
print(
f" @{tag_name} ... {t_status} in {time.time() - t_start:.3f}s"
)
context.nm_pid = nmci.lib.nm_pid()
context.crashed_step = False
print(("NetworkManager process id before: %s" % context.nm_pid))
context.log_cursor = nmci.lib.new_log_cursor()
nmci.lib.process_commands(context, "before_scenario")
duration = time.time() - time_begin
status = "failed" if excepts else "passed"
print(f"before_scenario ... {status} in {duration:.3f}s")
duration_el.text = f"({duration:.3f}s)"
if excepts:
context.before_scenario_step_el.set("class", "step failed")
context.embed("text/plain", "\n\n".join(excepts),
"Exception in before scenario tags")
assert False, "Exception in before scenario tags"