def setup_smart_profiles(request):
trap_external_ip = request.config.getoption("trap_external_ip")
logger.info("Integration test for fields smart profiles")
profile = {
"smart_profile_field": {
"frequency": 3,
"condition": {
"type": "field",
"field": "SNMPv2-MIB.sysDescr",
"patterns": [".*zeus.*"],
},
"varBinds": [
yaml_escape_list(sq("IP-MIB"), sq("icmpOutDestUnreachs"), 0),
yaml_escape_list(sq("IP-MIB"), sq("icmpOutEchoReps"), 0),
],
}
}
update_profiles(profile)
upgrade_helm(["inventory.yaml", "profiles.yaml"])
time.sleep(60)
update_file([f"{trap_external_ip},,2c,public,,,600,,t,"], "inventory.yaml")
upgrade_helm(["inventory.yaml", "profiles.yaml"])
time.sleep(30)
yield
update_file([f"{trap_external_ip},,2c,public,,,600,,t,t"], "inventory.yaml")
upgrade_helm(["inventory.yaml"])
time.sleep(20)
def setup_profiles(request):
trap_external_ip = request.config.getoption("trap_external_ip")
profile = {
"new_profile": {"frequency": 7, "varBinds": [yaml_escape_list(sq("IP-MIB"))]},
"generic_switch": {
"frequency": 5,
"varBinds": [yaml_escape_list(sq("UDP-MIB"))],
},
}
update_profiles(profile)
upgrade_helm(["profiles.yaml"])
time.sleep(60)
update_file(
[f"{trap_external_ip},,2c,public,,,600,new_profile;generic_switch,,"],
"inventory.yaml",
)
upgrade_helm(["inventory.yaml", "profiles.yaml"])
time.sleep(30)
yield
update_file(
[f"{trap_external_ip},,2c,public,,,600,new_profile;generic_switch,,t"],
"inventory.yaml",
)
upgrade_helm(["inventory.yaml"])
time.sleep(20)
def test_modify_frequency_field(self, request, setup_splunk):
trap_external_ip = request.config.getoption("trap_external_ip")
profile = {
"test_modify": {
"frequency": 7,
"varBinds": [yaml_escape_list(sq("UDP-MIB"))],
},
}
update_profiles(profile)
update_file(
[f"{trap_external_ip},,2c,public,,,600,test_modify,f,t"], "inventory.yaml"
)
upgrade_helm(["inventory.yaml", "profiles.yaml"])
time.sleep(60)
update_file(
[f"{trap_external_ip},,2c,public,,,600,test_modify,f,"], "inventory.yaml"
)
upgrade_helm(["inventory.yaml", "profiles.yaml"])
time.sleep(30)
search_string = """| mpreview index=netmetrics earliest=-30s | search profiles=test_modify frequency=7 """
result_count, metric_count = run_retried_single_search(
setup_splunk, search_string, 8
)
assert result_count > 0
assert metric_count > 0
def test_modify_varBinds_field(self, request, setup_splunk):
trap_external_ip = request.config.getoption("trap_external_ip")
profile = {
"test_modify": {
"frequency": 7,
"varBinds": [
yaml_escape_list(sq("TCP-MIB")),
yaml_escape_list(sq("IP-MIB"), sq("icmpOutDestUnreachs"), 0),
yaml_escape_list(sq("UCD-SNMP-MIB"), sq("laIndex")),
],
},
}
update_profiles(profile)
update_file(
[f"{trap_external_ip},,2c,public,,,600,test_modify,f,t"], "inventory.yaml"
)
upgrade_helm(["inventory.yaml", "profiles.yaml"])
time.sleep(60)
update_file(
[f"{trap_external_ip},,2c,public,,,600,test_modify,f,"], "inventory.yaml"
)
upgrade_helm(["inventory.yaml", "profiles.yaml"])
time.sleep(20)
search_string = """| mpreview index=netmetrics earliest=-15s | search profiles=test_modify TCP-MIB """
result_count, metric_count = run_retried_single_search(
setup_splunk, search_string, 3
)
assert result_count > 0
assert metric_count > 0
search_string = """| mpreview index=netmetrics earliest=-15s | search profiles=test_modify | search icmpOutDestUnreachs """
result_count, metric_count = run_retried_single_search(
setup_splunk, search_string, 3
)
assert result_count > 0
assert metric_count > 0
search_string = """| mpreview index=netmetrics earliest=-20s | search laIndex | dedup metric_name:sc4snmp.UCD-SNMP-MIB.laIndex """
result_count, metric_count = run_retried_single_search(
setup_splunk, search_string, 3
)
assert result_count == 3
assert metric_count == 3
def format_entry(e: any, no_quote_fields: list,
double_quote_fields: list) -> any:
# max 2 layers
if isinstance(e, str):
return sq(e)
res = ordereddict()
for k1, v1 in e.items():
if isinstance(v1, ordereddict):
res[k1] = v1
elif isinstance(v1, dict):
# break up v1
temp1 = ordereddict()
for k2, v2 in v1.items():
if k2 in no_quote_fields:
temp1[sq(k2)] = v2
elif k2 in double_quote_fields:
temp1[sq(k2)] = dq(v2)
else:
temp1[sq(k2)] = sq(v2)
temp1.fa.set_flow_style() # one line
if isinstance(v1, ordereddict):
temp1.ca.comment = v1.ca.comment
res[k1] = temp1
elif v1:
if k1 in no_quote_fields:
res[k1] = v1
elif k1 in double_quote_fields:
res[k1] = dq(v1)
else:
res[k1] = sq(v1)
# print(e)
# res.ca.comment = e.ca.comment # preserve comments
return res
def setup_profile(request):
trap_external_ip = request.config.getoption("trap_external_ip")
profile = {
"generic_switch": {
"frequency": 5,
"varBinds": [
yaml_escape_list(sq("TCP-MIB")),
yaml_escape_list(sq("IF-MIB"), sq("ifType"), 1),
],
}
}
update_profiles(profile)
upgrade_helm(["profiles.yaml"])
time.sleep(60)
update_file(
[f"{trap_external_ip},,2c,public,,,600,generic_switch,,"], "inventory.yaml"
)
upgrade_helm(["inventory.yaml", "profiles.yaml"])
time.sleep(30)
yield
upgrade_helm([f"{trap_external_ip},,2c,public,,,600,generic_switch,,t"])
time.sleep(20)
def setup_small_walk(request):
trap_external_ip = request.config.getoption("trap_external_ip")
profile = {
"walk1": {
"condition": {"type": "walk"},
"varBinds": [yaml_escape_list(sq("IP-MIB"))],
},
}
update_profiles(profile)
upgrade_helm(["inventory.yaml", "profiles.yaml"])
time.sleep(60)
update_file([f"{trap_external_ip},,2c,public,,,20,walk1,f,"], "inventory.yaml")
upgrade_helm(["inventory.yaml", "profiles.yaml"])
time.sleep(30)
yield
update_file([f"{trap_external_ip},,2c,public,,,20,walk1,f,t"], "inventory.yaml")
upgrade_helm(["inventory.yaml"])
time.sleep(20)
def test_disable_one_profile_and_reload(self, request, setup_splunk):
trap_external_ip = request.config.getoption("trap_external_ip")
logger.info("Integration test for deleting one profile and reloading")
profile = {
"new_profile": {
"frequency": 7,
"varBinds": [yaml_escape_list(sq("IP-MIB"))],
}
}
update_profiles(profile)
update_file(
[f"{trap_external_ip},,2c,public,,,600,new_profile,,"], "inventory.yaml"
)
upgrade_helm(["inventory.yaml", "profiles.yaml"])
time.sleep(70)
search_string = """| mpreview index=netmetrics| spath profiles | search profiles=generic_switch earliest=-20s """
result_count, metric_count = run_retried_single_search(
setup_splunk, search_string, 3
)
assert result_count == 0
assert metric_count == 0
def setup_modify_profile(request):
trap_external_ip = request.config.getoption("trap_external_ip")
profile = {
"test_modify": {
"frequency": 5,
"varBinds": [yaml_escape_list(sq("UDP-MIB"))],
},
}
update_profiles(profile)
upgrade_helm(["inventory.yaml", "profiles.yaml"])
time.sleep(60)
update_file(
[f"{trap_external_ip},,2c,public,,,600,test_modify,f,"], "inventory.yaml"
)
upgrade_helm(["inventory.yaml", "profiles.yaml"])
time.sleep(30)
yield
update_file(
[f"{trap_external_ip},,2c,public,,,600,test_modify,f,t"], "inventory.yaml"
)
upgrade_helm(["inventory.yaml"])
time.sleep(20)
def define_exp_parameters(experiment, exp_doc, node_list):
deployment_list = []
# Node cpu hog parameters
if ('cpu' in experiment) and ('node' in experiment):
for node in node_list:
for duration in range(50, 81, 1):
for cpu_cores in range(2, 5):
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][0]['value'] = str(duration)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][1]['value'] = str(cpu_cores)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][2]['value'] = sq(node)
deployment_list.append(copy.deepcopy(exp_doc))
# Pod cpu hog parameters
elif ('cpu' in experiment) and ('pod' in experiment):
for duration in range(50, 81, 1):
for cpu_cores in range(2, 5):
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][0]['value'] = str(cpu_cores)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][1]['value'] = str(duration)
deployment_list.append(copy.deepcopy(exp_doc))
# Node memory hog parameters
elif ('mem' in experiment) and ('node' in experiment):
for node in node_list:
for duration in range(50, 81, 1):
for ram_perc in range(20, 41):
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][0]['value'] = str(duration)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][1]['value'] = str(ram_perc)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][2]['value'] = sq(node)
deployment_list.append(copy.deepcopy(exp_doc))
# Pod memory hog parameters
elif ('mem' in experiment) and ('pod' in experiment):
for duration in range(50, 81, 1):
for ram in range(1500, 3000, 50):
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][0]['value'] = str(ram)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][1]['value'] = str(duration)
deployment_list.append(copy.deepcopy(exp_doc))
# Node io stress parameters
elif ('iostress' in experiment) and ('node' in experiment):
for node in node_list:
for duration in range(50, 81, 1):
for fsu_perc in range(20, 41):
for nr_workers in range(3, 6):
for nr_cpu in [1, 2]:
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][0]['value'] = str(duration)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][1]['value'] = str(fsu_perc)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][2]['value'] = str(nr_cpu)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][3]['value'] = str(nr_workers)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][4]['value'] = sq(node)
deployment_list.append(copy.deepcopy(exp_doc))
# Pod io stress parameters
elif ('iostress' in experiment) and ('pod' in experiment):
for duration in range(50, 81, 1):
for fsu_perc in range(20, 41):
for nr_workers in range(3, 6):
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][0]['value'] = str(duration)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][1]['value'] = str(fsu_perc)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][2]['value'] = str(nr_workers)
deployment_list.append(copy.deepcopy(exp_doc))
# Pod delete parameters
elif ('del' in experiment):
for duration in range(12, 20, 1):
for interval in range(3, 7):
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][0]['value'] = str(interval)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][1]['value'] = str(duration)
deployment_list.append(copy.deepcopy(exp_doc))
# Network latency parameters
elif ('netlat' in experiment):
for duration in range(50, 81, 1):
for latency in range(1600, 2800, 50):
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][0]['value'] = sq(exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][0]['value'])
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][1]['value'] = str(latency)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][2]['value'] = str(duration)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][3]['value'] = sq(exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][3]['value'])
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][4]['value'] = sq(exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][4]['value'])
deployment_list.append(copy.deepcopy(exp_doc))
# Network loss parameters
elif 'netloss' in experiment:
for duration in range(50, 81, 1):
for packet_loss_perc in range(80, 101):
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][0]['value'] = sq(exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][0]['value'])
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][1]['value'] = str(packet_loss_perc)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][2]['value'] = str(duration)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][3]['value'] = sq(exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][3]['value'])
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][4]['value'] = sq(exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][4]['value'])
deployment_list.append(copy.deepcopy(exp_doc))
# Network corruption parameters
elif 'netcorr' in experiment:
for duration in range(50, 81, 1):
for packet_corr_perc in range(80, 101):
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][0]['value'] = sq(exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][0]['value'])
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][1]['value'] = str(duration)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][2]['value'] = str(packet_corr_perc)
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][3]['value'] = sq(exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][3]['value'])
exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][4]['value'] = sq(exp_doc[0]['spec']['experiments'][0]['spec']['components']['env'][4]['value'])
deployment_list.append(copy.deepcopy(exp_doc))
return deployment_list
leases = IscDhcpLeases('/var/lib/dhcp/dhcpd.leases')
current_leases = leases.get_current()
lease_ips = []
for lease in current_leases.values():
m = re.search(r'([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)', str(lease))
if m:
lease_ips.append(m.group(1))
# Generate initial inventory file ##############################################
os.chdir('/home/nornir-ztp/inventory')
hosts = {}
for index, lease_ip in enumerate(lease_ips):
router_name = 'R' + str(index + 1)
hosts[router_name] = {'hostname': lease_ip}
# Add single quoted (sq) value in YAML dumper
device_type = {'device_type': sq('cisco_ios_telnet')}
conn_opts = {'connection_options': {'netmiko': {'extras': device_type}}}
hosts[router_name].update(conn_opts)
with open('hosts.yaml', 'w') as f:
data = yaml.dump(hosts, f)
# Reload inventory
nr = InitNornir(config_file="/home/nornir-ztp/config.yaml")
# Configure SSH keys on network elements #######################################
def gen_ssh_keys(task):
# Generate config commands
cfg = [
task.run(template_file, path="templates",