def test_ft_snmp_warmstart_trap():
"""
Author : Prasad Darnasi<*****@*****.**>
Verify that trap is sent when reboot is performed.
"""
check_flag = snmptrapd_checking()
if not check_flag:
st.report_fail("snmptrapd_not_running")
# trigger trap on DUT
reboot.config_save(vars.D1)
st.reboot(vars.D1, 'warm')
# Get the ip address of the switch after reboot
device_eth0_ip_addr()
# get data from capture
read_cmd = "cat {}".format(capture_file)
output = execute_command(ssh_conn_obj, read_cmd)
trap_lines = output.split("\n")[:-1]
result = any('warmStart' in x for x in trap_lines)
if result == 0:
for i in range(1, 4):
read_cmd = "cat {}".format(capture_file)
output = execute_command(ssh_conn_obj, read_cmd)
trap_lines = output.split("\n")[:-1]
result = any('warmStart' in x for x in trap_lines)
if result == 1:
break
st.wait(10)
if result == 0:
st.report_fail("snmp_output_failed", "warmStart")
else:
st.report_pass("test_case_passed")
def config_chef(chef_conn_obj, **kwargs):
if not chef_conn_obj or not kwargs.get('action'):
return False
cmd = "sudo -i"
con_obj.execute_command(chef_conn_obj, cmd)
action = kwargs['action']
if action == 'copy_files':
cmd = 'mv {} {}'.format(kwargs['src_file'], kwargs['dst_file'])
con_obj.execute_command(chef_conn_obj, cmd)
def upload_chef_cookbook(chef_conn_obj, path, file_name="sonic.json"):
"""
:param chef_conn_obj:
:param file_name:
:return:
"""
if not chef_conn_obj:
return False
command = "cd {}; knife upload cookbooks/sonic".format(path)
st.log("Remote Command: {}".format(command))
con_obj.execute_command(chef_conn_obj, command)
def write_ansible_host(connection_obj, host_string, inventory_path):
"""
Add hosts to /etc/ansible/hosts hostgroup section
:param connection_obj:
:param host_string:
:param inventory_path:
:return:
"""
if not connection_obj:
return False
command = "echo {} >> {}".format(host_string, inventory_path)
con_obj.execute_command(connection_obj, command)
def copy_ansible_host_bkp(connection_obj, inventory_path, new_inventory_path):
"""
:param connection_obj:
:param inventory_path:
:param new_inventory_path:
:return:
"""
if not connection_obj:
return False
command = "cp {} {}".format(inventory_path, new_inventory_path)
con_obj.execute_command(connection_obj, command)
return True
def copy_ssh_id(connection_obj, device_ip, username, password, key_path):
"""
:param connection_obj:
:param device_ip:
:param username:
:param password:
:param key_path:
:return:
"""
if not connection_obj:
return False
command = "sshpass -p '{2}' ssh-copy-id -i {0} {1}@{3}".format(
key_path, username, password, device_ip)
con_obj.execute_command(connection_obj, command)
def delete_chef_node(chef_conn_obj, node_name="testsonic"):
"""
:param chef_conn_obj:
:param file_name:
:return:
"""
if not chef_conn_obj:
return False
command = "sudo knife node delete {} -y".format(node_name)
st.log("Remote Command: {}".format(command))
con_obj.execute_command(chef_conn_obj, command)
command = "sudo knife client delete {} -y".format(node_name)
st.log("Remote Command: {}".format(command))
con_obj.execute_command(chef_conn_obj, command)
def create_chef_cookbook(chef_conn_obj, cookbook_path, file_name="sonic.json"):
"""
:param chef_conn_obj:
:param file_name:
:return:
"""
if not chef_conn_obj:
return False
command = "sudo chef generate cookbook {}/{}".format(
cookbook_path, file_name)
st.log("Remote Command: {}".format(command))
con_obj.execute_command(chef_conn_obj, command)
command = "chmod 777 {}/{}".format(cookbook_path, file_name)
st.log("Remote Command: {}".format(command))
con_obj.execute_command(chef_conn_obj, command)
def upload_role_chef_server(chef_conn_obj, role_dir, file_name="sonic.json"):
"""
:param chef_conn_obj:
:param file_name:
:return:
"""
if not chef_conn_obj:
return False
command = "cd {}".format(role_dir)
con_obj.execute_command(chef_conn_obj, command)
command = "sudo knife role from file {}".format(file_name)
st.log("Remote Command: {}".format(command))
out = con_obj.execute_command(chef_conn_obj, command)
if not out:
return False
console_log = utils_obj.remove_last_line_from_string(out)
st.log(console_log)
if console_log.find("ERROR") > 1:
return False
return True
def clear_syslog_from_remote_server(dut):
"""
Clear the logs from the syslog server
Author: Chaitanya Lohith Bollapragada ([email protected])
:param dut:
:return:
"""
syslog_ip = utils.ensure_service_params(dut, "syslog", "ip")
syslog_port = utils.ensure_service_params(dut, "syslog", "port")
syslog_username = utils.ensure_service_params(dut, "syslog", "username")
syslog_password = utils.ensure_service_params(dut, "syslog", "password")
syslog_path = utils.ensure_service_params(dut, "syslog", "path")
command = "sudo truncate -s 0 {}".format(syslog_path)
syslog_con_obj = conf_obj.connect_to_device(syslog_ip,
syslog_username,
syslog_password,
port=syslog_port)
conf_obj.execute_command(syslog_con_obj, command)
return True
def bootstrap_chef_node(chef_conn_obj, repo_path, device_ip, uname, password,
node_name):
"""
:param chef_conn_obj:
:param file_name:
:return:
"""
if not chef_conn_obj:
return False
ssh_obj = con_obj.connect_to_device(device_ip, uname, password)
if not ssh_obj:
password = '******'
command = "cd {}".format(repo_path)
st.log("Remote Command: {}".format(command))
con_obj.execute_command(chef_conn_obj, command)
command = "sudo knife bootstrap {} -x {} -P {} --sudo --ssh-verify-host-key never --node-ssl-verify-mode none -N {}"\
.format(device_ip, uname, password ,node_name)
st.log("Remote Command: {}".format(command))
return con_obj.execute_command(chef_conn_obj, command)
def snmptrapd_checking():
retval = False
# check and start the snmptrap on the given server.
ps_cmd = "ps -ealf | grep snmptrapd | grep -v grep"
st.log("Checking for snmptrap process existence with command '{}'".format(
ps_cmd))
output = execute_command(ssh_conn_obj, ps_cmd)
ps_lines = "\n".join(output.split("\n")[:-1])
if "snmptrapd" in ps_lines:
retval = True
return retval
def ansible_ping_host(connection_obj, host_name, debug=False):
"""
:param connection_obj:
:param host_name:
:param debug: True or False
:return:
"""
if not connection_obj:
return False
verbose = ""
if debug:
verbose = "-vvvv"
command = "ansible -m ping {} {}".format(host_name, verbose)
return con_obj.execute_command(connection_obj, command)
def verify_chef_cookbook_on_server(chef_conn_obj,
roles,
cookbook="sonic.json"):
"""
:param chef_conn_obj:
:param roles:
:param cookbook:
:return:
"""
command = "ls -la {}/{}".format(roles, cookbook)
st.log("Remote Command: {}".format(command))
files_list = con_obj.execute_command(chef_conn_obj, command)
st.log(str(files_list))
if files_list.find(cookbook) < 1:
return False
return True
def create_chef_node(chef_conn_obj, node_name):
"""
:param chef_conn_obj:
:param node_name:
:return:
"""
if not chef_conn_obj:
return False
command = "knife node create {}".format(node_name)
st.log("Remote Command: {}".format(command))
console_log = utils_obj.remove_last_line_from_string(
con_obj.execute_command(chef_conn_obj, command))
st.log(console_log)
if console_log.find("ERROR") > 1:
st.log(console_log)
return False
return True
def generate_certs(chef_conn_obj, repo_path):
if not chef_conn_obj:
return False
command = "sudo -i"
con_obj.execute_command(chef_conn_obj, command)
command = "cd {}".format(repo_path)
st.log("Remote Command: {}".format(command))
con_obj.execute_command(chef_conn_obj, command)
command = "./gencert.sh"
st.log("Remote Command: {}".format(command))
con_obj.execute_command(chef_conn_obj, command)
st.wait(5)
def update_node_run_list(chef_conn_obj, client_ip, recipe_role):
"""
:param chef_conn_obj:
:param client_ip:
:param recipe_role:
:return:
"""
if not chef_conn_obj:
return False
client_ip = client_ip.replace(".", "-")
command = "sudo knife node run_list add {} '{}'".format(
client_ip, recipe_role)
st.log("Remote Command: {}".format(command))
console_log = utils_obj.remove_last_line_from_string(
con_obj.execute_command(chef_conn_obj, command))
st.log(console_log)
if console_log.find("ERROR") > 1:
return False
return True
def operations_on_runlist(chef_conn_obj, run_list, action="remove"):
"""
:param chef_conn_obj:
:param run_list:
:param action:
:return:
"""
if not chef_conn_obj:
return False
command = "sudo knife node run_list {} {}".format(action, run_list)
st.log("Remote Command: {}".format(command))
out = con_obj.execute_command(chef_conn_obj, command)
if not out:
return False
console_log = utils_obj.remove_last_line_from_string(out)
st.log(console_log)
if console_log.find("ERROR") > 1:
return False
return True
def generate_certificate(chef_conn_obj, cert_path):
"""
:param chef_conn_obj:
:param cert_path:
:return:
"""
if not chef_conn_obj:
return False
# command = "cd {}".format(cert_path)
# con_obj.execute_command(chef_conn_obj, command)
# print(command)
command = "sudo knife configure client {}".format(cert_path)
st.log("Remote Command: {}".format(command))
result = con_obj.execute_command(chef_conn_obj, command)
st.log("Result: {}".format(result))
if result.find("Writing validation.pem") < 1:
return False
return True
def show_node_run_list(chef_conn_obj, node_name="sonic"):
"""
:param chef_conn_obj:
:param node_name:
:return:
"""
if not chef_conn_obj:
return False
command = "knife node show {} run_list".format(node_name)
st.log("Remote Command: {}".format(command))
out = con_obj.execute_command(chef_conn_obj, command)
if not out:
return False
console_log = utils_obj.remove_last_line_from_string(out)
st.log(console_log)
if console_log.find("ERROR") > 1:
st.log(console_log)
return False
return True
def updating_node_run_list_operations(ssh_conn_obj, hostname, chef_params,
role_file):
st.log("Changing directory to {}".format(chef_params.path))
st.log(
con_obj.execute_command(ssh_conn_obj,
"cd {}".format(chef_params.path)))
st.log("Uploading the role to chef server...")
update_role = chef_obj.upload_role_chef_server(
ssh_conn_obj, "{}/{}".format(chef_params.roles, role_file))
if not update_role:
st.log("error_update_role_chef_server")
st.report_fail("error_update_role_chef_server")
st.log("Updating node run list ...")
update_node_list = chef_obj.update_node_run_list(
ssh_conn_obj, hostname,
"recipe[sonic::vlan],role[sonic],recipe[sonic::fdb],recipe[sonic::lag],recipe[sonic::interface]"
)
if not update_node_list:
st.log("error_update_node_run_list")
st.report_fail("error_update_node_run_list")
def ansible_cmd(connection_obj,
host_name,
mod_string,
cmd_string,
option=None):
"""
:param connection_obj:
:param host_name:
:param mod_string:
:param cmd_string:
:param option:
:return:
"""
if not connection_obj:
return False
command = "ansible -m {1} -a \"{2}\" {0}".format(host_name, mod_string,
cmd_string)
if option:
command = "ansible -m {1} -a \"{2}\" {0} {3}".format(
host_name, mod_string, cmd_string, option)
return con_obj.execute_command(connection_obj, command)
def test_ft_snmp_nsnotifyshutdown_trap():
"""
Author : Prasad Darnasi<*****@*****.**>
Verify that trap is sent when snmp docker is restarted.
"""
check_flag = snmptrapd_checking()
if not check_flag:
st.report_fail("snmptrapd_not_running")
# trigger trap on DUT
basic_obj.docker_operation(vars.D1, "snmp", "restart")
# get data from capture
read_cmd = "cat {}".format(capture_file)
output = execute_command(ssh_conn_obj, read_cmd)
trap_lines = output.split("\n")[:-1]
result = any(data.nsNotifyShutdown in x for x in trap_lines)
if result == 0:
st.report_fail("snmp_output_failed", "nsNotifyShutdown")
else:
st.report_pass("test_case_passed")
def get_syslog_from_remote_server(dut,
severity=None,
filter_list=None,
lines=None):
"""
Get the logs from the syslog server
Author: Chaitanya Lohith Bollapragada ([email protected])
:param dut:
:param severity:
:param filter_list:
:param lines:
:return:
"""
syslog_ip = utils.ensure_service_params(dut, "syslog", "ip")
syslog_port = utils.ensure_service_params(dut, "syslog", "port")
syslog_username = utils.ensure_service_params(dut, "syslog", "username")
syslog_password = utils.ensure_service_params(dut, "syslog", "password")
syslog_path = utils.ensure_service_params(dut, "syslog", "path")
if filter_list is None:
filter_list = []
filter_list = list(filter_list) if isinstance(filter_list,
list) else [filter_list]
command = "cat {}".format(syslog_path)
if severity:
command += " | grep '{}'".format(severity)
for each_filter in filter_list:
command += " | grep '{}'".format(each_filter)
if lines:
command += "| tail -n {} ".format(lines)
syslog_con_obj = conf_obj.connect_to_device(syslog_ip,
syslog_username,
syslog_password,
port=syslog_port)
syslog_file_contents = conf_obj.execute_command(syslog_con_obj, command)
return syslog_file_contents
def ansible_play_book(connection_obj,
inventory_path,
playbook_name,
cmd_string=None):
"""
:param connection_obj:
:param inventory_path:
:param playbook_name:
:param cmd_string:
:return:
"""
if not connection_obj:
return False
command = "sudo ansible-playbook {}/{}".format(inventory_path,
playbook_name)
if cmd_string:
cmd_string_li = list(cmd_string) if isinstance(cmd_string,
list) else [cmd_string]
command = "sudo ansible-playbook {}/{}".format(inventory_path,
playbook_name)
for each_cmd_string in cmd_string_li:
command += " -e {} ".format(each_cmd_string)
return con_obj.execute_command(connection_obj, command)
def update_node_run_list(chef_conn_obj, node, recipe_role, action='add'):
"""
:param chef_conn_obj:
:param client_ip:
:param recipe_role:
:return:
"""
if not chef_conn_obj:
return False
# client_ip = client_ip.replace(".", "-")
command = "knife node run_list {} {} '{}'".format(action, node,
recipe_role)
st.log("Remote Command: {}".format(command))
out = con_obj.execute_command(chef_conn_obj, command)
if not out:
return False
console_log = utils_obj.remove_last_line_from_string(out)
st.log(console_log)
if console_log.find("ERROR") > 1:
return False
return True
def chef_server_operations(ssh_conn_obj, dut, chef_params, role_file,
file_path, destination_path):
"""
Common function to do chef server side operations
Author: Chaitanya Vella ([email protected])
:param ssh_conn_obj:
:param dut:
:param chef_params:
:param role_file:
:param file_path:
:param destination_path:
:return:
"""
basic_obj.copy_file_from_client_to_server(ssh_conn_obj,
src_path=file_path,
dst_path=destination_path)
st.log("Generating {} file on the chef server ".format(
chef_params.validation_file))
st.log("Changing directory to {}".format(chef_params.path))
st.log(
con_obj.execute_command(ssh_conn_obj,
"cd {}".format(chef_params.path)))
st.log("Uploading the role to chef server...")
update_role = chef_obj.upload_role_chef_server(
ssh_conn_obj, "{}/{}".format(chef_params.roles, role_file))
if not update_role:
st.log("error_update_role_chef_server")
st.report_fail("error_update_role_chef_server")
st.log("Updating node run list ...")
update_node_list = chef_obj.update_node_run_list(
ssh_conn_obj, node_name,
"recipe[sonic::vlan],role[{}]".format(hostname))
if not update_node_list:
st.log("error_update_node_run_list")
st.report_fail("error_update_node_run_list")
return True
def test_ft_ssh_config_reload_docker():
"""
Author : Prudvi Mangadu ([email protected])
"""
result = True
get_docker_ps(vars.D1)
count = get_and_match_docker_count(vars.D1)
ssh_d1 = connect_to_device(st.get_mgmt_ip(vars.D1), ssh_data.usr_default,
ssh_data.pwd_final)
if ssh_d1:
st.log(
"Executing command - 'sudo config reload -y &' in to the SSH session."
)
st.log(execute_command(ssh_d1, 'sudo config reload -y &'))
st.wait(5, 'After executing "config reload" cmd on SSH session.')
st.log("Forcefully disconnecting the SSH session..")
ssh_disconnect(ssh_d1)
else:
st.error('Cannot SSH into Device with default credentials')
st.report_fail("ssh_failed")
if not poll_wait(verify_docker_status, 180, vars.D1, 'Exited'):
st.error(
"Post 'config reload' from SSH, dockers are not auto recovered.")
result = False
if result:
if not poll_wait(get_and_match_docker_count, 180, vars.D1, count):
st.error("Post 'config reload' from SSH, ALL dockers are not UP.")
result = False
if not result:
st.log("Test Failed: So recovering the device by reboot.")
st.reboot(vars.D1)
st.report_fail("test_case_failed")
st.report_pass("test_case_passed")
def test_ft_snmp_link_up_trap():
"""
Author : Prasad Darnasi<*****@*****.**>
Verify that trap is sent when a link is UP.
"""
check_flag = snmptrapd_checking()
if not check_flag:
st.report_fail("snmptrapd_not_running")
# trigger trap on DUT
intf_obj.interface_shutdown(vars.D1, vars.D1T1P1)
intf_obj.interface_noshutdown(vars.D1, vars.D1T1P1)
# get data from capture
read_cmd = "cat {}".format(capture_file)
output = execute_command(ssh_conn_obj, read_cmd)
trap_lines = output.split("\n")[:-1]
result = any('linkUp' in x for x in trap_lines)
if result == 0:
st.report_fail("snmp_output_failed", "linkUp")
else:
st.report_pass("test_case_passed")
def bootstrap_chef_node(chef_conn_obj, device_ip, uname, password, node_name):
"""
:param chef_conn_obj:
:param file_name:
:return:
"""
if not chef_conn_obj:
return False
command = "cd chef-repo"
st.log("Remote Command: {}".format(command))
con_obj.execute_command(chef_conn_obj, command)
command = "./gencert.sh"
st.log("Remote Command: {}".format(command))
con_obj.execute_command(chef_conn_obj, command)
st.wait(5)
command = "knife bootstrap {} -x {} -P {} --sudo --ssh-verify-host-key never --node-ssl-verify-mode none -N {}"\
.format(device_ip, uname, password ,node_name)
st.log("Remote Command: {}".format(command))
con_obj.execute_command(chef_conn_obj, command)
