def init_nvp_controller(self,ip):
"""
cloudnet_obj = CloudNet(ip, set_root_ssh_access=False, auth=True,
auth_cli=True, set_ntp=False, manager_transport=None,
manager_port=None, parent=None, debug=True)
"""
cloudnet_obj = CloudNet(ip, passwd='nicira', auth=False, set_ntp=False, debug=True)
cloudnet_obj.ncli.set_root_ssh_access()
cloudnet_obj.perm_auth_expect(passwd='nicira')
# This also restarts api server
cloudnet_obj.ncli.set_service_certificate()
wait_until(cloudnet_obj.api_is_listening, minimum=15)
# Clear Controller
cloudnet_obj.ncli.clear_runtime_state(force=True)
cloudnet_obj.ncli.set_management_address(ip)
cloudnet_obj.ncli.join_cluster(ip, force=True)
cloudnet_obj.ncli.set_root_ssh_access()
cloudnet_obj.perm_auth_expect(passwd='nicira')
cloudnet_obj.connect()
def check_critical_services(dut):
"""
@summary: Use systemctl to check whether all the critical services have expected status. ActiveState of all
services must be "active". SubState of all services must be "running".
@param dut: The AnsibleHost object of DUT. For interacting with DUT.
"""
logging.info("Wait until all critical services are fully started")
assert wait_until(300, 20, critical_services_fully_started, dut), "Not all critical services are fully started"
logging.info("Check critical service status")
for service in critical_services:
status = get_service_status(dut, service)
assert status["ActiveState"] == "active", \
"ActiveState of %s is %s, expected: active" % (service, status["ActiveState"])
assert status["SubState"] == "running", \
"SubState of %s is %s, expected: active" % (service, status["SubState"])
def test_reload_configuration(localhost, ansible_adhoc, testbed):
"""
@summary: This test case is to reload the configuration and check platform status
"""
hostname = testbed['dut']
ans_host = AnsibleHost(ansible_adhoc, hostname)
ans_host.command("show platform summary")
lab_conn_graph_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), \
"../../ansible/files/lab_connection_graph.xml")
conn_graph_facts = localhost.conn_graph_facts(host=hostname, filename=lab_conn_graph_file).\
contacted['localhost']['ansible_facts']
interfaces = conn_graph_facts["device_conn"]
asic_type = ans_host.shell(
"show platform summary | awk '/ASIC: / {print$2}'")["stdout"].strip()
logging.info("Reload configuration")
ans_host.command("sudo config reload -y")
logging.info("Wait until all critical services are fully started")
check_critical_services(ans_host)
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, all_transceivers_detected, ans_host, interfaces), \
"Not all transceivers are detected in 300 seconds"
logging.info("Check interface status")
time.sleep(60)
check_interface_status(ans_host, interfaces)
logging.info("Check transceiver status")
check_transceiver_basic(ans_host, interfaces)
if asic_type in ["mellanox"]:
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "mellanox")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
from check_hw_mgmt_service import check_hw_management_service
from check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(ans_host)
logging.info("Check sysfs")
check_sysfs(ans_host)
def restart_service_and_check(localhost, dut, service):
"""
Restart specified service and check platform status
"""
dut.command("show platform summary")
lab_conn_graph_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), \
"../../ansible/files/lab_connection_graph.xml")
conn_graph_facts = localhost.conn_graph_facts(host=dut.hostname, filename=lab_conn_graph_file).\
contacted['localhost']['ansible_facts']
interfaces = conn_graph_facts["device_conn"]
asic_type = dut.shell(
"show platform summary | awk '/ASIC: / {print$2}'")["stdout"].strip()
logging.info("Restart the %s service" % service)
dut.command("sudo systemctl restart %s" % service)
logging.info("Wait until all critical services are fully started")
check_critical_services(dut)
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, all_transceivers_detected, dut, interfaces), \
"Not all transceivers are detected in 300 seconds"
logging.info("Check interface status")
time.sleep(60)
check_interface_status(dut, interfaces)
logging.info("Check transceiver status")
check_transceiver_basic(dut, interfaces)
if asic_type in ["mellanox"]:
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "mellanox")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
from check_hw_mgmt_service import check_hw_management_service
from check_sysfs import check_sysfs
logging.info("Check the hw-management service")
check_hw_management_service(dut)
logging.info("Check sysfs")
check_sysfs(dut)
def reboot(duthost, localhost, reboot_type='cold', delay=10, \
timeout=0, wait=0, wait_for_ssh=True, wait_warmboot_finalizer=False, warmboot_finalizer_timeout=0,\
reboot_helper=None, reboot_kwargs=None):
"""
reboots DUT
:param duthost: DUT host object
:param localhost: local host object
:param reboot_type: reboot type (cold, fast, warm)
:param delay: delay between ssh availability checks
:param timeout: timeout for waiting ssh port state change
:param wait: time to wait for DUT to initialize
:param wait_for_ssh: Wait for SSH startup
:param wait_warmboot_finalizer=True: Wait for WARMBOOT_FINALIZER done
:param reboot_helper: helper function to execute the power toggling
:param reboot_kwargs: arguments to pass to the reboot_helper
:return:
"""
# pool for executing tasks asynchronously
pool = ThreadPool()
dut_ip = duthost.mgmt_ip
hostname = duthost.hostname
try:
reboot_ctrl = reboot_ctrl_dict[reboot_type]
reboot_command = reboot_ctrl['command'] if reboot_type != REBOOT_TYPE_POWEROFF else None
if timeout == 0:
timeout = reboot_ctrl['timeout']
if wait == 0:
wait = reboot_ctrl['wait']
if warmboot_finalizer_timeout == 0 and 'warmboot_finalizer_timeout' in reboot_ctrl:
warmboot_finalizer_timeout = reboot_ctrl['warmboot_finalizer_timeout']
except KeyError:
raise ValueError('invalid reboot type: "{} for {}"'.format(reboot_type, hostname))
def execute_reboot_command():
logger.info('rebooting {} with command "{}"'.format(hostname, reboot_command))
return duthost.command(reboot_command)
def execute_reboot_helper():
logger.info('rebooting {} with helper "{}"'.format(hostname, reboot_helper))
return reboot_helper(reboot_kwargs)
dut_datetime = duthost.get_now_time()
DUT_ACTIVE.clear()
if reboot_type != REBOOT_TYPE_POWEROFF:
reboot_res = pool.apply_async(execute_reboot_command)
else:
assert reboot_helper is not None, "A reboot function must be provided for power off reboot"
reboot_res = pool.apply_async(execute_reboot_helper)
logger.info('waiting for ssh to drop on {}'.format(hostname))
res = localhost.wait_for(host=dut_ip,
port=SONIC_SSH_PORT,
state='absent',
search_regex=SONIC_SSH_REGEX,
delay=delay,
timeout=timeout,
module_ignore_errors=True)
if res.is_failed or ('msg' in res and 'Timeout' in res['msg']):
if reboot_res.ready():
logger.error('reboot result: {} on {}'.format(reboot_res.get(), hostname))
raise Exception('DUT {} did not shutdown'.format(hostname))
if not wait_for_ssh:
return
# TODO: add serial output during reboot for better debuggability
# This feature requires serial information to be present in
# testbed information
logger.info('waiting for ssh to startup on {}'.format(hostname))
res = localhost.wait_for(host=dut_ip,
port=SONIC_SSH_PORT,
state='started',
search_regex=SONIC_SSH_REGEX,
delay=delay,
timeout=timeout,
module_ignore_errors=True)
if res.is_failed or ('msg' in res and 'Timeout' in res['msg']):
raise Exception('DUT {} did not startup'.format(hostname))
logger.info('ssh has started up on {}'.format(hostname))
logger.info('waiting for switch {} to initialize'.format(hostname))
time.sleep(wait)
# Wait warmboot-finalizer service
if reboot_type == REBOOT_TYPE_WARM and wait_warmboot_finalizer:
logger.info('waiting for warmboot-finalizer service to finish on {}'.format(hostname))
ret = wait_until(warmboot_finalizer_timeout, 5, 0, check_warmboot_finalizer_inactive, duthost)
if not ret:
raise Exception('warmboot-finalizer service timeout on DUT {}'.format(hostname))
DUT_ACTIVE.set()
logger.info('{} reboot finished on {}'.format(reboot_type, hostname))
pool.terminate()
dut_uptime = duthost.get_up_time()
logger.info('DUT {} up since {}'.format(hostname, dut_uptime))
assert float(dut_uptime.strftime("%s")) > float(dut_datetime.strftime("%s")), "Device {} did not reboot".format(hostname)
def wait_until_fan_speed_set_to_default(dut):
wait_until(300, 10, fan_speed_set_to_default, dut)
def reboot_and_check(localhost, dut, reboot_type="cold"):
"""
Perform the specified type of reboot and check platform status.
"""
dut.command("show platform summary")
lab_conn_graph_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), \
"../../ansible/files/lab_connection_graph.xml")
conn_graph_facts = localhost.conn_graph_facts(host=dut.hostname, filename=lab_conn_graph_file).\
contacted['localhost']['ansible_facts']
interfaces = conn_graph_facts["device_conn"]
asic_type = dut.shell("show platform summary | awk '/ASIC: / {print$2}'")["stdout"].strip()
logging.info("Run %s reboot on DUT" % reboot_type)
if reboot_type == "cold":
reboot_cmd = "sudo reboot &"
reboot_timeout = 300
elif reboot_type == "fast":
reboot_cmd = "sudo fast-reboot &"
reboot_timeout = 180
elif reboot_type == "warm":
reboot_cmd = "sudo warm-reboot &"
reboot_timeout = 180
else:
assert False, "Reboot type %s is not supported" % reboot_type
dut.shell(reboot_cmd)
logging.info("Wait for DUT to go down")
localhost.wait_for(host=dut.hostname, port=22, state="stopped", delay=10, timeout=120)
logging.info("Wait for DUT to come back")
localhost.wait_for(host=dut.hostname, port=22, state="started", delay=10, timeout=reboot_timeout)
logging.info("Wait until all critical services are fully started")
check_critical_services(dut)
logging.info("Wait some time for all the transceivers to be detected")
assert wait_until(300, 20, all_transceivers_detected, dut, interfaces), \
"Not all transceivers are detected in 300 seconds"
logging.info("Check interface status")
check_interface_status(dut, interfaces)
logging.info("Check transceiver status")
check_transceiver_basic(dut, interfaces)
if asic_type in ["mellanox"]:
current_file_dir = os.path.dirname(os.path.realpath(__file__))
sub_folder_dir = os.path.join(current_file_dir, "mellanox")
if sub_folder_dir not in sys.path:
sys.path.append(sub_folder_dir)
from check_hw_mgmt_service import check_hw_management_service
from check_hw_mgmt_service import wait_until_fan_speed_set_to_default
from check_sysfs import check_sysfs
logging.info("Wait until fan speed is set to default")
wait_until_fan_speed_set_to_default(dut)
logging.info("Check the hw-management service")
check_hw_management_service(dut)
logging.info("Check sysfs")
check_sysfs(dut)
