def stop_master_router(self, vpc):
self.logger.debug("Stopping Master Router of VPC '%s'...", vpc.name)
routers = list_routers(self.api_client, domainid=self.domain.id, account=self.account.name, vpcid=vpc.id)
for router in routers:
if router.redundantstate == 'MASTER':
cmd = stopRouter.stopRouterCmd()
cmd.id = router.id
# This will not fail-over gracefully and cause a ~3.6sec downtime
# cmd.forced = 'true'
self.api_client.stopRouter(cmd)
break
for router in routers:
if router.state == 'Running':
hosts = list_hosts(self.api_client, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid)
self.assertTrue(isinstance(hosts, list))
host = next(iter(hosts or []), None)
try:
host.user, host.passwd = get_host_credentials(self.config, host.name)
get_process_status(host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "sh /opt/cosmic/router/scripts/checkrouter.sh ")
except KeyError as e:
raise Exception("Exception: %s" % e)
self.logger.debug("Master Router of VPC '%s' stopped", vpc.name)
def test_01_single_VPC_iptables_policies(self):
""" Test iptables default INPUT/FORWARD policies on VPC router """
self.logger.debug("Starting test_01_single_VPC_iptables_policies")
routers = self.entity_manager.query_routers()
self.assertEqual(isinstance(routers, list), True, "Check for list routers response return valid data")
self.entity_manager.create_network(self.services["vpc_network_offering"], self.vpc.id, "10.1.1.1")
self.entity_manager.create_network(self.services["vpc_network_offering_no_lb"], self.vpc.id, "10.1.2.1")
self.entity_manager.add_nat_rules(self.vpc.id)
self.entity_manager.do_vpc_test()
for router in routers:
if not router.isredundantrouter and router.vpcid:
hosts = list_hosts(self.apiclient, id=router.hostid)
self.assertEqual(isinstance(hosts, list), True, "Check for list hosts response return valid data")
host = hosts[0]
host.user = self.hostConfig["username"]
host.passwd = self.hostConfig["password"]
host.port = self.services["configurableData"]["host"]["port"]
tables = [self.services["configurableData"]["input"], self.services["configurableData"]["forward"]]
for table in tables:
result = None
if self.hypervisor.lower() in ("vmware", "hyperv"):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"iptables -L %s" % table,
hypervisor=self.hypervisor,
)
else:
try:
result = get_process_status(
host.ipaddress,
host.port,
host.user,
host.passwd,
router.linklocalip,
"iptables -L %s" % table,
)
except KeyError:
self.skipTest(
"Provide a marvin config file with host\
credentials to run %s"
% self._testMethodName
)
self.logger.debug("iptables -L %s: %s" % (table, result))
res = str(result)
self.assertEqual(res.count("DROP"), 1, "%s Default Policy should be DROP" % table)
def check_routers_state(self,count=2, status_to_check="MASTER", expected_count=1, showall=False):
vals = ["MASTER", "BACKUP", "UNKNOWN"]
cnts = [0, 0, 0]
self.wait_for_vrrp()
result = "UNKNOWN"
self.query_routers(count, showall)
for router in self.routers:
if router.state == "Running":
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list host returns a valid list"
)
host = hosts[0]
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = str(get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"sh /opt/cloud/bin/checkrouter.sh ",
hypervisor=self.hypervisor
))
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = str(get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"sh /opt/cloud/bin/checkrouter.sh "
))
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials to\
check router services")
if result.count(status_to_check) == 1:
cnts[vals.index(status_to_check)] += 1
if cnts[vals.index(status_to_check)] != expected_count:
self.fail("Expected '%s' routers at state '%s', but found '%s'!" % (expected_count, status_to_check, cnts[vals.index(status_to_check)]))
def download_builtin_templates(apiclient, zoneid, hypervisor, host,
linklocalip, interval=60):
"""After setup wait till builtin templates are downloaded"""
# Change IPTABLES Rules
get_process_status(
host["ipaddress"],
host["port"],
host["username"],
host["password"],
linklocalip,
"iptables -P INPUT ACCEPT"
)
time.sleep(interval)
# Find the BUILTIN Templates for given Zone, Hypervisor
list_template_response = list_templates(
apiclient,
hypervisor=hypervisor,
zoneid=zoneid,
templatefilter='self'
)
if not isinstance(list_template_response, list):
raise Exception("Failed to download BUILTIN templates")
# Ensure all BUILTIN templates are downloaded
templateid = None
for template in list_template_response:
if template.templatetype == "BUILTIN":
templateid = template.id
# Sleep to ensure that template is in downloading state after adding
# Sec storage
time.sleep(interval)
while True:
template_response = list_templates(
apiclient,
id=templateid,
zoneid=zoneid,
templatefilter='self'
)
template = template_response[0]
# If template is ready,
# template.status = Download Complete
# Downloading - x% Downloaded
# Error - Any other string
if template.status == 'Download Complete':
break
elif 'Downloaded' in template.status:
time.sleep(interval)
elif 'Installing' not in template.status:
raise Exception("ErrorInDownload")
return
def wait_vm_ready(self, router, vmip):
self.logger.debug("Check whether VM %s is up" % vmip)
max_tries = 15
test_tries = 0
ping_result = 0
host = self.get_host_details(router)
while test_tries < max_tries:
try:
ping_result = get_process_status(
host.ipaddress,
host.port,
host.user,
host.passwd,
router.linklocalip,
"ping -c 1 " + vmip + ">/dev/null; echo $?"
)
# Return value 0 means we were able to ping
if int(ping_result[0]) == 0:
self.logger.debug("VM %s is pingable, give it 10s to request the password" % vmip)
time.sleep(10)
return True
except KeyError:
self.skipTest("Provide a marvin config file with host credentials to run %s" % self._testMethodName)
self.logger.debug("Ping result from the Router on IP '%s' is -> '%s'" % (router.linklocalip, ping_result[0]))
test_tries += 1
self.logger.debug("Executing vm ping %s/%s" % (test_tries, max_tries))
time.sleep(5)
return False
def test_dnsmasq_service_dhcp_dns_config(self, router, dns_server):
host = self.get_host_details(router)
nic = 'eth3'
# Get the dnsmasq config entry
command_to_execute = "grep '%s,6' /etc/dnsmasq.d/%s.conf" % (nic, nic)
dnsmasq_dns_config_result = ""
try:
dnsmasq_dns_config_result = get_process_status(
host.ipaddress,
host.port,
host.user,
host.passwd,
router.linklocalip,
command_to_execute)
except KeyError:
self.skipTest(
"Provide a marvin config file with host\
credentials to run %s" %
self._testMethodName)
command_result = str(dnsmasq_dns_config_result)
self.logger.debug("Got this response: %s " % command_result)
# Check to see if our VM is in the password file
if command_result.count(dns_server) == 0:
return False
return True
def test_dhcphopts(self, ipaddress, router):
hosts = list_hosts(
self.apiclient,
id=router.hostid)
self.assertEqual(
isinstance(hosts, list),
True,
"Check for list hosts response return valid data")
host = hosts[0]
host.user = self.hostConfig['username']
host.passwd = self.hostConfig['password']
host.port = self.services["configurableData"]["host"]["port"]
host_tag = ipaddress.replace(".","_")
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"cat /etc/dhcpopts.txt | grep %s" % (host_tag),
hypervisor=self.hypervisor)
else:
try:
result = get_process_status(
host.ipaddress,
host.port,
host.user,
host.passwd,
router.linklocalip,
"cat /etc/dhcpopts.txt | grep %s" % (host_tag))
except KeyError:
self.skipTest(
"Provide a marvin config file with host\
credentials to run %s" %
self._testMethodName)
self.logger.debug("cat /etc/dhcpopts.txt | grep %s RESULT IS ==> %s" % (host_tag, result))
res = str(result)
self.assertNotEqual(
res.count(host_tag),
0,
"DHCP opts file does not contain exception for IP ==> %s!" % res)
def test_password_file_not_empty(self, vm, router):
hosts = list_hosts(
self.apiclient,
id=router.hostid)
self.assertEqual(
isinstance(hosts, list),
True,
"Check for list hosts response return valid data")
host = hosts[0]
host.user = self.hostConfig['username']
host.passwd = self.hostConfig['password']
host.port = self.services["configurableData"]["host"]["port"]
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"cat /var/cache/cloud/passwords-%s | grep %s | sed 's/=/ /g' | awk '{print $1}'" % (vm.nic[0].gateway, vm.nic[0].ipaddress),
hypervisor=self.hypervisor
)
else:
try:
result = get_process_status(
host.ipaddress,
host.port,
host.user,
host.passwd,
router.linklocalip,
"cat /var/cache/cloud/passwords-%s | grep %s | sed 's/=/ /g' | awk '{print $1}'" % (vm.nic[0].gateway, vm.nic[0].ipaddress))
except KeyError:
self.skipTest(
"Provide a marvin config file with host\
credentials to run %s" %
self._testMethodName)
self.logger.debug("cat /var/cache/cloud/passwords-%s | grep %s | sed 's/=/ /g' | awk '{print $1}' RESULT IS ==> %s" % (vm.nic[0].gateway, vm.nic[0].ipaddress, result))
res = str(result)
self.assertEqual(
res.count(vm.nic[0].ipaddress),
1,
"Password file is empty or doesn't exist!")
def test_dhcphosts(self, vm, router):
hosts = list_hosts(
self.apiclient,
id=router.hostid)
self.assertEqual(
isinstance(hosts, list),
True,
"Check for list hosts response return valid data")
host = hosts[0]
host.user = self.hostConfig['username']
host.passwd = self.hostConfig['password']
host.port = self.services["configurableData"]["host"]["port"]
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"cat /etc/dhcphosts.txt | grep %s | sed 's/\,/ /g' | awk '{print $2}'" % (vm.nic[0].ipaddress),
hypervisor=self.hypervisor)
else:
try:
result = get_process_status(
host.ipaddress,
host.port,
host.user,
host.passwd,
router.linklocalip,
"cat /etc/dhcphosts.txt | grep %s | sed 's/\,/ /g' | awk '{print $2}'" % (vm.nic[0].ipaddress))
except KeyError:
self.skipTest(
"Provide a marvin config file with host\
credentials to run %s" %
self._testMethodName)
self.logger.debug("cat /etc/dhcphosts.txt | grep %s | sed 's/\,/ /g' | awk '{print $2}' RESULT IS ==> %s" % (vm.nic[0].ipaddress, result))
res = str(result)
self.assertEqual(
res.count(vm.nic[0].ipaddress),
1,
"DHCP hosts file contains duplicate IPs ==> %s!" % res)
def test_01_routervm_iptables_policies(self):
""" Test iptables default INPUT/FORWARD policy on RouterVM """
self.logger.debug("Starting test_01_routervm_iptables_policies")
vm1 = self.entity_manager.deployvm()
routers = self.entity_manager.query_routers()
self.assertEqual(
isinstance(routers, list), True,
"Check for list routers response return valid data")
for router in routers:
if not router.isredundantrouter and not router.vpcid:
hosts = list_hosts(
self.apiclient,
id=router.hostid)
self.assertEqual(
isinstance(hosts, list),
True,
"Check for list hosts response return valid data")
host = hosts[0]
host.user = self.services["configurableData"]["host"]["username"]
host.passwd = self.services["configurableData"]["host"]["password"]
host.port = self.services["configurableData"]["host"]["port"]
tables = [self.services["configurableData"]["input"], self.services["configurableData"]["forward"]]
for table in tables:
try:
result = get_process_status(
host.ipaddress,
host.port,
host.user,
host.passwd,
router.linklocalip,
'iptables -L %s' % table)
except KeyError:
self.skipTest(
"Provide a marvin config file with host\
credentials to run %s" %
self._testMethodName)
self.logger.debug("iptables -L %s: %s" % (table, result))
res = str(result)
self.assertEqual(
res.count("policy DROP"),
1,
"%s Default Policy should be DROP" % table)
def getCommandResultFromRouter(self, router, command):
"""Run given command on router and return the result"""
if (self.hypervisor.lower() == 'vmware'
or self.hypervisor.lower() == 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
command,
hypervisor=self.hypervisor
)
else:
hosts = list_hosts(
self.apiclient,
id=router.hostid,
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check for list hosts response return valid data"
)
host = hosts[0]
host.user = self.services["configurableData"]["host"]["username"]
host.passwd = self.services["configurableData"]["host"]["password"]
result = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
command
)
return result
def get_router_state(self, router):
host = self.get_host_details(router)
router_state = "UNKNOWN"
try:
router_state = get_process_status(
host.ipaddress,
host.port,
host.user,
host.passwd,
router.linklocalip,
"/opt/cosmic/router/scripts/checkrouter.sh | cut -d\" \" -f2"
)
except:
self.logger.debug("Oops, unable to determine redundant state for router with link local address %s" % (router.linklocalip))
pass
self.logger.debug("The router with link local address %s reports state %s" % (router.linklocalip, router_state))
return router_state[0]
def check_routers_interface(self,count=2, interface_to_check="eth1", expected_exists=True, showall=False):
result = ""
self.query_routers(count, showall)
for router in self.routers:
if router.state == "Running":
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list host returns a valid list"
)
host = hosts[0]
try:
host.user, host.passwd = get_host_credentials(self.config, host.ipaddress)
result = str(get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"ip a | grep %s | grep state | awk '{print $9;}'" % interface_to_check
))
except KeyError:
self.skipTest("Marvin configuration has no host credentials to check router services")
if expected_exists:
if (result.count("UP") == 1) or (result.count("DOWN") == 1):
self.logger.debug("Expected interface '%s' to exist and it does!" % interface_to_check)
else:
self.fail("Expected interface '%s' to exist, but it didn't!" % interface_to_check)
else:
if (result.count("UP") == 1) or (result.count("DOWN") == 1):
self.fail("Expected interface '%s' to not exist, but it did!" % interface_to_check)
else:
self.logger.debug("Expected interface '%s' to not exist, and it didn't!" % interface_to_check)
def test_02_dns_service_on_alias_ip(self):
"""Deploy guest vm in new CIDR and verify dns service on alias ip
1.Deploy guest vm in new cidr
2.Verify dns service listens on alias ip in VR
"""
list_router_response = list_routers(self.apiclient,
zoneid=self.zone.id,
listall=True)
self.assertEqual(isinstance(list_router_response, list), True,
"Check list response returns a valid list")
router = list_router_response[0]
hosts = list_hosts(self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid)
self.assertEqual(isinstance(hosts, list), True,
"Check list host returns a valid list")
host = hosts[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(router.state, 'Running',
"Check list router response for router state")
port = self.testdata['configurableData']['host']["publicport"]
username = self.testdata['configurableData']['host']["username"]
password = self.testdata['configurableData']['host']["password"]
# SSH to host so that host key is saved in first
# attempt
SshClient(host.ipaddress, port, username, password)
proc = self.alias_ip + ":53"
result = get_process_status(host.ipaddress, port, username, password,
router.linklocalip,
"netstat -atnp | grep %s" % proc)
res = str(result)
self.debug("Dns process status on alias ip: %s" % res)
self.assertNotEqual(
res.find(proc) - 1, "dnsmasq service is not running on alias ip")
return
def test_password_server_logs(self, vm, router):
host = self.get_host_details(router)
router_state = self.get_router_state(router)
if router.isredundantrouter and router_state != "MASTER":
print "Found router in non-MASTER state '" + router.redundantstate + "' so skipping test."
return True
# Get the related passwd server logs for our vm
command_to_execute = "grep %s /var/log/messages" % vm.nic[0].ipaddress
password_log_result = ""
try:
password_log_result = get_process_status(
host.ipaddress,
host.port,
host.user,
host.passwd,
router.linklocalip,
command_to_execute)
except KeyError:
self.skipTest(
"Provide a marvin config file with host\
credentials to run %s" %
self._testMethodName)
command_result = str(password_log_result)
self.logger.debug("Got this response: %s " % command_result)
# Check to see if our VM is in the password file
if command_result.count("password saved for VM IP") == 0:
return False
# Check if the password was retrieved from the passwd server. If it is, the actual password is replaced with 'saved_password'
if command_result.count("password sent to") == 0:
return False
return True
def test_password_server_logs(self, vm, router):
host = self.get_host_details(router)
router_state = self.get_router_state(router)
if router.isredundantrouter and router_state != "MASTER":
print "Found router in non-MASTER state '" + router.redundantstate + "' so skipping test."
return True
# Get the related passwd server logs for our vm
command_to_execute = "grep %s /var/log/messages" % vm.nic[0].ipaddress
password_log_result = ""
try:
password_log_result = get_process_status(
host.ipaddress,
host.port,
host.user,
host.passwd,
router.linklocalip,
command_to_execute)
except KeyError:
self.skipTest(
"Provide a marvin config file with host\
credentials to run %s" %
self._testMethodName)
command_result = str(password_log_result)
self.logger.debug("Got this response: %s " % command_result)
# Check to see if our VM is in the password file
if command_result.count("password saved for VM IP") == 0:
return False
# Check if the password was retrieved from the passwd server. If it is, the actual password is replaced with 'saved_password'
if command_result.count("password sent to") == 0:
return False
return True
def test_process_running(self, find_process, router):
host = self.get_host_details(router)
number_of_processes_found = 0
try:
number_of_processes_found = get_process_status(
host.ipaddress,
host.port,
host.user,
host.passwd,
router.linklocalip,
"ps aux | grep " + find_process + " | grep -v grep | wc -l"
)
except KeyError:
self.skipTest("Provide a marvin config file with host credentials to run %s" % self._testMethodName)
self.logger.debug("Result from the Router on IP '%s' is -> Number of processess found: '%s'" % (router.linklocalip, number_of_processes_found[0]))
expected_nr_or_processes = 1
self.assertEqual(int(number_of_processes_found[0]), expected_nr_or_processes,
msg="Router should have " + str(expected_nr_or_processes) + " '" + find_process + "' processes running, found " + str(number_of_processes_found[0]))
def test_process_running(self, find_process, router):
host = self.get_host_details(router)
number_of_processes_found = 0
try:
number_of_processes_found = get_process_status(
host.ipaddress,
host.port,
host.user,
host.passwd,
router.linklocalip,
"ps aux | grep " + find_process + " | grep -v grep | wc -l"
)
except KeyError:
self.skipTest("Provide a marvin config file with host credentials to run %s" % self._testMethodName)
self.logger.debug("Result from the Router on IP '%s' is -> Number of processess found: '%s'" % (router.linklocalip, number_of_processes_found[0]))
expected_nr_or_processes = 1
self.assertEqual(int(number_of_processes_found[0]), expected_nr_or_processes,
msg="Router should have " + str(expected_nr_or_processes) + " '" + find_process + "' processes running, found " + str(number_of_processes_found[0]))
def test_03_ssvm_internals(self):
"""Test SSVM Internals"""
# Validate the following
# 1. The SSVM check script should not return any
# WARN|ERROR|FAIL messages
# 2. If you are unable to login to the SSVM with the signed key
# then test is deemed a failure
# 3. There should be only one ""cloud"" process running within the SSVM
# 4. If no process is running/multiple process are running
# then the test is a failure
list_ssvm_response = list_ssvms(self.apiclient,
systemvmtype='secondarystoragevm',
state='Running',
zoneid=self.zone.id)
self.assertEqual(isinstance(list_ssvm_response, list), True,
"Check list response returns a valid list")
ssvm = list_ssvm_response[0]
hosts = list_hosts(self.apiclient, id=ssvm.hostid)
self.assertEqual(isinstance(hosts, list), True,
"Check list response returns a valid list")
host = hosts[0]
self.debug("Running SSVM check script")
if self.hypervisor.lower() in ('vmware', 'hyperv'):
# SSH into SSVMs is done via management server for Vmware and
# Hyper-V
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
ssvm.privateip,
"/usr/local/cloud/systemvm/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL",
hypervisor=self.hypervisor)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress, 22, host.user, host.passwd,
ssvm.linklocalip,
"/usr/local/cloud/systemvm/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL"
)
except KeyError:
self.skipTest("Marvin configuration has no host\
credentials to check router services")
res = str(result)
self.debug("SSVM script output: %s" % res)
self.assertEqual(res.count("ERROR"), 1, "Check for Errors in tests")
self.assertEqual(res.count("WARNING"), 1,
"Check for warnings in tests")
# Check status of cloud service
if self.hypervisor.lower() in ('vmware', 'hyperv'):
# SSH into SSVMs is done via management server for Vmware and
# Hyper-V
result = get_process_status(self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
ssvm.privateip,
"service cloud status",
hypervisor=self.hypervisor)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(host.ipaddress, 22, host.user,
host.passwd, ssvm.linklocalip,
"service cloud status")
except KeyError:
self.skipTest("Marvin configuration has no host\
credentials to check router services")
res = str(result)
self.debug("Cloud Process status: %s" % res)
# cloud.com service (type=secstorage) is running: process id: 2346
self.assertEqual(res.count("is running"), 1,
"Check cloud service is running or not")
linklocal_ip = None
# Check status of cloud service
if self.hypervisor.lower() in ('vmware', 'hyperv'):
# SSH into SSVMs is done via management server for Vmware and
# Hyper-V
linklocal_ip = ssvm.privateip
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
ssvm.privateip,
"cat /var/cache/cloud/cmdline | xargs | sed \"s/ /\\n/g\" | grep eth1ip= | sed \"s/\=/ /g\" | awk '{print $2}'",
hypervisor=self.hypervisor)
else:
try:
linklocal_ip = ssvm.linklocalip
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress, 22, host.user, host.passwd,
ssvm.linklocalip,
"cat /var/cache/cloud/cmdline | xargs | sed \"s/ /\\n/g\" | grep eth0ip= | sed \"s/\=/ /g\" | awk '{print $2}'"
)
except KeyError:
self.skipTest("Marvin configuration has no host\
credentials to check router services")
res = result[0]
self.debug("Cached Link Local IP: %s" % res)
self.assertEqual(
linklocal_ip, res,
"The cached Link Local should be the same as the current Link Local IP, but they are different! Current ==> %s; Cached ==> %s "
% (linklocal_ip, res))
return
def test_11_ss_nfs_version_on_ssvm(self):
"""Test NFS Version on Secondary Storage mounted properly on SSVM
"""
# 1) List SSVM in zone
# 2) Get id and url from mounted nfs store
# 3) Update NFS version for previous image store
# 4) Stop SSVM
# 5) Check NFS version of mounted nfs store after SSVM starts
nfs_version = self.config.nfsVersion
if nfs_version == None:
self.skipTest('No NFS version provided in test data')
#List SSVM for zone id
list_ssvm_response = list_ssvms(self.apiclient,
systemvmtype='secondarystoragevm',
state='Running',
zoneid=self.zone.id)
self.assertNotEqual(list_ssvm_response, None)
self.assertEqual(isinstance(list_ssvm_response, list), True,
"Check list response returns a valid list")
self.assertEqual(len(list_ssvm_response), 1,
"Check list System VMs response")
ssvm = list_ssvm_response[0]
image_stores_response = ImageStore.list(self.apiclient,
zoneid=self.zone.id)
if self.hypervisor.lower() in ('vmware', 'hyperv'):
# SSH into SSVMs is done via management server for Vmware and Hyper-V
result = get_process_status(self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
ssvm.privateip,
"mount | grep 'type nfs'",
hypervisor=self.hypervisor)
for res in result:
split_res = res.split("on")
mounted_img_store_url = split_res[0].strip()
for img_store in image_stores_response:
img_store_url = str(img_store.url)
if img_store_url.startswith("nfs://"):
img_store_url = img_store_url[6:]
#Add colon after ip address to match output from mount command
first_slash = img_store_url.find('/')
img_store_url = img_store_url[
0:first_slash] + ':' + img_store_url[first_slash:]
if img_store_url == mounted_img_store_url:
img_store_id = img_store.id
break
self.assertNotEqual(img_store_id, None,
"Check image store id mounted on SSVM")
#Update NFS version for image store mounted on SSVM
updateConfigurationCmd = updateConfiguration.updateConfigurationCmd()
updateConfigurationCmd.name = "secstorage.nfs.version"
updateConfigurationCmd.value = nfs_version
updateConfigurationCmd.imagestoreuuid = img_store_id
updateConfigurationResponse = self.apiclient.updateConfiguration(
updateConfigurationCmd)
self.logger.debug("updated the parameter %s with value %s" %
(updateConfigurationResponse.name,
updateConfigurationResponse.value))
#Stop SSVM
self.debug("Stopping SSVM: %s" % ssvm.id)
cmd = stopSystemVm.stopSystemVmCmd()
cmd.id = ssvm.id
self.apiclient.stopSystemVm(cmd)
def checkForRunningSSVM():
new_list_ssvm_response = list_ssvms(self.apiclient, id=ssvm.id)
if isinstance(new_list_ssvm_response, list):
return new_list_ssvm_response[0].state == 'Running', None
res, _ = wait_until(self.services["sleep"], self.services["timeout"],
checkForRunningSSVM)
if not res:
self.fail("List SSVM call failed!")
new_list_ssvm_response = list_ssvms(self.apiclient, id=ssvm.id)
self.assertNotEqual(new_list_ssvm_response, None)
self.assertEqual(isinstance(new_list_ssvm_response, list), True,
"Check list response returns a valid list")
ssvm = new_list_ssvm_response[0]
self.debug("SSVM state after debug: %s" % ssvm.state)
self.assertEqual(ssvm.state, 'Running',
"Check whether SSVM is running or not")
# Wait for the agent to be up
self.waitForSystemVMAgent(ssvm.name)
#Check NFS version on mounted image store
result = get_process_status(self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
ssvm.privateip,
"mount | grep '%s'" %
mounted_img_store_url,
hypervisor=self.hypervisor)
self.assertNotEqual(result, None)
self.assertEqual(len(result), 1, "Check result length")
res = result[0]
mounted_nfs_version = res.split("vers=")[1][0:1]
self.assertEqual(
int(mounted_nfs_version), int(nfs_version),
"Check mounted NFS version to be the same as provided")
return
def test_01_VRServiceFailureAlerting(self):
if self.zone.networktype == "Basic":
list_router_response = list_routers(
self.apiclient,
listall="true"
)
else:
list_router_response = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid
)
self.assertEqual(
isinstance(list_router_response, list),
True,
"Check list response returns a valid list"
)
router = list_router_response[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(
router.state,
'Running',
"Check list router response for router state"
)
alertSubject = "Monitoring Service on VR " + router.name
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"service dnsmasq stop",
hypervisor=self.hypervisor
)
else:
try:
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list host returns a valid list"
)
host = hosts[0]
result = get_process_status(
host.ipaddress,
22,
self.services["configurableData"]["host"]["username"],
self.services["configurableData"]["host"]["password"],
router.linklocalip,
"service apache2 stop"
)
except Exception as e:
raise Exception("Exception raised in getting host\
credentials: %s " % e)
res = str(result)
self.debug("apache process status: %s" % res)
configs = Configurations.list(
self.apiclient,
name='router.alerts.check.interval'
)
# Set the value for one more minute than
# actual range to be on safer side
waitingPeriod = (
int(configs[0].value) + 600) # in seconds
time.sleep(waitingPeriod)
# wait for (router.alerts.check.interval + 10) minutes meanwhile monitor service on
# VR starts the apache service (
# router.alerts.check.interval default value is
# 30minutes)
qresultset = self.dbclient.execute(
"select id from alert where subject = '%s' ORDER BY id DESC LIMIT 1;" %
str(alertSubject))
self.assertNotEqual(
len(qresultset),
0,
"Check DB Query result set"
)
return
def test_03_RVR_Network_check_router_state(self):
""" Test redundant router internals """
self.logger.debug("Starting test_03_RVR_Network_check_router_state...")
hypervisor = self.testClient.getHypervisorInfo()
self.logger.debug("Creating Network Offering with default egress FALSE")
network_offering_egress_false = NetworkOffering.create(
self.apiclient,
self.services["nw_off_persistent_RVR_egress_false"],
conservemode=True
)
network_offering_egress_false.update(self.apiclient, state='Enabled')
self.logger.debug("Creating network with network offering: %s" % network_offering_egress_false.id)
network = Network.create(
self.apiclient,
self.services["network"],
accountid=self.account.name,
domainid=self.account.domainid,
networkofferingid=network_offering_egress_false.id,
zoneid=self.zone.id
)
self.logger.debug("Created network with ID: %s" % network.id)
networks = Network.list(
self.apiclient,
id=network.id,
listall=True
)
self.assertEqual(
isinstance(networks, list),
True,
"List networks should return a valid response for created network"
)
nw_response = networks[0]
self.logger.debug("Deploying VM in account: %s" % self.account.name)
virtual_machine = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
templateid=self.template.id,
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id,
networkids=[str(network.id)]
)
self.logger.debug("Deployed VM in network: %s" % network.id)
self.cleanup.insert(0, network_offering_egress_false)
self.cleanup.insert(0, network)
self.cleanup.insert(0, virtual_machine)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List Vms should return a valid list"
)
vm = vms[0]
self.assertEqual(
vm.state,
"Running",
"VM should be in running state after deployment"
)
self.logger.debug("Listing routers for network: %s" % network.name)
routers = Router.list(
self.apiclient,
networkid=network.id,
listall=True
)
self.assertEqual(
isinstance(routers, list),
True,
"list router should return Primary and backup routers"
)
self.assertEqual(
len(routers),
2,
"Length of the list router should be 2 (Backup & Primary)"
)
vals = ["PRIMARY", "BACKUP", "UNKNOWN"]
cnts = [0, 0, 0]
result = "UNKNOWN"
for router in routers:
if router.state == "Running":
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list host returns a valid list"
)
host = hosts[0]
if hypervisor.lower() in ('vmware', 'hyperv'):
result = str(get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"sh /opt/cloud/bin/checkrouter.sh ",
hypervisor=hypervisor
))
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = str(get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"sh /opt/cloud/bin/checkrouter.sh "
))
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials to\
check router services")
if result.count(vals[0]) == 1:
cnts[vals.index(vals[0])] += 1
if cnts[vals.index('PRIMARY')] != 1:
self.fail("No Primary or too many primary routers found %s" % cnts[vals.index('PRIMARY')])
return
def test_01_single_VPC_iptables_policies(self):
""" Test iptables default INPUT/FORWARD policies on VPC router """
self.logger.debug("Starting test_01_single_VPC_iptables_policies")
routers = self.entity_manager.query_routers()
self.assertEqual(isinstance(routers, list), True,
"Check for list routers response return valid data")
self.entity_manager.create_network(
self.services["vpc_network_offering"], self.vpc.id, "10.1.1.1")
self.entity_manager.create_network(
self.services["vpc_network_offering_no_lb"], self.vpc.id,
"10.1.2.1")
self.entity_manager.add_nat_rules(self.vpc.id)
self.entity_manager.do_vpc_test()
for router in routers:
if not router.isredundantrouter and router.vpcid:
hosts = list_hosts(self.apiclient, id=router.hostid)
self.assertEqual(
isinstance(hosts, list), True,
"Check for list hosts response return valid data")
host = hosts[0]
host.user = self.hostConfig['username']
host.passwd = self.hostConfig['password']
host.port = self.services["configurableData"]["host"]["port"]
tables = [
self.services["configurableData"]["input"],
self.services["configurableData"]["forward"]
]
for table in tables:
result = None
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
'iptables -L %s' % table,
hypervisor=self.hypervisor)
else:
try:
result = get_process_status(
host.ipaddress, host.port, host.user,
host.passwd, router.linklocalip,
'iptables -L %s' % table)
except KeyError:
self.skipTest(
"Provide a marvin config file with host\
credentials to run %s" %
self._testMethodName)
self.logger.debug("iptables -L %s: %s" % (table, result))
res = str(result)
self.assertEqual(
res.count("DROP"), 1,
"%s Default Policy should be DROP" % table)
def test_05_del_cidr_verify_alias_removal(self):
"""Destroy lastvm in the CIDR and verifly alias removal
1.Deploy guest vm in new cidr
2.Verify ip alias creation
3.Destroy vm and wait for it to expunge
4.Verify ip alias removal after vm expunge
"""
list_router_response = list_routers(self.apiclient,
zoneid=self.zone.id,
listall=True)
self.assertEqual(isinstance(list_router_response, list), True,
"Check list response returns a valid list")
router = list_router_response[0]
hosts = list_hosts(self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid)
self.assertEqual(isinstance(hosts, list), True,
"Check list host returns a valid list")
host = hosts[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(router.state, 'Running',
"Check list router response for router state")
port = self.testdata['configurableData']['host']["publicport"]
username = self.testdata['configurableData']['host']["username"]
password = self.testdata['configurableData']['host']["password"]
# SSH to host so that host key is saved in first
# attempt
SshClient(host.ipaddress, port, username, password)
proc = "ip addr show eth0"
result = get_process_status(host.ipaddress, port, username, password,
router.linklocalip, proc)
res = str(result)
self.debug("ip alias configuration on VR: %s" % res)
self.assertNotEqual(
res.find(self.alias_ip) - 1, "ip alias is not created on VR eth0")
self.virtual_machine.delete(self.apiclient)
self.debug("Verify that expunging the last vm in the CIDR should\
delete the ip alias from VR")
ip_alias2 = self.dbclient.execute(
"select ip4_address from nic_ip_alias;")
self.assertEqual(isinstance(ip_alias2, list), True,
"Error in sql query")
self.assertEqual(len(ip_alias2), 0,
"Failure in clearing ip alias entry from cloud db")
proc = "ip addr show eth0"
result = get_process_status(host.ipaddress, port, username, password,
router.linklocalip, proc)
res = str(result)
self.assertEqual(
res.find(self.alias_ip), -1,
"Failed to clean up ip alias from VR even after\
last vm expunge in the CIDR")
self.debug("IP alias got deleted from VR successfully.")
self.cleanup.remove(self.virtual_machine)
return
def test_04_cpvm_internals(self):
"""Test CPVM Internals"""
# Validate the following
# 1. test that telnet access on 8250 is available to
# the management server for the CPVM
# 2. No telnet access, test FAIL
# 3. Service cloud status should report cloud agent status to be
# running
list_cpvm_response = list_ssvms(
self.apiclient,
systemvmtype='consoleproxy',
state='Running',
zoneid=self.zone.id
)
self.assertEqual(
isinstance(list_cpvm_response, list),
True,
"Check list response returns a valid list"
)
cpvm = list_cpvm_response[0]
hosts = list_hosts(
self.apiclient,
id=cpvm.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list response returns a valid list"
)
host = hosts[0]
try:
telnetlib.Telnet(
str(self.apiclient.connection.mgtSvr),
'8250'
)
self.debug("Telnet management server (IP: %s)" %
self.apiclient.connection.mgtSvr)
except Exception as e:
self.fail(
"Telnet Access failed for %s: %s" %
(self.apiclient.connection.mgtSvr, e)
)
self.debug("Checking cloud process status")
if self.hypervisor.lower() in ('vmware', 'hyperv'):
# SSH into SSVMs is done via management server for Vmware and
# Hyper-V
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
cpvm.privateip,
"service cloud status",
hypervisor=self.hypervisor
)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
cpvm.linklocalip,
"service cloud status"
)
except KeyError:
self.skipTest(
"Marvin configuration has no host\
credentials to check router services")
res = str(result)
self.debug("Cloud Process status: %s" % res)
self.assertEqual(
res.count("is running"),
1,
"Check cloud service is running or not"
)
return
def test_04_restart_network_wo_cleanup(self):
"""Test restart network without cleanup
"""
# Validate the following
# 1. When cleanup = false, router is restarted and
# all services inside the router are restarted
# 2. check 'uptime' to see if the actual restart happened
timeout = 10
# Network should be in Implemented or Setup stage before restart
while True:
networks = list_networks(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid
)
self.assertEqual(
isinstance(networks, list),
True,
"Check list response returns a valid list"
)
network = networks[0]
if network.state in ["Implemented", "Setup"]:
break
elif timeout == 0:
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
self.debug(
"Restarting network with ID: %s, Network state: %s" % (
network.id,
network.state
))
cmd = restartNetwork.restartNetworkCmd()
cmd.id = network.id
cmd.cleanup = False
self.apiclient.restartNetwork(cmd)
# Get router details after restart
list_router_response = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid
)
self.assertEqual(
isinstance(list_router_response, list),
True,
"Check list response returns a valid list"
)
router = list_router_response[0]
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list response returns a valid list"
)
host = hosts[0]
if self.hypervisor.lower() in ('vmware', 'hyperv'):
res = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"uptime",
hypervisor=self.hypervisor
)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
res = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"uptime"
)
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials\
to check router services")
# res = 12:37:14 up 1 min, 0 users, load average: 0.61, 0.22, 0.08
# Split result to check the uptime
result = res[0].split()
self.debug("Router Uptime: %s" % result)
self.assertEqual(
str(result[1]),
'up',
"Check router is running or not"
)
if str(result[3]) == "min,":
self.assertEqual(
(int(result[2]) < 3),
True,
"Check uptime is less than 3 mins or not"
)
else:
self.assertEqual(
str(result[3]),
'sec,',
"Check uptime is in seconds"
)
return
def test_01_Multiple_RemoteAccessVPN_Connections_To_VPC_Ping_Guest_VM_Multiple_Times(self):
""" Test case no : Test Multiple VPN Connections to a VPN Server on VPC
# Validate the following for Each VPN VM Client
# 1. Create VPN User on the VPC
# 2. Configure the VPN Client VM with the required Information
# 3. Initialize the VPN Client Services on the VPN Client
# 4. Start the VPN Client Services on the VPN Client
# 5. Wait for 30 seconds before attempting to ping
# 6. Conduct the Ping Test on the VM
# After the deployment VPN Client VMs and the post deployment steps, do the following steps:
# 7. Wait for 60 seconds
# 8. Check Routers pppX NICs Information
"""
for vm in xrange(0,int(TestMultipleVPNAccessonVPC.services["vpnclient_count"])):
vpn_user_name = ''.join((str(vm),"-user"))
vpn_password = ''.join((str(vm),"-pass"))
self.debug("VPN User Name created with %s " % vpn_user_name)
self.debug("VPN Password created with %s " % vpn_password)
self.debug("Create new VPN User to use the Remote Access Service enabled on the VPC")
newVPNUser = VpnUser.create(
TestMultipleVPNAccessonVPC.api_client_vpn_server_reg_user,
vpn_user_name,
vpn_password,
rand_name=False
)
self.debug("VPN User %s got created Successfully " % vpn_user_name)
self.debug("Configure the VPN Client Services on the VM deployed for VPN client purpose.")
TestMultipleVPNAccessonVPC.configureVPNClientServicesFile(
TestMultipleVPNAccessonVPC.vpnclientvms[vm],
"/tmp/vpnclient_services.sh",
TestMultipleVPNAccessonVPC.listFirstVPCPublicIpAddress[0].ipaddress,
TestMultipleVPNAccessonVPC.listfirstNetworkTier[0].cidr,
TestMultipleVPNAccessonVPC.FirstVPNonFirstVPC.presharedkey,
vpn_user_name,
vpn_password
)
self.debug("Configuration of VPN Client VM %d Done " % (vm))
self.debug("Initialize the VPN Client Services on the VPN Client")
TestMultipleVPNAccessonVPC.vpnClientServicesInit(
TestMultipleVPNAccessonVPC.vpnclientvms[vm],
"/tmp/vpnclient_services.sh"
)
self.debug("Initiation of VPN Client Services on VM %d Done " % (vm))
self.debug("Start the VPN Client Services on the VPN Client")
TestMultipleVPNAccessonVPC.vpnClientServicesStart(
TestMultipleVPNAccessonVPC.vpnclientvms[vm],
"/tmp/vpnclient_services.sh"
)
self.debug("VPN Client Services on VM %d Started Successfully " % (vm))
self.debug("Wait for 30 seconds before attempting to ping")
time.sleep(30)
self.debug("Conduct the Ping Test on the VM %d" % (vm))
thread.start_new_thread(self.ping_vm,(
TestMultipleVPNAccessonVPC.vpnclientvms[vm],
TestMultipleVPNAccessonVPC.vm1.nic[0].ipaddress,
25000,
15,
"Thread-{0}".format(vm)
))
self.debug("Waiting for 60 seconds.........")
time.sleep(60)
self.debug("End of 60 seconds.........")
# Find router associated with user account
list_router_response = list_routers(
self.apiclient,
vpcid= TestMultipleVPNAccessonVPC.firstvpc.id,
listall=True
)
self.assertEqual(
isinstance(list_router_response, list),
True,
"Check list response returns a valid list"
)
router = list_router_response[0]
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list response returns a valid list"
)
host = hosts[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(
router.state,
'Running',
"Check list router response for router state"
)
if self.hypervisor.lower() == 'vmware':
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"ifconfig | grep ppp",
hypervisor=self.hypervisor
)
else:
try:
host.user, host.passwd = get_host_credentials(self.config, host.ipaddress)
result = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"ifconfig | grep ppp"
)
self.debug("Routers pppX NICs Information : %s" % str(result))
except KeyError:
self.skipTest("Marvin configuration has no host credentials to check router services")
def test_03_RVR_Network_check_router_state(self):
""" Test redundant router internals """
self.logger.debug("Starting test_03_RVR_Network_check_router_state...")
network_offering_egress_false = get_default_redundant_isolated_network_offering(
self.apiclient)
self.logger.debug("Creating network with network offering: %s" %
network_offering_egress_false.id)
network = Network.create(
self.apiclient,
self.services["network"],
accountid=self.account.name,
domainid=self.account.domainid,
networkofferingid=network_offering_egress_false.id,
zoneid=self.zone.id)
self.logger.debug("Created network with ID: %s" % network.id)
networks = Network.list(self.apiclient, id=network.id, listall=True)
self.assertEqual(
isinstance(networks, list), True,
"List networks should return a valid response for created network")
self.logger.debug("Deploying VM in account: %s" % self.account.name)
virtual_machine = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
templateid=self.template.id,
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id,
networkids=[str(network.id)])
self.logger.debug("Deployed VM in network: %s" % network.id)
self.cleanup.insert(0, network)
self.cleanup.insert(0, virtual_machine)
vms = VirtualMachine.list(self.apiclient,
id=virtual_machine.id,
listall=True)
self.assertEqual(isinstance(vms, list), True,
"List Vms should return a valid list")
vm = vms[0]
self.assertEqual(vm.state, "Running",
"VM should be in running state after deployment")
self.logger.debug("Listing routers for network: %s" % network.name)
routers = Router.list(self.apiclient,
networkid=network.id,
listall=True)
self.assertEqual(
isinstance(routers, list), True,
"list router should return Master and backup routers")
self.assertEqual(
len(routers), 2,
"Length of the list router should be 2 (Backup & master)")
vals = ["MASTER", "BACKUP", "UNKNOWN"]
cnts = [0, 0, 0]
result = "UNKNOWN"
for router in routers:
if router.state == "Running":
hosts = list_hosts(self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid)
self.assertEqual(isinstance(hosts, list), True,
"Check list host returns a valid list")
host = hosts[0]
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = str(
get_process_status(
host.ipaddress, 22, host.user, host.passwd,
router.linklocalip,
"sh /opt/cosmic/router/scripts/checkrouter.sh "))
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials to\
check router services")
if result.count(vals[0]) == 1:
cnts[vals.index(vals[0])] += 1
if cnts[vals.index('MASTER')] != 1:
self.fail("No Master or too many master routers found %s" %
cnts[vals.index('MASTER')])
return
def test_01_RouterStopCreateFW(self):
"""Test router stop create Firewall rule
"""
# validate the following
# 1. 1. listFirewallRules (filter by ipaddressid of sourcenat)
# 2. rule should be for ports 1-600 and in state=Active
# (optional backend)
# 3. verify on router using iptables -t nat -nvx if rules are applied
# Get the router details associated with account
routers = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
)
self.assertEqual(
isinstance(routers, list),
True,
"Check for list routers response return valid data"
)
self.assertNotEqual(
len(routers),
0,
"Check list router response"
)
router = routers[0]
self.debug("Stopping the router: %s" % router.id)
# Stop the router
cmd = stopRouter.stopRouterCmd()
cmd.id = router.id
self.apiclient.stopRouter(cmd)
routers = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
)
self.assertEqual(
isinstance(routers, list),
True,
"Check for list routers response return valid data"
)
router = routers[0]
self.assertEqual(
router.state,
'Stopped',
"Check list router response for router state"
)
public_ips = list_publicIP(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid
)
self.assertEqual(
isinstance(public_ips, list),
True,
"Check for list public IP response return valid data"
)
public_ip = public_ips[0]
# Create Firewall rule with configurations from settings file
fw_rule = FireWallRule.create(
self.apiclient,
ipaddressid=public_ip.id,
protocol='TCP',
cidrlist=[self.services["fwrule"]["cidr"]],
startport=self.services["fwrule"]["startport"],
endport=self.services["fwrule"]["endport"]
)
self.debug("Created firewall rule: %s" % fw_rule.id)
self.debug("Starting the router: %s" % router.id)
# Start the router
cmd = startRouter.startRouterCmd()
cmd.id = router.id
self.apiclient.startRouter(cmd)
routers = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
)
self.assertEqual(
isinstance(routers, list),
True,
"Check for list routers response return valid data"
)
router = routers[0]
self.assertEqual(
router.state,
'Running',
"Check list router response for router state"
)
# After Router start, FW rule should be in Active state
fw_rules = list_firewall_rules(
self.apiclient,
id=fw_rule.id,
)
self.assertEqual(
isinstance(fw_rules, list),
True,
"Check for list FW rules response return valid data"
)
self.assertEqual(
fw_rules[0].state,
'Active',
"Check list load balancing rules"
)
self.assertEqual(
fw_rules[0].startport,
self.services["fwrule"]["startport"],
"Check start port of firewall rule"
)
self.assertEqual(
fw_rules[0].endport,
self.services["fwrule"]["endport"],
"Check end port of firewall rule"
)
# For DNS and DHCP check 'dnsmasq' process status
if (self.hypervisor.lower() == 'vmware'
or self.hypervisor.lower() == 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
'iptables -t nat -L',
hypervisor=self.hypervisor
)
else:
hosts = list_hosts(
self.apiclient,
id=router.hostid,
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check for list hosts response return valid data"
)
host = hosts[0]
try:
result = get_process_status(
host.ipaddress,
22,
self.hostConfig['username'],
self.hostConfig['password'],
router.linklocalip,
'iptables -t nat -L'
)
except KeyError:
self.skipTest(
"Provide a marvin config file with host\
credentials to run %s" %
self._testMethodName)
self.debug("iptables -t nat -L: %s" % result)
self.debug("Public IP: %s" % public_ip.ipaddress)
res = str(result)
self.assertEqual(
res.count(str(public_ip.ipaddress)),
1,
"Check public IP address"
)
return
def check_routers_state(self, count=2, status_to_check="MASTER", expected_count=1, showall=False):
vals = ["MASTER", "BACKUP", "UNKNOWN", "TESTFAILED"]
cnts = [0, 0, 0]
result = "TESTFAILED"
self.logger.debug('check_routers_state count: %s, status_to_check: %s, expected_count: %s, showall: %s' % (count, status_to_check, expected_count, showall))
vrrp_interval = Configurations.list(self.apiclient, name="router.redundant.vrrp.interval")
self.logger.debug("router.redundant.vrrp.interval is ==> %s" % vrrp_interval)
total_sleep = 20
if vrrp_interval:
total_sleep = (int(vrrp_interval[0].value) * 4) + 10
else:
self.logger.debug("Could not retrieve the key 'router.redundant.vrrp.interval'. Sleeping for 10 seconds.")
'''
Sleep (router.redundant.vrrp.interval * 4) seconds here because VRRP will have to be reconfigured. Due to the configuration changes,
it will start a new election and that will take ~4 multiplied by the advertisement interval seconds. Next to that, we need some time
for the router to be reconfigured, so adding 10 seconds to be on the safe side.
'''
time.sleep(total_sleep)
self.query_routers(count, showall)
for router in self.routers:
if router.state == "Running":
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list host returns a valid list"
)
host = hosts[0]
for _ in range(5):
host.user, host.passwd = get_host_credentials(self.config, host.name)
result = str(get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"sh /opt/cosmic/router/scripts/checkrouter.sh "
))
self.logger.debug('check_routers_state router: %s, result: %s' % (router.name, result))
if result.count(status_to_check) == 1:
cnts[vals.index(status_to_check)] += 1
break
elif result.count("UNKNOWN") == 1:
time.sleep(5)
else:
break
if cnts[vals.index(status_to_check)] != expected_count:
self.logger.debug("Investigate! not MASTER/BACKUP")
while True:
time.sleep(1)
self.fail("Expected '%s' router[s] at state '%s', but found '%s'! Result: %s" % (expected_count, status_to_check, cnts[vals.index(status_to_check)], result))
def test_11_ss_nfs_version_on_ssvm(self):
"""Test NFS Version on Secondary Storage mounted properly on SSVM
"""
# 1) List SSVM in zone
# 2) Get id and url from mounted nfs store
# 3) Update NFS version for previous image store
# 4) Stop SSVM
# 5) Check NFS version of mounted nfs store after SSVM starts
nfs_version = self.config.nfsVersion
if nfs_version == None:
self.skipTest('No NFS version provided in test data')
#List SSVM for zone id
list_ssvm_response = list_ssvms(
self.apiclient,
systemvmtype='secondarystoragevm',
state='Running',
zoneid=self.zone.id
)
self.assertNotEqual(
list_ssvm_response,
None
)
self.assertEqual(
isinstance(list_ssvm_response, list),
True,
"Check list response returns a valid list"
)
self.assertEqual(
len(list_ssvm_response),
1,
"Check list System VMs response"
)
ssvm = list_ssvm_response[0]
image_stores_response = ImageStore.list(self.apiclient,zoneid=self.zone.id)
if self.hypervisor.lower() in ('vmware', 'hyperv'):
# SSH into SSVMs is done via management server for Vmware and Hyper-V
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
ssvm.privateip,
"mount | grep 'type nfs'",
hypervisor=self.hypervisor)
for res in result:
split_res = res.split("on")
mounted_img_store_url = split_res[0].strip()
for img_store in image_stores_response:
img_store_url = str(img_store.url)
if img_store_url.startswith("nfs://"):
img_store_url = img_store_url[6:]
#Add colon after ip address to match output from mount command
first_slash = img_store_url.find('/')
img_store_url = img_store_url[0:first_slash] + ':' + img_store_url[first_slash:]
if img_store_url == mounted_img_store_url:
img_store_id = img_store.id
break
self.assertNotEqual(
img_store_id,
None,
"Check image store id mounted on SSVM"
)
#Update NFS version for image store mounted on SSVM
updateConfigurationCmd = updateConfiguration.updateConfigurationCmd()
updateConfigurationCmd.name = "secstorage.nfs.version"
updateConfigurationCmd.value = nfs_version
updateConfigurationCmd.imagestoreuuid = img_store_id
updateConfigurationResponse = self.apiclient.updateConfiguration(updateConfigurationCmd)
self.logger.debug("updated the parameter %s with value %s"%(updateConfigurationResponse.name, updateConfigurationResponse.value))
#Stop SSVM
self.debug("Stopping SSVM: %s" % ssvm.id)
cmd = stopSystemVm.stopSystemVmCmd()
cmd.id = ssvm.id
self.apiclient.stopSystemVm(cmd)
def checkForRunningSSVM():
new_list_ssvm_response = list_ssvms(
self.apiclient,
id=ssvm.id
)
if isinstance(new_list_ssvm_response, list):
return new_list_ssvm_response[0].state == 'Running', None
res, _ = wait_until(self.services["sleep"], self.services["timeout"], checkForRunningSSVM)
if not res:
self.fail("List SSVM call failed!")
new_list_ssvm_response = list_ssvms(
self.apiclient,
id=ssvm.id
)
self.assertNotEqual(
new_list_ssvm_response,
None
)
self.assertEqual(
isinstance(new_list_ssvm_response, list),
True,
"Check list response returns a valid list"
)
ssvm = new_list_ssvm_response[0]
self.debug("SSVM state after debug: %s" % ssvm.state)
self.assertEqual(
ssvm.state,
'Running',
"Check whether SSVM is running or not"
)
# Wait for the agent to be up
self.waitForSystemVMAgent(ssvm.name)
#Check NFS version on mounted image store
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
ssvm.privateip,
"mount | grep '%s'"%mounted_img_store_url,
hypervisor=self.hypervisor)
self.assertNotEqual(
result,
None
)
self.assertEqual(
len(result),
1,
"Check result length"
)
res = result[0]
mounted_nfs_version = res.split("vers=")[1][0:1]
self.assertEqual(
int(mounted_nfs_version),
int(nfs_version),
"Check mounted NFS version to be the same as provided"
)
return
def test_07_stop_start_VR_verify_ip_alias(self):
"""Reboot VR and verify ip alias
1.Deploy guest vm in new cidr
2.Verify ip alias creation
3.Stop and Start VR
4.Verify ip alias on VR
"""
list_router_response = list_routers(
self.apiclient,
zoneid=self.zone.id,
listall=True
)
self.assertEqual(
isinstance(list_router_response, list),
True,
"Check list response returns a valid list"
)
router = list_router_response[0]
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list host returns a valid list"
)
host = hosts[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(
router.state,
'Running',
"Check list router response for router state"
)
port = self.testdata['configurableData']['host']["publicport"]
username = self.testdata['configurableData']['host']["username"]
password = self.testdata['configurableData']['host']["password"]
# SSH to host so that host key is saved in first
# attempt
SshClient(host.ipaddress, port, username, password)
proc = "ip addr show eth0"
result = get_process_status(
host.ipaddress,
port,
username,
password,
router.linklocalip,
proc
)
res = str(result)
self.debug("ip alias configuration on VR: %s" % res)
self.assertNotEqual(
res.find(self.alias_ip)
- 1,
"ip alias is not created on VR eth0"
)
self.debug("Stopping VR")
Router.stop(
self.apiclient,
router.id,
)
self.debug("Starting VR")
Router.start(
self.apiclient,
router.id
)
list_router_response = list_routers(
self.apiclient,
zoneid=self.zone.id,
listall=True
)
self.assertEqual(
isinstance(list_router_response, list),
True,
"Check list response returns a valid list"
)
router = list_router_response[0]
self.assertEqual(
router.state,
'Running',
"Router is not in running state after reboot"
)
self.debug("VR is up and Running")
result = get_process_status(
host.ipaddress,
port,
username,
password,
router.linklocalip,
proc
)
res = str(result)
self.assertNotEqual(
res.find(self.alias_ip),
- 1,
"IP alias not present on VR after VR stop and start"
)
return
def test_05_del_cidr_verify_alias_removal(self):
"""Destroy lastvm in the CIDR and verifly alias removal
1.Deploy guest vm in new cidr
2.Verify ip alias creation
3.Destroy vm and wait for it to expunge
4.Verify ip alias removal after vm expunge
"""
list_router_response = list_routers(
self.apiclient,
zoneid=self.zone.id,
listall=True
)
self.assertEqual(
isinstance(list_router_response, list),
True,
"Check list response returns a valid list"
)
router = list_router_response[0]
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list host returns a valid list"
)
host = hosts[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(
router.state,
'Running',
"Check list router response for router state"
)
port = self.testdata['configurableData']['host']["publicport"]
username = self.testdata['configurableData']['host']["username"]
password = self.testdata['configurableData']['host']["password"]
# SSH to host so that host key is saved in first
# attempt
SshClient(host.ipaddress, port, username, password)
proc = "ip addr show eth0"
result = get_process_status(
host.ipaddress,
port,
username,
password,
router.linklocalip,
proc
)
res = str(result)
self.debug("ip alias configuration on VR: %s" % res)
self.assertNotEqual(
res.find(self.alias_ip)
- 1,
"ip alias is not created on VR eth0"
)
self.virtual_machine.delete(self.apiclient)
self.debug(
"Verify that expunging the last vm in the CIDR should\
delete the ip alias from VR")
ip_alias2 = self.dbclient.execute(
"select ip4_address from nic_ip_alias;"
)
self.assertEqual(
isinstance(ip_alias2, list),
True,
"Error in sql query"
)
self.assertEqual(
len(ip_alias2),
0,
"Failure in clearing ip alias entry from cloud db"
)
proc = "ip addr show eth0"
result = get_process_status(
host.ipaddress,
port,
username,
password,
router.linklocalip,
proc
)
res = str(result)
self.assertEqual(
res.find(
self.alias_ip),
- 1,
"Failed to clean up ip alias from VR even after\
last vm expunge in the CIDR")
self.debug("IP alias got deleted from VR successfully.")
self.cleanup.remove(self.virtual_machine)
return
def test_02_router_internal_adv(self):
"""Test router internal advanced zone
"""
# Validate the following
# 1. Router does dhcp, dns, gateway, LB, PF, FW
# 2. verify that dhcp, dns ports are open on UDP
# 3. dnsmasq, haproxy processes should be running
# Find router associated with user account
list_router_response = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid
)
self.assertEqual(
isinstance(list_router_response, list),
True,
"Check list response returns a valid list"
)
router = list_router_response[0]
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list response returns a valid list"
)
host = hosts[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(
router.state,
'Running',
"Check list router response for router state"
)
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"service dnsmasq status",
hypervisor=self.hypervisor
)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"service dnsmasq status"
)
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials\
to check router services")
res = str(result)
self.debug("Dnsmasq process status: %s" % res)
self.assertEqual(
res.count("running"),
1,
"Check dnsmasq service is running or not"
)
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"service haproxy status",
hypervisor=self.hypervisor
)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"service haproxy status"
)
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials\
to check router services")
res = str(result)
self.assertEqual(
res.count("running"),
1,
"Check haproxy service is running or not"
)
self.debug("Haproxy process status: %s" % res)
return
def test_02_router_internal_adv(self):
"""Test router internal advanced zone
"""
# Validate the following
# 1. Router does dhcp, dns, gateway, LB, PF, FW
# 2. verify that dhcp, dns ports are open on UDP
# 3. dnsmasq, haproxy processes should be running
# Find router associated with user account
list_router_response = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid
)
self.assertEqual(
isinstance(list_router_response, list),
True,
"Check list response returns a valid list"
)
router = list_router_response[0]
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list response returns a valid list"
)
host = hosts[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(
router.state,
'Running',
"Check list router response for router state"
)
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"service dnsmasq status",
hypervisor=self.hypervisor
)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"service dnsmasq status"
)
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials\
to check router services")
res = str(result)
self.debug("Dnsmasq process status: %s" % res)
self.assertEqual(
res.count("running"),
1,
"Check dnsmasq service is running or not"
)
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"service haproxy status",
hypervisor=self.hypervisor
)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"service haproxy status"
)
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials\
to check router services")
res = str(result)
self.assertEqual(
res.count("running"),
1,
"Check haproxy service is running or not"
)
self.debug("Haproxy process status: %s" % res)
return
def test_01_FTPModulesInVR(self):
"""
@desc: Verify FTP modules are loaded in VR of advance zone
step1 : create a VR in advance zone
step2: Verify FTP modules are there in created VR
"""
if self.zone.networktype == "Basic":
self.skipTest("This test can be run only in advance zone")
# create a virtual machine
vm = VirtualMachine.create(self.api_client,
self.services["small"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.small_offering.id,
mode=self.services["mode"])
self.assertIsNotNone(vm, "Failed to deploy virtual machine")
self.cleanup.append(vm)
response = VirtualMachine.list(self.api_client, id=vm.id)
status = validateList(response)
self.assertEqual(status[0], PASS,
"list vm response returned invalid list")
list_router_response = list_routers(self.apiclient,
account=self.account.name,
domainid=self.account.domainid)
status = validateList(list_router_response)
self.assertEqual(status[0], PASS,
"Check list response returns a valid list")
router = list_router_response[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(router.state, 'Running',
"Check list router response for router state")
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"lsmod | grep ftp",
hypervisor=self.hypervisor)
else:
try:
hosts = list_hosts(self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid)
self.assertEqual(isinstance(hosts, list), True,
"Check list host returns a valid list")
host = hosts[0]
result = get_process_status(
host.ipaddress, 22,
self.services["configurableData"]["host"]["username"],
self.services["configurableData"]["host"]["password"],
router.linklocalip, "lsmod | grep ftp")
except Exception as e:
raise Exception("Exception raised in getting host\
credentials: %s " % e)
res = str(result)
self.debug("lsmod | grep ftp: %s" % res)
if "nf_nat_ftp" in res and "nf_conntrack_ftp" in res:
ismoduleinstalled = True
else:
ismoduleinstalled = False
self.assertEqual(
ismoduleinstalled, True,
"nf_conntrack_ftp and nf_nat_ftp modules not installed on routers")
return
def test_04_restart_network_wo_cleanup(self):
"""Test restart network without cleanup
"""
# Validate the following
# 1. When cleanup = false, router is restarted and
# all services inside the router are restarted
# 2. check 'uptime' to see if the actual restart happened
timeout = 10
# Network should be in Implemented or Setup stage before restart
while True:
networks = list_networks(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid
)
self.assertEqual(
isinstance(networks, list),
True,
"Check list response returns a valid list"
)
network = networks[0]
if network.state in ["Implemented", "Setup"]:
break
elif timeout == 0:
break
else:
time.sleep(self.services["sleep"])
timeout = timeout - 1
self.debug(
"Restarting network with ID: %s, Network state: %s" % (
network.id,
network.state
))
cmd = restartNetwork.restartNetworkCmd()
cmd.id = network.id
cmd.cleanup = False
self.apiclient.restartNetwork(cmd)
# Get router details after restart
list_router_response = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid
)
self.assertEqual(
isinstance(list_router_response, list),
True,
"Check list response returns a valid list"
)
router = list_router_response[0]
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list response returns a valid list"
)
host = hosts[0]
if self.hypervisor.lower() in ('vmware', 'hyperv'):
res = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"uptime",
hypervisor=self.hypervisor
)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
res = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"uptime"
)
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials\
to check router services")
# res = 12:37:14 up 1 min, 0 users, load average: 0.61, 0.22, 0.08
# Split result to check the uptime
result = res[0].split()
self.debug("Router Uptime: %s" % result)
self.assertEqual(
str(result[1]),
'up',
"Check router is running or not"
)
if str(result[3]) == "min,":
self.assertEqual(
(int(result[2]) < 3),
True,
"Check uptime is less than 3 mins or not"
)
else:
self.assertEqual(
str(result[3]),
'sec,',
"Check uptime is in seconds"
)
return
def check_routers_state(self,
count=2,
status_to_check="MASTER",
expected_count=1,
showall=False):
vals = ["MASTER", "BACKUP", "UNKNOWN", "TESTFAILED"]
cnts = [0, 0, 0]
result = "TESTFAILED"
self.logger.debug(
'check_routers_state count: %s, status_to_check: %s, expected_count: %s, showall: %s'
% (count, status_to_check, expected_count, showall))
vrrp_interval = Configurations.list(
self.apiclient, name="router.redundant.vrrp.interval")
self.logger.debug("router.redundant.vrrp.interval is ==> %s" %
vrrp_interval)
total_sleep = 20
if vrrp_interval:
total_sleep = (int(vrrp_interval[0].value) * 4) + 10
else:
self.logger.debug(
"Could not retrieve the key 'router.redundant.vrrp.interval'. Sleeping for 10 seconds."
)
'''
Sleep (router.redundant.vrrp.interval * 4) seconds here because VRRP will have to be reconfigured. Due to the configuration changes,
it will start a new election and that will take ~4 multiplied by the advertisement interval seconds. Next to that, we need some time
for the router to be reconfigured, so adding 10 seconds to be on the safe side.
'''
time.sleep(total_sleep)
self.query_routers(count, showall)
for router in self.routers:
if router.state == "Running":
hosts = list_hosts(self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid)
self.assertEqual(isinstance(hosts, list), True,
"Check list host returns a valid list")
host = hosts[0]
try:
for _ in range(5):
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = str(
get_process_status(
host.ipaddress, 22, host.user, host.passwd,
router.linklocalip,
"sh /opt/cosmic/router/scripts/checkrouter.sh "
))
self.logger.debug(
'check_routers_state router: %s, result: %s' %
(router.name, result))
if result.count(status_to_check) == 1:
cnts[vals.index(status_to_check)] += 1
break
elif result.count("UNKNOWN") == 1:
time.sleep(5)
else:
break
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials to\
check router services")
if cnts[vals.index(status_to_check)] != expected_count:
self.fail(
"Expected '%s' router[s] at state '%s', but found '%s'! Result: %s"
% (expected_count, status_to_check,
cnts[vals.index(status_to_check)], result))
def test_01_create_tier_Vmxnet3(self):
"""
Test to create vpc tier with nic type as Vmxnet3
#1.Set global setting parameter "vmware.systemvm.nic.device.type"
to "Vmxnet3"
#2.Create VPC
#3.Create one tier
#4.Deploy one guest vm in the tier created in step3
"""
if self.hypervisor.lower() not in ['vmware']:
self.skipTest("This test can only run on vmware setup")
nic_types = Configurations.list(self.apiclient,
name="vmware.systemvm.nic.device.type")
self.assertEqual(
validateList(nic_types)[0], PASS, "Invalid list config")
nic_type = nic_types[0].value
reset = False
if nic_type.lower() != "vmxnet3":
self.updateConfigurAndRestart("vmware.systemvm.nic.device.type",
"Vmxnet3")
reset = True
self.services["vpc"]["cidr"] = "10.1.1.1/16"
self.debug("creating a VPC network in the account: %s" %
self.account.name)
try:
vpc = VPC.create(self.apiclient,
self.services["vpc"],
vpcofferingid=self.vpc_off.id,
zoneid=self.zone.id,
account=self.account.name,
domainid=self.account.domainid)
vpc_res = VPC.list(self.apiclient, id=vpc.id)
self.assertEqual(
validateList(vpc_res)[0], PASS,
"Invalid response from listvpc")
self.network_offering = NetworkOffering.create(
self.apiclient,
self.services["network_offering"],
conservemode=False)
# Enable Network offering
self.network_offering.update(self.apiclient, state='Enabled')
self.cleanup.append(self.network_offering)
gateway = vpc.cidr.split('/')[0]
# Split the cidr to retrieve gateway
# for eg. cidr = 10.0.0.1/24
# Gateway = 10.0.0.1
# Creating network using the network offering created
self.debug("Creating network with network offering: %s" %
self.network_offering.id)
network = Network.create(
self.apiclient,
self.services["network"],
accountid=self.account.name,
domainid=self.account.domainid,
networkofferingid=self.network_offering.id,
zoneid=self.zone.id,
gateway=gateway,
vpcid=vpc.id)
self.debug("Created network with ID: %s" % network.id)
vm = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id,
networkids=[str(network.id)])
self.assertIsNotNone(vm, "VM creation failed")
self.debug("Deployed VM in network: %s" % network.id)
vm_res = VirtualMachine.list(self.apiclient, id=vm.id)
self.assertEqual(
validateList(vm_res)[0], PASS,
"list vm returned invalid response")
vr_res = Router.list(self.apiclient, vpcid=vpc.id, listall="true")
self.assertEqual(
validateList(vr_res)[0], PASS, "list vrs failed for vpc")
vr_linklocal_ip = vr_res[0].linklocalip
result = get_process_status(self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
vr_linklocal_ip,
'lspci | grep "Ethernet controller"',
hypervisor=self.hypervisor)
self.assertEqual(
validateList(result)[0], PASS,
"We didn't find NICS with adapter type VMXNET3")
reg = re.compile("VMware VMXNET3")
count = 0
for line in result:
if reg.search(line):
count += 1
self.assertEqual(count, 3,
"Not all NICs on VR are of type VMXNET3")
except Exception as e:
self.fail("NIC creation failed for vpc tier with systemvm nic \
adapter type as Vmxnet3: %s" % e)
finally:
if reset:
self.updateConfigurAndRestart(
"vmware.systemvm.nic.device.type", nic_type)
return
def test_07_stop_start_VR_verify_ip_alias(self):
"""Reboot VR and verify ip alias
1.Deploy guest vm in new cidr
2.Verify ip alias creation
3.Stop and Start VR
4.Verify ip alias on VR
"""
list_router_response = list_routers(self.apiclient,
zoneid=self.zone.id,
listall=True)
self.assertEqual(isinstance(list_router_response, list), True,
"Check list response returns a valid list")
router = list_router_response[0]
hosts = list_hosts(self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid)
self.assertEqual(isinstance(hosts, list), True,
"Check list host returns a valid list")
host = hosts[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(router.state, 'Running',
"Check list router response for router state")
port = self.testdata['configurableData']['host']["publicport"]
username = self.testdata['configurableData']['host']["username"]
password = self.testdata['configurableData']['host']["password"]
# SSH to host so that host key is saved in first
# attempt
SshClient(host.ipaddress, port, username, password)
proc = "ip addr show eth0"
result = get_process_status(host.ipaddress, port, username, password,
router.linklocalip, proc)
res = str(result)
self.debug("ip alias configuration on VR: %s" % res)
self.assertNotEqual(
res.find(self.alias_ip) - 1, "ip alias is not created on VR eth0")
self.debug("Stopping VR")
Router.stop(
self.apiclient,
router.id,
)
self.debug("Starting VR")
Router.start(self.apiclient, router.id)
list_router_response = list_routers(self.apiclient,
zoneid=self.zone.id,
listall=True)
self.assertEqual(isinstance(list_router_response, list), True,
"Check list response returns a valid list")
router = list_router_response[0]
self.assertEqual(router.state, 'Running',
"Router is not in running state after reboot")
self.debug("VR is up and Running")
result = get_process_status(host.ipaddress, port, username, password,
router.linklocalip, proc)
res = str(result)
self.assertNotEqual(
res.find(self.alias_ip), -1,
"IP alias not present on VR after VR stop and start")
return
def test_04_cpvm_internals(self):
"""Test CPVM Internals"""
# Validate the following
# 1. test that telnet access on 8250 is available to
# the management server for the CPVM
# 2. No telnet access, test FAIL
# 3. Service cloud status should report cloud agent status to be
# running
list_cpvm_response = list_ssvms(self.apiclient,
systemvmtype='consoleproxy',
state='Running',
zoneid=self.zone.id)
self.assertEqual(isinstance(list_cpvm_response, list), True,
"Check list response returns a valid list")
cpvm = list_cpvm_response[0]
hosts = list_hosts(self.apiclient, id=cpvm.hostid)
self.assertEqual(isinstance(hosts, list), True,
"Check list response returns a valid list")
host = hosts[0]
try:
telnetlib.Telnet(str(self.apiclient.connection.mgtSvr), '8250')
self.logger.debug("Telnet management server (IP: %s)" %
self.apiclient.connection.mgtSvr)
except Exception as e:
self.fail("Telnet Access failed for %s: %s" %
(self.apiclient.connection.mgtSvr, e))
self.logger.debug("Checking cosmic-agent process status")
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(host.ipaddress, 22, host.user,
host.passwd, cpvm.linklocalip,
"systemctl status cosmic-agent")
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials to check router services"
)
res = str(result)
self.logger.debug("cosmic-agent Process status: %s" % res)
self.assertEqual(res.count("active (running)"), 1,
"Check cosmic-agent service is running or not")
linklocal_ip = None
# Check status of cosmic-agent service
try:
linklocal_ip = cpvm.linklocalip
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress, 22, host.user, host.passwd, cpvm.linklocalip,
"cat /etc/cosmic/agent/agent.properties | grep eth0ip= | cut -d= -f2"
)
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials to check router services"
)
res = result[0]
self.logger.debug("Cached Link Local IP: %s" % res)
self.assertEqual(
linklocal_ip, res,
"The cached Link Local should be the same as the current Link Local IP, but they are different! Current ==> %s; Cached ==> %s "
% (linklocal_ip, res))
return
def test_01_FTPModulesInVR(self):
"""
@desc: Verify FTP modules are loaded in VR of advance zone
step1 : create a VR in advance zone
step2: Verify FTP modules are there in created VR
"""
if self.zone.networktype == "Basic":
self.skipTest("This test can be run only in advance zone")
# create a virtual machine
vm = VirtualMachine.create(
self.api_client,
self.services["small"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.small_offering.id,
mode=self.services["mode"]
)
self.assertIsNotNone(vm, "Failed to deploy virtual machine")
self.cleanup.append(vm)
response = VirtualMachine.list(self.api_client, id=vm.id)
status = validateList(response)
self.assertEqual(
status[0],
PASS,
"list vm response returned invalid list")
list_router_response = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid
)
status = validateList(list_router_response)
self.assertEqual(
status[0], PASS, "Check list response returns a valid list")
router = list_router_response[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(
router.state,
'Running',
"Check list router response for router state"
)
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"lsmod | grep ftp",
hypervisor=self.hypervisor
)
else:
try:
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list host returns a valid list"
)
host = hosts[0]
result = get_process_status(
host.ipaddress,
22,
self.hostConfig["username"],
self.hostConfig["password"],
router.linklocalip,
"lsmod | grep ftp"
)
except Exception as e:
raise Exception("Exception raised in getting host\
credentials: %s " % e)
res = str(result)
self.debug("lsmod | grep ftp: %s" % res)
if "nf_nat_ftp" in res and "nf_conntrack_ftp" in res:
ismoduleinstalled = True
else:
ismoduleinstalled = False
self.assertEqual(
ismoduleinstalled,
True,
"nf_conntrack_ftp and nf_nat_ftp modules not installed on routers")
return
def test_04_userdata_service_on_alias_IP(self):
"""Deploy guest vm in new CIDR and verify userdata service on alias ip
1.Deploy guest vm in new cidr
2.Verify userdata service(apache2) listens on alias ip in VR
"""
list_router_response = list_routers(
self.apiclient,
zoneid=self.zone.id,
listall=True
)
self.assertEqual(
isinstance(list_router_response, list),
True,
"Check list response returns a valid list"
)
router = list_router_response[0]
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list host returns a valid list"
)
host = hosts[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(
router.state,
'Running',
"Check list router response for router state"
)
port = self.testdata['configurableData']['host']["publicport"]
username = self.testdata['configurableData']['host']["username"]
password = self.testdata['configurableData']['host']["password"]
# SSH to host so that host key is saved in first
# attempt
SshClient(host.ipaddress, port, username, password)
proc = "apache2"
result = get_process_status(
host.ipaddress,
port,
username,
password,
router.linklocalip,
"netstat -atnp | grep %s" % proc
)
res = str(result)
self.debug("userdata process status on VR: %s" % res)
self.assertNotEqual(
res.find(self.alias_ip + ":80 ")
- 1,
"password service is not running on alias ip"
)
return
def test_01_VRServiceFailureAlerting(self):
if self.zone.networktype == "Basic":
list_router_response = list_routers(self.apiclient, listall="true")
else:
list_router_response = list_routers(self.apiclient,
account=self.account.name,
domainid=self.account.domainid)
self.assertEqual(isinstance(list_router_response, list), True,
"Check list response returns a valid list")
router = list_router_response[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(router.state, 'Running',
"Check list router response for router state")
alertSubject = "Monitoring Service on VR " + router.name
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"service dnsmasq stop",
hypervisor=self.hypervisor)
else:
try:
hosts = list_hosts(self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid)
self.assertEqual(isinstance(hosts, list), True,
"Check list host returns a valid list")
host = hosts[0]
result = get_process_status(host.ipaddress, 22,
self.hostConfig["username"],
self.hostConfig["password"],
router.linklocalip,
"service apache2 stop")
except Exception as e:
raise Exception("Exception raised in getting host\
credentials: %s " % e)
res = str(result)
self.debug("apache process status: %s" % res)
configs = Configurations.list(self.apiclient,
name='router.alerts.check.interval')
# Set the value for one more minute than
# actual range to be on safer side
waitingPeriod = (int(configs[0].value) + 60) # in seconds
time.sleep(waitingPeriod)
# wait for (router.alerts.check.interval + 10) minutes meanwhile
# monitor service on VR starts the apache service (
# router.alerts.check.interval default value is
# 30minutes)
qresultset = self.dbclient.execute(
"select id from alert where subject like\
'%{0}%' ORDER BY id DESC LIMIT 1;".format(
str(alertSubject)))
self.assertNotEqual(len(qresultset), 0, "Check DB Query result set")
return
def test_03_RVR_Network_check_router_state(self):
""" Test redundant router internals """
self.logger.debug("Starting test_03_RVR_Network_check_router_state...")
hypervisor = self.testClient.getHypervisorInfo()
self.logger.debug("Creating Network Offering with default egress FALSE")
network_offering_egress_false = NetworkOffering.create(
self.apiclient,
self.services["nw_off_persistent_RVR_egress_false"],
conservemode=True
)
network_offering_egress_false.update(self.apiclient, state='Enabled')
self.logger.debug("Creating network with network offering: %s" % network_offering_egress_false.id)
network = Network.create(
self.apiclient,
self.services["network"],
accountid=self.account.name,
domainid=self.account.domainid,
networkofferingid=network_offering_egress_false.id,
zoneid=self.zone.id
)
self.logger.debug("Created network with ID: %s" % network.id)
networks = Network.list(
self.apiclient,
id=network.id,
listall=True
)
self.assertEqual(
isinstance(networks, list),
True,
"List networks should return a valid response for created network"
)
nw_response = networks[0]
self.logger.debug("Deploying VM in account: %s" % self.account.name)
virtual_machine = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
templateid=self.template.id,
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id,
networkids=[str(network.id)]
)
self.logger.debug("Deployed VM in network: %s" % network.id)
self.cleanup.insert(0, network_offering_egress_false)
self.cleanup.insert(0, network)
self.cleanup.insert(0, virtual_machine)
vms = VirtualMachine.list(
self.apiclient,
id=virtual_machine.id,
listall=True
)
self.assertEqual(
isinstance(vms, list),
True,
"List Vms should return a valid list"
)
vm = vms[0]
self.assertEqual(
vm.state,
"Running",
"VM should be in running state after deployment"
)
self.logger.debug("Listing routers for network: %s" % network.name)
routers = Router.list(
self.apiclient,
networkid=network.id,
listall=True
)
self.assertEqual(
isinstance(routers, list),
True,
"list router should return Master and backup routers"
)
self.assertEqual(
len(routers),
2,
"Length of the list router should be 2 (Backup & master)"
)
vals = ["MASTER", "BACKUP", "UNKNOWN"]
cnts = [0, 0, 0]
result = "UNKNOWN"
for router in routers:
if router.state == "Running":
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list host returns a valid list"
)
host = hosts[0]
if hypervisor.lower() in ('vmware', 'hyperv'):
result = str(get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"sh /opt/cloud/bin/checkrouter.sh ",
hypervisor=hypervisor
))
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = str(get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"sh /opt/cloud/bin/checkrouter.sh "
))
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials to\
check router services")
if result.count(vals[0]) == 1:
cnts[vals.index(vals[0])] += 1
if cnts[vals.index('MASTER')] != 1:
self.fail("No Master or too many master routers found %s" % cnts[vals.index('MASTER')])
return
def test_04_cpvm_internals(self):
"""Test CPVM Internals"""
# Validate the following
# 1. test that telnet access on 8250 is available to
# the management server for the CPVM
# 2. No telnet access, test FAIL
# 3. Service cloud status should report cloud agent status to be
# running
list_cpvm_response = list_ssvms(self.apiclient,
systemvmtype='consoleproxy',
state='Running',
zoneid=self.zone.id)
self.assertEqual(isinstance(list_cpvm_response, list), True,
"Check list response returns a valid list")
cpvm = list_cpvm_response[0]
hosts = list_hosts(self.apiclient, id=cpvm.hostid)
self.assertEqual(isinstance(hosts, list), True,
"Check list response returns a valid list")
host = hosts[0]
try:
telnetlib.Telnet(str(self.apiclient.connection.mgtSvr), '8250')
self.debug("Telnet management server (IP: %s)" %
self.apiclient.connection.mgtSvr)
except Exception as e:
self.fail("Telnet Access failed for %s: %s" %
(self.apiclient.connection.mgtSvr, e))
self.debug("Checking cloud process status")
if self.hypervisor.lower() in ('vmware', 'hyperv'):
# SSH into SSVMs is done via management server for Vmware and
# Hyper-V
result = get_process_status(self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
cpvm.privateip,
"service cloud status",
hypervisor=self.hypervisor)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(host.ipaddress, 22, host.user,
host.passwd, cpvm.linklocalip,
"service cloud status")
except KeyError:
self.skipTest("Marvin configuration has no host\
credentials to check router services")
res = str(result)
self.debug("Cloud Process status: %s" % res)
self.assertEqual(res.count("is running"), 1,
"Check cloud service is running or not")
linklocal_ip = None
# Check status of cloud service
if self.hypervisor.lower() in ('vmware', 'hyperv'):
# SSH into SSVMs is done via management server for Vmware and
# Hyper-V
linklocal_ip = cpvm.privateip
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
cpvm.privateip,
"cat /var/cache/cloud/cmdline | xargs | sed \"s/ /\\n/g\" | grep eth1ip= | sed \"s/\=/ /g\" | awk '{print $2}'",
hypervisor=self.hypervisor)
else:
try:
linklocal_ip = cpvm.linklocalip
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress, 22, host.user, host.passwd,
cpvm.linklocalip,
"cat /var/cache/cloud/cmdline | xargs | sed \"s/ /\\n/g\" | grep eth0ip= | sed \"s/\=/ /g\" | awk '{print $2}'"
)
except KeyError:
self.skipTest("Marvin configuration has no host\
credentials to check router services")
res = result[0]
self.debug("Cached Link Local IP: %s" % res)
self.assertEqual(
linklocal_ip, res,
"The cached Link Local should be the same as the current Link Local IP, but they are different! Current ==> %s; Cached ==> %s "
% (linklocal_ip, res))
return
def test_01_router_internal_basic(self):
"""Test router internal basic zone
"""
# Validate the following
# 1. Router only does dhcp
# 2. Verify that ports 67 (DHCP) and 53 (DNS) are open on UDP
# by checking status of dnsmasq process
# Find router associated with user account
if self.zone.networktype == "Basic":
list_router_response = list_routers(
self.apiclient,
listall="true"
)
else:
list_router_response = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid
)
self.assertEqual(
isinstance(list_router_response, list),
True,
"Check list response returns a valid list"
)
router = list_router_response[0]
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list host returns a valid list"
)
host = hosts[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(
router.state,
'Running',
"Check list router response for router state"
)
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"service dnsmasq status",
hypervisor=self.hypervisor
)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"service dnsmasq status"
)
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials to\
check router services")
res = str(result)
self.debug("Dnsmasq process status: %s" % res)
self.assertEqual(
res.count("running"),
1,
"Check dnsmasq service is running or not"
)
return
def test_03_ssvm_internals(self):
"""Test SSVM Internals"""
# Validate the following
# 1. The SSVM check script should not return any
# WARN|ERROR|FAIL messages
# 2. If you are unable to login to the SSVM with the signed key
# then test is deemed a failure
# 3. There should be only one ""cloud"" process running within the SSVM
# 4. If no process is running/multiple process are running
# then the test is a failure
list_ssvm_response = list_ssvms(self.apiclient,
systemvmtype='secondarystoragevm',
state='Running',
zoneid=self.zone.id)
self.assertEqual(isinstance(list_ssvm_response, list), True,
"Check list response returns a valid list")
ssvm = list_ssvm_response[0]
hosts = list_hosts(self.apiclient, id=ssvm.hostid)
self.assertEqual(isinstance(hosts, list), True,
"Check list response returns a valid list")
host = hosts[0]
self.logger.debug("Running SSVM check script")
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress, 22, host.user, host.passwd, ssvm.linklocalip,
"/opt/cosmic/agent/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL"
)
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials to check router services"
)
res = str(result)
self.logger.debug("SSVM script output: %s" % res)
self.assertEqual(res.count("ERROR"), 1, "Check for Errors in tests")
self.assertEqual(res.count("WARNING"), 1,
"Check for warnings in tests")
# Check status of cosmic-agent service
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(host.ipaddress, 22, host.user,
host.passwd, ssvm.linklocalip,
"systemctl status cosmic-agent")
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials to check router services"
)
res = str(result)
self.logger.debug("cosmic-agent Process status: %s" % res)
# cloud.com service (type=secstorage) is running: process id: 2346
self.assertEqual(res.count("active (running)"), 1,
"Check cosmic-agent service is running or not")
linklocal_ip = None
# Check status of cloud service
try:
linklocal_ip = ssvm.linklocalip
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress, 22, host.user, host.passwd, ssvm.linklocalip,
"cat /etc/cosmic/agent/agent.properties | grep eth0ip= | cut -d= -f2"
)
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials to check router services"
)
res = result[0]
self.logger.debug("Cached Link Local IP: %s" % res)
self.assertEqual(
linklocal_ip, res,
"The cached Link Local should be the same as the current Link Local IP, but they are different! Current ==> %s; Cached ==> %s "
% (linklocal_ip, res))
return
def test_iptable_rules(self):
"""Test iptable rules in case we have IP associated with a network which is in
different pubic IP range from that of public IP range that has source NAT IP.
When IP is associated we should see a rule '-i eth3 -o eth0 -m state --state RELATED,ESTABLISHED -j ACCEPT' in FORWARD table.
When IP is dis-associated we should see a rule in the FORWARD table is deleted.
"""
# Validate the following:
# 1. Create a new public IP range and dedicate to a account
# 2. Acquire a IP from new public range
# 3. Create a firewall rule to open up the port, so that IP is associated with network
# 5. Login to VR and verify routing tables, there should be Table_eth3
# 6. Delete firewall rule, since its last IP, routing table Table_eth3 should be deleted
self.services["extrapubliciprange"]["zoneid"] = self.services["zoneid"]
self.public_ip_range = PublicIpRange.create(
self.apiclient,
self.services["extrapubliciprange"]
)
self.cleanup.append(self.public_ip_range)
logger.debug("Dedicating Public IP range to the account");
dedicate_public_ip_range_response = PublicIpRange.dedicate(
self.apiclient,
self.public_ip_range.vlan.id,
account=self.account.name,
domainid=self.account.domainid
)
ip_address = PublicIPAddress.create(
self.apiclient,
self.account.name,
self.zone.id,
self.account.domainid,
self.services["virtual_machine"]
)
self.cleanup.append(ip_address)
# Check if VM is in Running state before creating NAT and firewall rules
vm_response = VirtualMachine.list(
self.apiclient,
id=self.virtual_machine.id
)
self.assertEqual(
isinstance(vm_response, list),
True,
"Check list VM returns a valid list"
)
self.assertNotEqual(
len(vm_response),
0,
"Check Port Forwarding Rule is created"
)
self.assertEqual(
vm_response[0].state,
'Running',
"VM state should be Running before creating a NAT rule."
)
# Open up firewall port for SSH
firewall_rule = FireWallRule.create(
self.apiclient,
ipaddressid=ip_address.ipaddress.id,
protocol=self.services["natrule"]["protocol"],
cidrlist=['0.0.0.0/0'],
startport=self.services["natrule"]["publicport"],
endport=self.services["natrule"]["publicport"]
)
self.cleanup.append(firewall_rule)
# Get the router details associated with account
routers = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
)
router = routers[0]
if (self.hypervisor.lower() == 'vmware'
or self.hypervisor.lower() == 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
'iptables -t filter -L FORWARD -v',
hypervisor=self.hypervisor
)
else:
hosts = list_hosts(
self.apiclient,
id=router.hostid,
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check for list hosts response return valid data"
)
host = hosts[0]
host.user = self.hostConfig['username']
host.passwd = self.hostConfig['password']
try:
result = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
'iptables -t filter -L FORWARD -v'
)
except KeyError:
self.skipTest(
"Provide a marvin config file with host\
credentials to run %s" %
self._testMethodName)
logger.debug("iptables -t filter -L FORWARD -v: %s" % result)
res = str(result)
self.assertEqual(
res.count("eth3 eth0 anywhere anywhere state RELATED,ESTABLISHED"),
1,
"Check to ensure there is a iptable rule to accept the RELATED,ESTABLISHED traffic"
)
firewall_rule.delete(self.apiclient)
self.cleanup.remove(firewall_rule)
def test_03_ssvm_internals(self):
"""Test SSVM Internals"""
# Validate the following
# 1. The SSVM check script should not return any
# WARN|ERROR|FAIL messages
# 2. If you are unable to login to the SSVM with the signed key
# then test is deemed a failure
# 3. There should be only one ""cloud"" process running within the SSVM
# 4. If no process is running/multiple process are running
# then the test is a failure
list_ssvm_response = list_ssvms(
self.apiclient,
systemvmtype='secondarystoragevm',
state='Running',
zoneid=self.zone.id
)
self.assertEqual(
isinstance(list_ssvm_response, list),
True,
"Check list response returns a valid list"
)
ssvm = list_ssvm_response[0]
hosts = list_hosts(
self.apiclient,
id=ssvm.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list response returns a valid list"
)
host = hosts[0]
self.debug("Running SSVM check script")
if self.hypervisor.lower() in ('vmware', 'hyperv'):
# SSH into SSVMs is done via management server for Vmware and
# Hyper-V
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
ssvm.privateip,
"/usr/local/cloud/systemvm/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL",
hypervisor=self.hypervisor)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
ssvm.linklocalip,
"/usr/local/cloud/systemvm/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL")
except KeyError:
self.skipTest(
"Marvin configuration has no host\
credentials to check router services")
res = str(result)
self.debug("SSVM script output: %s" % res)
self.assertEqual(
res.count("ERROR"),
1,
"Check for Errors in tests"
)
self.assertEqual(
res.count("WARNING"),
1,
"Check for warnings in tests"
)
# Check status of cloud service
if self.hypervisor.lower() in ('vmware', 'hyperv'):
# SSH into SSVMs is done via management server for Vmware and
# Hyper-V
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
ssvm.privateip,
"service cloud status",
hypervisor=self.hypervisor
)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
ssvm.linklocalip,
"service cloud status"
)
except KeyError:
self.skipTest(
"Marvin configuration has no host\
credentials to check router services")
res = str(result)
self.debug("Cloud Process status: %s" % res)
# cloud.com service (type=secstorage) is running: process id: 2346
self.assertEqual(
res.count("is running"),
1,
"Check cloud service is running or not"
)
return
def test_01_create_tier_Vmxnet3(self):
"""
Test to create vpc tier with nic type as Vmxnet3
#1.Set global setting parameter "vmware.systemvm.nic.device.type"
to "Vmxnet3"
#2.Create VPC
#3.Create one tier
#4.Deploy one guest vm in the tier created in step3
"""
if self.hypervisor.lower() not in ['vmware']:
self.skipTest("This test can only run on vmware setup")
nic_types = Configurations.list(
self.apiclient,
name="vmware.systemvm.nic.device.type"
)
self.assertEqual(validateList(nic_types)[0], PASS, "Invalid list config")
nic_type = nic_types[0].value
reset = False
if nic_type.lower() != "vmxnet3":
self.updateConfigurAndRestart("vmware.systemvm.nic.device.type", "Vmxnet3")
reset = True
self.services["vpc"]["cidr"] = "10.1.1.1/16"
self.debug("creating a VPC network in the account: %s" %
self.account.name)
try:
vpc = VPC.create(
self.apiclient,
self.services["vpc"],
vpcofferingid=self.vpc_off.id,
zoneid=self.zone.id,
account=self.account.name,
domainid=self.account.domainid
)
vpc_res = VPC.list(self.apiclient, id=vpc.id)
self.assertEqual(validateList(vpc_res)[0], PASS, "Invalid response from listvpc")
self.network_offering = NetworkOffering.create(
self.apiclient,
self.services["network_offering"],
conservemode=False
)
# Enable Network offering
self.network_offering.update(self.apiclient, state='Enabled')
self.cleanup.append(self.network_offering)
gateway = vpc.cidr.split('/')[0]
# Split the cidr to retrieve gateway
# for eg. cidr = 10.0.0.1/24
# Gateway = 10.0.0.1
# Creating network using the network offering created
self.debug("Creating network with network offering: %s" %
self.network_offering.id)
network = Network.create(
self.apiclient,
self.services["network"],
accountid=self.account.name,
domainid=self.account.domainid,
networkofferingid=self.network_offering.id,
zoneid=self.zone.id,
gateway=gateway,
vpcid=vpc.id
)
self.debug("Created network with ID: %s" % network.id)
vm = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id,
networkids=[str(network.id)]
)
self.assertIsNotNone(vm, "VM creation failed")
self.debug("Deployed VM in network: %s" % network.id)
vm_res = VirtualMachine.list(self.apiclient, id=vm.id)
self.assertEqual(
validateList(vm_res)[0],
PASS,
"list vm returned invalid response"
)
vr_res = Router.list(
self.apiclient,
vpcid=vpc.id,
listall="true"
)
self.assertEqual(validateList(vr_res)[0], PASS, "list vrs failed for vpc")
vr_linklocal_ip = vr_res[0].linklocalip
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
vr_linklocal_ip,
'lspci | grep "Ethernet controller"',
hypervisor=self.hypervisor
)
self.assertEqual(
validateList(result)[0],
PASS,
"We didn't find NICS with adapter type VMXNET3"
)
reg = re.compile("VMware VMXNET3")
count = 0
for line in result:
if reg.search(line):
count += 1
self.assertEqual(
count,
3,
"Not all NICs on VR are of type VMXNET3"
)
except Exception as e:
self.fail("NIC creation failed for vpc tier with systemvm nic \
adapter type as Vmxnet3: %s" % e)
finally:
if reset:
self.updateConfigurAndRestart("vmware.systemvm.nic.device.type", nic_type)
return
def test_01_RouterStopCreateFW(self):
"""Test router stop create Firewall rule
"""
# validate the following
# 1. 1. listFirewallRules (filter by ipaddressid of sourcenat)
# 2. rule should be for ports 1-600 and in state=Active
# (optional backend)
# 3. verify on router using iptables -t nat -nvx if rules are applied
# Get the router details associated with account
routers = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
)
self.assertEqual(isinstance(routers, list), True,
"Check for list routers response return valid data")
self.assertNotEqual(len(routers), 0, "Check list router response")
router = routers[0]
self.debug("Stopping the router: %s" % router.id)
# Stop the router
cmd = stopRouter.stopRouterCmd()
cmd.id = router.id
self.apiclient.stopRouter(cmd)
routers = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
)
self.assertEqual(isinstance(routers, list), True,
"Check for list routers response return valid data")
router = routers[0]
self.assertEqual(router.state, 'Stopped',
"Check list router response for router state")
public_ips = list_publicIP(self.apiclient,
account=self.account.name,
domainid=self.account.domainid)
self.assertEqual(
isinstance(public_ips, list), True,
"Check for list public IP response return valid data")
public_ip = public_ips[0]
# Create Firewall rule with configurations from settings file
fw_rule = FireWallRule.create(
self.apiclient,
ipaddressid=public_ip.id,
protocol='TCP',
cidrlist=[self.services["fwrule"]["cidr"]],
startport=self.services["fwrule"]["startport"],
endport=self.services["fwrule"]["endport"])
self.debug("Created firewall rule: %s" % fw_rule.id)
self.debug("Starting the router: %s" % router.id)
# Start the router
cmd = startRouter.startRouterCmd()
cmd.id = router.id
self.apiclient.startRouter(cmd)
routers = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid,
)
self.assertEqual(isinstance(routers, list), True,
"Check for list routers response return valid data")
router = routers[0]
self.assertEqual(router.state, 'Running',
"Check list router response for router state")
# After Router start, FW rule should be in Active state
fw_rules = list_firewall_rules(
self.apiclient,
id=fw_rule.id,
)
self.assertEqual(isinstance(fw_rules, list), True,
"Check for list FW rules response return valid data")
self.assertEqual(fw_rules[0].state, 'Active',
"Check list load balancing rules")
self.assertEqual(fw_rules[0].startport,
str(self.services["fwrule"]["startport"]),
"Check start port of firewall rule")
self.assertEqual(fw_rules[0].endport,
str(self.services["fwrule"]["endport"]),
"Check end port of firewall rule")
# For DNS and DHCP check 'dnsmasq' process status
if (self.hypervisor.lower() == 'vmware'
or self.hypervisor.lower() == 'hyperv'):
result = get_process_status(self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
'iptables -t nat -L',
hypervisor=self.hypervisor)
else:
hosts = list_hosts(
self.apiclient,
id=router.hostid,
)
self.assertEqual(
isinstance(hosts, list), True,
"Check for list hosts response return valid data")
host = hosts[0]
host.user = self.services["configurableData"]["host"]["username"]
host.passwd = self.services["configurableData"]["host"]["password"]
try:
result = get_process_status(host.ipaddress, 22, host.user,
host.passwd, router.linklocalip,
'iptables -t nat -L')
except KeyError:
self.skipTest("Provide a marvin config file with host\
credentials to run %s" % self._testMethodName)
self.debug("iptables -t nat -L: %s" % result)
self.debug("Public IP: %s" % public_ip.ipaddress)
res = str(result)
self.assertEqual(res.count(str(public_ip.ipaddress)), 1,
"Check public IP address")
return
def test_01_router_internal_basic(self):
"""Test router internal basic zone
"""
# Validate the following
# 1. Router only does dhcp
# 2. Verify that ports 67 (DHCP) and 53 (DNS) are open on UDP
# by checking status of dnsmasq process
# Find router associated with user account
if self.zone.networktype == "Basic":
list_router_response = list_routers(
self.apiclient,
listall="true"
)
else:
list_router_response = list_routers(
self.apiclient,
account=self.account.name,
domainid=self.account.domainid
)
self.assertEqual(
isinstance(list_router_response, list),
True,
"Check list response returns a valid list"
)
router = list_router_response[0]
hosts = list_hosts(
self.apiclient,
zoneid=router.zoneid,
type='Routing',
state='Up',
id=router.hostid
)
self.assertEqual(
isinstance(hosts, list),
True,
"Check list host returns a valid list"
)
host = hosts[0]
self.debug("Router ID: %s, state: %s" % (router.id, router.state))
self.assertEqual(
router.state,
'Running',
"Check list router response for router state"
)
if self.hypervisor.lower() in ('vmware', 'hyperv'):
result = get_process_status(
self.apiclient.connection.mgtSvr,
22,
self.apiclient.connection.user,
self.apiclient.connection.passwd,
router.linklocalip,
"service dnsmasq status",
hypervisor=self.hypervisor
)
else:
try:
host.user, host.passwd = get_host_credentials(
self.config, host.ipaddress)
result = get_process_status(
host.ipaddress,
22,
host.user,
host.passwd,
router.linklocalip,
"service dnsmasq status"
)
except KeyError:
self.skipTest(
"Marvin configuration has no host credentials to\
check router services")
res = str(result)
self.debug("Dnsmasq process status: %s" % res)
self.assertEqual(
res.count("running"),
1,
"Check dnsmasq service is running or not"
)
return
