def stop_master_router(self, vpc): self.logger.debug("Stopping Master Router of VPC '%s'...", vpc.name) routers = list_routers(self.api_client, domainid=self.domain.id, account=self.account.name, vpcid=vpc.id) for router in routers: if router.redundantstate == 'MASTER': cmd = stopRouter.stopRouterCmd() cmd.id = router.id # This will not fail-over gracefully and cause a ~3.6sec downtime # cmd.forced = 'true' self.api_client.stopRouter(cmd) break for router in routers: if router.state == 'Running': hosts = list_hosts(self.api_client, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid) self.assertTrue(isinstance(hosts, list)) host = next(iter(hosts or []), None) try: host.user, host.passwd = get_host_credentials(self.config, host.name) get_process_status(host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "sh /opt/cosmic/router/scripts/checkrouter.sh ") except KeyError as e: raise Exception("Exception: %s" % e) self.logger.debug("Master Router of VPC '%s' stopped", vpc.name)
def test_01_router_internal_basic(self): """Test router internal basic zone """ # Validate the following # 1. Router only does dhcp # 2. Verify that ports 67 (DHCP) and 53 (DNS) are open on UDP # by checking status of dnsmasq process # Find router associated with user account if self.zone.networktype == "Basic": list_router_response = list_routers(self.apiclient, listall="true") else: list_router_response = list_routers(self.apiclient, account=self.account.name, domainid=self.account.domainid) self.assertEqual(isinstance(list_router_response, list), True, "Check list response returns a valid list") router = list_router_response[0] hosts = list_hosts(self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid) self.assertEqual(isinstance(hosts, list), True, "Check list host returns a valid list") host = hosts[0] self.debug("Router ID: %s, state: %s" % (router.id, router.state)) self.assertEqual(router.state, 'Running', "Check list router response for router state") if self.hypervisor.lower() in ('vmware', 'hyperv'): result = get_process_status(self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, router.linklocalip, "service dnsmasq status", hypervisor=self.hypervisor) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status(host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "service dnsmasq status") except KeyError: self.skipTest("Marvin configuration has no host credentials to\ check router services") res = str(result) self.debug("Dnsmasq process status: %s" % res) self.assertEqual(res.count("running"), 1, "Check dnsmasq service is running or not") return
def check_routers_state(self,count=2, status_to_check="MASTER", expected_count=1, showall=False): vals = ["MASTER", "BACKUP", "UNKNOWN"] cnts = [0, 0, 0] self.wait_for_vrrp() result = "UNKNOWN" self.query_routers(count, showall) for router in self.routers: if router.state == "Running": hosts = list_hosts( self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list host returns a valid list" ) host = hosts[0] if self.hypervisor.lower() in ('vmware', 'hyperv'): result = str(get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, router.linklocalip, "sh /opt/cloud/bin/checkrouter.sh ", hypervisor=self.hypervisor )) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = str(get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "sh /opt/cloud/bin/checkrouter.sh " )) except KeyError: self.skipTest( "Marvin configuration has no host credentials to\ check router services") if result.count(status_to_check) == 1: cnts[vals.index(status_to_check)] += 1 if cnts[vals.index(status_to_check)] != expected_count: self.fail("Expected '%s' routers at state '%s', but found '%s'!" % (expected_count, status_to_check, cnts[vals.index(status_to_check)]))
def check_master_status(self,count=2, showall=False): vals = ["MASTER", "BACKUP", "UNKNOWN"] cnts = [0, 0, 0] result = "UNKNOWN" self.query_routers(count, showall) for router in self.routers: if router.state == "Running": hosts = list_hosts( self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list host returns a valid list" ) host = hosts[0] if self.hypervisor.lower() in ('vmware', 'hyperv'): result = str(get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, router.linklocalip, "sh /opt/cloud/bin/checkrouter.sh ", hypervisor=self.hypervisor )) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = str(get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "sh /opt/cloud/bin/checkrouter.sh " )) except KeyError: self.skipTest( "Marvin configuration has no host credentials to\ check router services") if result.count(vals[0]) == 1: cnts[vals.index(vals[0])] += 1 if cnts[vals.index('MASTER')] != 1: self.fail("No Master or too many master routers found %s" % cnts[vals.index('MASTER')])
def check_routers_interface(self, count=2, interface_to_check="eth1", expected_exists=True, showall=False): result = "" self.query_routers(count, showall) for router in self.routers: if router.state == "Running": hosts = list_hosts(self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid) self.assertEqual(isinstance(hosts, list), True, "Check list host returns a valid list") host = hosts[0] try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = str( get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "ip a | grep %s | grep state | awk '{print $9;}'" % interface_to_check)) except KeyError: self.skipTest( "Marvin configuration has no host credentials to check router services" ) if expected_exists: if (result.count("UP") == 1) or (result.count("DOWN") == 1): self.logger.debug( "Expected interface '%s' to exist and it does!" % interface_to_check) else: self.fail( "Expected interface '%s' to exist, but it didn't!" % interface_to_check) else: if (result.count("UP") == 1) or (result.count("DOWN") == 1): self.fail( "Expected interface '%s' to not exist, but it did!" % interface_to_check) else: self.logger.debug( "Expected interface '%s' to not exist, and it didn't!" % interface_to_check)
def check_routers_state(self, count=2, status_to_check="PRIMARY", expected_count=1, showall=False): vals = ["PRIMARY", "BACKUP", "UNKNOWN", "FAULT"] cnts = [0, 0, 0, 0] self.wait_for_vrrp() result = "UNKNOWN" self.query_routers(count, showall) for router in self.routers: if router.state == "Running": hosts = list_hosts(self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid) self.assertEqual(isinstance(hosts, list), True, "Check list host returns a valid list") host = hosts[0] if self.hypervisor.lower() in ('vmware', 'hyperv'): result = str( get_process_status(self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, router.linklocalip, "sh /opt/cloud/bin/checkrouter.sh ", hypervisor=self.hypervisor)) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = str( get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "sh /opt/cloud/bin/checkrouter.sh ")) except KeyError: self.skipTest( "Marvin configuration has no host credentials to\ check router services") if result.count(status_to_check) == 1: cnts[vals.index(status_to_check)] += 1 if cnts[vals.index(status_to_check)] != expected_count: self.fail("Expected '%s' routers at state '%s', but found '%s'!" % (expected_count, status_to_check, cnts[vals.index(status_to_check)]))
def get_router_host(self, router): self.assertEqual(router.state, 'Running', "Check list router response for router state") hosts = list_hosts(self.apiclient, id=router.hostid) self.assertEqual(isinstance(hosts, list), True, "Check for list hosts response return valid data") host = hosts[0] if host.hypervisor.lower() not in "kvm": return host.user, host.password = get_host_credentials( self.config, host.ipaddress) host.port = 22 return host
def getRouterProcessStatus(self, router, cmd): if router.id not in self.routerDetailsMap or self.routerDetailsMap[router.id] is None: connect_ip = self.apiclient.connection.mgtSvr connect_user = self.apiclient.connection.user connect_passwd = self.apiclient.connection.passwd hypervisor = self.hypervisor if self.hypervisor.lower() not in ('vmware', 'hyperv'): hosts = Host.list( self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list host returns a valid list" ) host = hosts[0] connect_ip = host.ipaddress hypervisor = None try: connect_user, connect_passwd= get_host_credentials( self.config, host.ipaddress) except KeyError: self.skipTest( "Marvin configuration has no host credentials to\ check router services") details = {} details['connect_ip'] = connect_ip details['connect_user'] = connect_user details['connect_passwd'] = connect_passwd details['hypervisor'] = hypervisor self.routerDetailsMap[router.id] = details result = get_process_status( self.routerDetailsMap[router.id]['connect_ip'], 22, self.routerDetailsMap[router.id]['connect_user'], self.routerDetailsMap[router.id]['connect_passwd'], router.linklocalip, cmd, hypervisor=self.routerDetailsMap[router.id]['hypervisor'] ) self.assertTrue(type(result) == list, "%s on router %s returned invalid result" % (cmd, router.id)) result = '\n'.join(result) return result
def check_routers_interface(self,count=2, interface_to_check="eth1", expected_exists=True, showall=False): result = "" self.query_routers(count, showall) for router in self.routers: if router.state == "Running": hosts = list_hosts( self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list host returns a valid list" ) host = hosts[0] try: host.user, host.passwd = get_host_credentials(self.config, host.ipaddress) result = str(get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "ip a | grep %s | grep state | awk '{print $9;}'" % interface_to_check )) except KeyError: self.skipTest("Marvin configuration has no host credentials to check router services") if expected_exists: if (result.count("UP") == 1) or (result.count("DOWN") == 1): self.logger.debug("Expected interface '%s' to exist and it does!" % interface_to_check) else: self.fail("Expected interface '%s' to exist, but it didn't!" % interface_to_check) else: if (result.count("UP") == 1) or (result.count("DOWN") == 1): self.fail("Expected interface '%s' to not exist, but it did!" % interface_to_check) else: self.logger.debug("Expected interface '%s' to not exist, and it didn't!" % interface_to_check)
def stop_master_router(self, vpc): self.logger.debug("Stopping Master Router of VPC '%s'...", vpc.name) routers = list_routers(self.api_client, domainid=self.domain.id, account=self.account.name, vpcid=vpc.id) for router in routers: if router.redundantstate == 'MASTER': cmd = stopRouter.stopRouterCmd() cmd.id = router.id cmd.forced = 'true' self.api_client.stopRouter(cmd) break routers = list_routers(self.api_client, domainid=self.domain.id, account=self.account.name, vpcid=vpc.id) for router in routers: if router.state == 'Running': hosts = list_hosts(self.api_client, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid) self.assertTrue(isinstance(hosts, list)) host = next(iter(hosts or []), None) try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "sh /opt/cosmic/router/scripts/checkrouter.sh ") except KeyError as e: raise Exception("Exception: %s" % e) self.logger.debug("Master Router of VPC '%s' stopped", vpc.name)
def verify_network_rules(self, vm_id): virtual_machine = VirtualMachine.list(self.apiclient, id=vm_id) vm = virtual_machine[0] hosts = list_hosts(self.apiclient, id=vm.hostid) host = hosts[0] if host.hypervisor.lower() not in "kvm": return host.user, host.password = get_host_credentials( self.config, host.ipaddress) for nic in vm.nic: secips = "" if len(nic.secondaryip) > 0: for secip in nic.secondaryip: secips += secip.ipaddress + ";" command = "/usr/share/cloudstack-common/scripts/vm/network/security_group.py verify_network_rules --vmname %s --vmip %s --vmmac %s --nicsecips '%s'" % ( vm.instancename, nic.ipaddress, nic.macaddress, secips) self.logger.debug("Executing command '%s' in host %s" % (command, host.ipaddress)) result = execute_command_in_host(host.ipaddress, 22, host.user, host.password, command) if len(result) > 0: self.fail( "The iptables/ebtables rules for nic %s on vm %s on host %s are not correct" % (nic.ipaddress, vm.instancename, host.name))
def test_01_RouterStopCreateFW(self): """Test router stop create Firewall rule """ # validate the following # 1. 1. listFirewallRules (filter by ipaddressid of sourcenat) # 2. rule should be for ports 1-600 and in state=Active # (optional backend) # 3. verify on router using iptables -t nat -nvx if rules are applied # Get the router details associated with account routers = list_routers( self.apiclient, account=self.account.name, domainid=self.account.domainid, ) self.assertEqual( isinstance(routers, list), True, "Check for list routers response return valid data" ) self.assertNotEqual( len(routers), 0, "Check list router response" ) router = routers[0] self.debug("Stopping the router: %s" % router.id) # Stop the router cmd = stopRouter.stopRouterCmd() cmd.id = router.id self.apiclient.stopRouter(cmd) routers = list_routers( self.apiclient, account=self.account.name, domainid=self.account.domainid, ) self.assertEqual( isinstance(routers, list), True, "Check for list routers response return valid data" ) router = routers[0] self.assertEqual( router.state, 'Stopped', "Check list router response for router state" ) public_ips = list_publicIP( self.apiclient, account=self.account.name, domainid=self.account.domainid ) self.assertEqual( isinstance(public_ips, list), True, "Check for list public IP response return valid data" ) public_ip = public_ips[0] # Create Firewall rule with configurations from settings file fw_rule = FireWallRule.create( self.apiclient, ipaddressid=public_ip.id, protocol='TCP', cidrlist=[self.services["fw_rule"]["cidr"]], startport=self.services["fw_rule"]["startport"], endport=self.services["fw_rule"]["endport"] ) self.debug("Created firewall rule: %s" % fw_rule.id) self.debug("Starting the router: %s" % router.id) # Start the router cmd = startRouter.startRouterCmd() cmd.id = router.id self.apiclient.startRouter(cmd) routers = list_routers( self.apiclient, account=self.account.name, domainid=self.account.domainid, ) self.assertEqual( isinstance(routers, list), True, "Check for list routers response return valid data" ) router = routers[0] self.assertEqual( router.state, 'Running', "Check list router response for router state" ) # After Router start, FW rule should be in Active state fw_rules = list_firewall_rules( self.apiclient, id=fw_rule.id, ) self.assertEqual( isinstance(fw_rules, list), True, "Check for list FW rules response return valid data" ) self.assertEqual( fw_rules[0].state, 'Active', "Check list load balancing rules" ) self.assertEqual( fw_rules[0].startport, str(self.services["fw_rule"]["startport"]), "Check start port of firewall rule" ) self.assertEqual( fw_rules[0].endport, str(self.services["fw_rule"]["endport"]), "Check end port of firewall rule" ) # For DNS and DHCP check 'dnsmasq' process status if (self.hypervisor.lower() == 'vmware' or self.hypervisor.lower() == 'hyperv'): result = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, router.linklocalip, 'iptables -t nat -L', hypervisor=self.hypervisor ) else: hosts = list_hosts( self.apiclient, id=router.hostid, ) self.assertEqual( isinstance(hosts, list), True, "Check for list hosts response return valid data" ) host = hosts[0] host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) try: result = get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, 'iptables -t nat -L' ) except KeyError: self.skipTest( "Provide a marvin config file with host\ credentials to run %s" % self._testMethodName) self.debug("iptables -t nat -L: %s" % result) self.debug("Public IP: %s" % public_ip.ipaddress) res = str(result) self.assertEqual( res.count(str(public_ip.ipaddress)), 1, "Check public IP address" ) return
def test_03_ssvm_internals(self): """Test SSVM Internals""" # Validate the following # 1. The SSVM check script should not return any # WARN|ERROR|FAIL messages # 2. If you are unable to login to the SSVM with the signed key # then test is deemed a failure # 3. There should be only one ""cloud"" process running within the SSVM # 4. If no process is running/multiple process are running # then the test is a failure list_ssvm_response = list_ssvms(self.apiclient, systemvmtype='secondarystoragevm', state='Running', zoneid=self.zone.id) self.assertEqual(isinstance(list_ssvm_response, list), True, "Check list response returns a valid list") ssvm = list_ssvm_response[0] hosts = list_hosts(self.apiclient, id=ssvm.hostid) self.assertEqual(isinstance(hosts, list), True, "Check list response returns a valid list") host = hosts[0] self.debug("Running SSVM check script") if self.hypervisor.lower() in ('vmware', 'hyperv'): # SSH into SSVMs is done via management server for Vmware and # Hyper-V result = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, ssvm.privateip, "/usr/local/cloud/systemvm/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL", hypervisor=self.hypervisor) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, ssvm.linklocalip, "/usr/local/cloud/systemvm/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL" ) except KeyError: self.skipTest("Marvin configuration has no host\ credentials to check router services") res = str(result) self.debug("SSVM script output: %s" % res) self.assertEqual(res.count("ERROR"), 1, "Check for Errors in tests") self.assertEqual(res.count("WARNING"), 1, "Check for warnings in tests") # Check status of cloud service if self.hypervisor.lower() in ('vmware', 'hyperv'): # SSH into SSVMs is done via management server for Vmware and # Hyper-V result = get_process_status(self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, ssvm.privateip, "service cloud status", hypervisor=self.hypervisor) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status(host.ipaddress, 22, host.user, host.passwd, ssvm.linklocalip, "service cloud status") except KeyError: self.skipTest("Marvin configuration has no host\ credentials to check router services") res = str(result) self.debug("Cloud Process status: %s" % res) # cloud.com service (type=secstorage) is running: process id: 2346 self.assertEqual(res.count("is running"), 1, "Check cloud service is running or not") linklocal_ip = None # Check status of cloud service if self.hypervisor.lower() in ('vmware', 'hyperv'): # SSH into SSVMs is done via management server for Vmware and # Hyper-V linklocal_ip = ssvm.privateip result = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, ssvm.privateip, "cat /var/cache/cloud/cmdline | xargs | sed \"s/ /\\n/g\" | grep eth1ip= | sed \"s/\=/ /g\" | awk '{print $2}'", hypervisor=self.hypervisor) else: try: linklocal_ip = ssvm.linklocalip host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, ssvm.linklocalip, "cat /var/cache/cloud/cmdline | xargs | sed \"s/ /\\n/g\" | grep eth0ip= | sed \"s/\=/ /g\" | awk '{print $2}'" ) except KeyError: self.skipTest("Marvin configuration has no host\ credentials to check router services") res = result[0] self.debug("Cached Link Local IP: %s" % res) self.assertEqual( linklocal_ip, res, "The cached Link Local should be the same as the current Link Local IP, but they are different! Current ==> %s; Cached ==> %s " % (linklocal_ip, res)) return
def test_04_cpvm_internals(self): """Test CPVM Internals""" # Validate the following # 1. test that telnet access on 8250 is available to # the management server for the CPVM # 2. No telnet access, test FAIL # 3. Service cloud status should report cloud agent status to be # running list_cpvm_response = list_ssvms(self.apiclient, systemvmtype='consoleproxy', state='Running', zoneid=self.zone.id) self.assertEqual(isinstance(list_cpvm_response, list), True, "Check list response returns a valid list") cpvm = list_cpvm_response[0] hosts = list_hosts(self.apiclient, id=cpvm.hostid) self.assertEqual(isinstance(hosts, list), True, "Check list response returns a valid list") host = hosts[0] try: telnetlib.Telnet(str(self.apiclient.connection.mgtSvr), '8250') self.logger.debug("Telnet management server (IP: %s)" % self.apiclient.connection.mgtSvr) except Exception as e: self.fail("Telnet Access failed for %s: %s" % (self.apiclient.connection.mgtSvr, e)) self.logger.debug("Checking cosmic-agent process status") try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status(host.ipaddress, 22, host.user, host.passwd, cpvm.linklocalip, "systemctl status cosmic-agent") except KeyError: self.skipTest( "Marvin configuration has no host credentials to check router services" ) res = str(result) self.logger.debug("cosmic-agent Process status: %s" % res) self.assertEqual(res.count("active (running)"), 1, "Check cosmic-agent service is running or not") linklocal_ip = None # Check status of cosmic-agent service try: linklocal_ip = cpvm.linklocalip host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, cpvm.linklocalip, "cat /etc/cosmic/agent/agent.properties | grep eth0ip= | cut -d= -f2" ) except KeyError: self.skipTest( "Marvin configuration has no host credentials to check router services" ) res = result[0] self.logger.debug("Cached Link Local IP: %s" % res) self.assertEqual( linklocal_ip, res, "The cached Link Local should be the same as the current Link Local IP, but they are different! Current ==> %s; Cached ==> %s " % (linklocal_ip, res)) return
def test_03_RVR_Network_check_router_state(self): """ Test redundant router internals """ self.logger.debug("Starting test_03_RVR_Network_check_router_state...") hypervisor = self.testClient.getHypervisorInfo() self.logger.debug("Creating Network Offering with default egress FALSE") network_offering_egress_false = NetworkOffering.create( self.apiclient, self.services["nw_off_persistent_RVR_egress_false"], conservemode=True ) network_offering_egress_false.update(self.apiclient, state='Enabled') self.logger.debug("Creating network with network offering: %s" % network_offering_egress_false.id) network = Network.create( self.apiclient, self.services["network"], accountid=self.account.name, domainid=self.account.domainid, networkofferingid=network_offering_egress_false.id, zoneid=self.zone.id ) self.logger.debug("Created network with ID: %s" % network.id) networks = Network.list( self.apiclient, id=network.id, listall=True ) self.assertEqual( isinstance(networks, list), True, "List networks should return a valid response for created network" ) nw_response = networks[0] self.logger.debug("Deploying VM in account: %s" % self.account.name) virtual_machine = VirtualMachine.create( self.apiclient, self.services["virtual_machine"], templateid=self.template.id, accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering.id, networkids=[str(network.id)] ) self.logger.debug("Deployed VM in network: %s" % network.id) self.cleanup.insert(0, network_offering_egress_false) self.cleanup.insert(0, network) self.cleanup.insert(0, virtual_machine) vms = VirtualMachine.list( self.apiclient, id=virtual_machine.id, listall=True ) self.assertEqual( isinstance(vms, list), True, "List Vms should return a valid list" ) vm = vms[0] self.assertEqual( vm.state, "Running", "VM should be in running state after deployment" ) self.logger.debug("Listing routers for network: %s" % network.name) routers = Router.list( self.apiclient, networkid=network.id, listall=True ) self.assertEqual( isinstance(routers, list), True, "list router should return Primary and backup routers" ) self.assertEqual( len(routers), 2, "Length of the list router should be 2 (Backup & Primary)" ) vals = ["PRIMARY", "BACKUP", "UNKNOWN"] cnts = [0, 0, 0] result = "UNKNOWN" for router in routers: if router.state == "Running": hosts = list_hosts( self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list host returns a valid list" ) host = hosts[0] if hypervisor.lower() in ('vmware', 'hyperv'): result = str(get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, router.linklocalip, "sh /opt/cloud/bin/checkrouter.sh ", hypervisor=hypervisor )) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = str(get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "sh /opt/cloud/bin/checkrouter.sh " )) except KeyError: self.skipTest( "Marvin configuration has no host credentials to\ check router services") if result.count(vals[0]) == 1: cnts[vals.index(vals[0])] += 1 if cnts[vals.index('PRIMARY')] != 1: self.fail("No Primary or too many primary routers found %s" % cnts[vals.index('PRIMARY')]) return
def test_04_restart_network_wo_cleanup(self): """Test restart network without cleanup """ # Validate the following # 1. When cleanup = false, router is restarted and # all services inside the router are restarted # 2. check 'uptime' to see if the actual restart happened timeout = 10 # Network should be in Implemented or Setup stage before restart while True: networks = list_networks( self.apiclient, account=self.account.name, domainid=self.account.domainid ) self.assertEqual( isinstance(networks, list), True, "Check list response returns a valid list" ) network = networks[0] if network.state in ["Implemented", "Setup"]: break elif timeout == 0: break else: time.sleep(self.services["sleep"]) timeout = timeout - 1 self.debug( "Restarting network with ID: %s, Network state: %s" % ( network.id, network.state )) cmd = restartNetwork.restartNetworkCmd() cmd.id = network.id cmd.cleanup = False self.apiclient.restartNetwork(cmd) # Get router details after restart list_router_response = list_routers( self.apiclient, account=self.account.name, domainid=self.account.domainid ) self.assertEqual( isinstance(list_router_response, list), True, "Check list response returns a valid list" ) router = list_router_response[0] hosts = list_hosts( self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list response returns a valid list" ) host = hosts[0] if self.hypervisor.lower() in ('vmware', 'hyperv'): res = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, router.linklocalip, "uptime", hypervisor=self.hypervisor ) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) res = get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "uptime" ) except KeyError: self.skipTest( "Marvin configuration has no host credentials\ to check router services") # res = 12:37:14 up 1 min, 0 users, load average: 0.61, 0.22, 0.08 # Split result to check the uptime result = res[0].split() self.debug("Router Uptime: %s" % result) self.assertEqual( str(result[1]), 'up', "Check router is running or not" ) if str(result[3]) == "min,": self.assertEqual( (int(result[2]) < 3), True, "Check uptime is less than 3 mins or not" ) else: self.assertEqual( str(result[3]), 'sec,', "Check uptime is in seconds" ) return
def check_routers_state(self, count=2, status_to_check="MASTER", expected_count=1, showall=False): vals = ["MASTER", "BACKUP", "UNKNOWN", "TESTFAILED"] cnts = [0, 0, 0] result = "TESTFAILED" self.logger.debug( 'check_routers_state count: %s, status_to_check: %s, expected_count: %s, showall: %s' % (count, status_to_check, expected_count, showall)) vrrp_interval = Configurations.list( self.apiclient, name="router.redundant.vrrp.interval") self.logger.debug("router.redundant.vrrp.interval is ==> %s" % vrrp_interval) total_sleep = 20 if vrrp_interval: total_sleep = (int(vrrp_interval[0].value) * 4) + 10 else: self.logger.debug( "Could not retrieve the key 'router.redundant.vrrp.interval'. Sleeping for 10 seconds." ) ''' Sleep (router.redundant.vrrp.interval * 4) seconds here because VRRP will have to be reconfigured. Due to the configuration changes, it will start a new election and that will take ~4 multiplied by the advertisement interval seconds. Next to that, we need some time for the router to be reconfigured, so adding 10 seconds to be on the safe side. ''' time.sleep(total_sleep) self.query_routers(count, showall) for router in self.routers: if router.state == "Running": hosts = list_hosts(self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid) self.assertEqual(isinstance(hosts, list), True, "Check list host returns a valid list") host = hosts[0] try: for _ in range(5): host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = str( get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "sh /opt/cosmic/router/scripts/checkrouter.sh " )) self.logger.debug( 'check_routers_state router: %s, result: %s' % (router.name, result)) if result.count(status_to_check) == 1: cnts[vals.index(status_to_check)] += 1 break elif result.count("UNKNOWN") == 1: time.sleep(5) else: break except KeyError: self.skipTest( "Marvin configuration has no host credentials to\ check router services") if cnts[vals.index(status_to_check)] != expected_count: self.fail( "Expected '%s' router[s] at state '%s', but found '%s'! Result: %s" % (expected_count, status_to_check, cnts[vals.index(status_to_check)], result))
def test_01_router_internal_basic(self): """Test router internal basic zone """ # Validate the following # 1. Router only does dhcp # 2. Verify that ports 67 (DHCP) and 53 (DNS) are open on UDP # by checking status of dnsmasq process # Find router associated with user account if self.zone.networktype == "Basic": list_router_response = list_routers( self.apiclient, listall="true" ) else: list_router_response = list_routers( self.apiclient, account=self.account.name, domainid=self.account.domainid ) self.assertEqual( isinstance(list_router_response, list), True, "Check list response returns a valid list" ) router = list_router_response[0] hosts = list_hosts( self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list host returns a valid list" ) host = hosts[0] self.debug("Router ID: %s, state: %s" % (router.id, router.state)) self.assertEqual( router.state, 'Running', "Check list router response for router state" ) if self.hypervisor.lower() in ('vmware', 'hyperv'): result = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, router.linklocalip, "service dnsmasq status", hypervisor=self.hypervisor ) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "service dnsmasq status" ) except KeyError: self.skipTest( "Marvin configuration has no host credentials to\ check router services") res = str(result) self.debug("Dnsmasq process status: %s" % res) self.assertEqual( res.count("running"), 1, "Check dnsmasq service is running or not" ) return
def check_routers_state(self, count=2, status_to_check="MASTER", expected_count=1, showall=False): vals = ["MASTER", "BACKUP", "UNKNOWN", "TESTFAILED"] cnts = [0, 0, 0] result = "TESTFAILED" self.logger.debug('check_routers_state count: %s, status_to_check: %s, expected_count: %s, showall: %s' % (count, status_to_check, expected_count, showall)) vrrp_interval = Configurations.list(self.apiclient, name="router.redundant.vrrp.interval") self.logger.debug("router.redundant.vrrp.interval is ==> %s" % vrrp_interval) total_sleep = 20 if vrrp_interval: total_sleep = (int(vrrp_interval[0].value) * 4) + 10 else: self.logger.debug("Could not retrieve the key 'router.redundant.vrrp.interval'. Sleeping for 10 seconds.") ''' Sleep (router.redundant.vrrp.interval * 4) seconds here because VRRP will have to be reconfigured. Due to the configuration changes, it will start a new election and that will take ~4 multiplied by the advertisement interval seconds. Next to that, we need some time for the router to be reconfigured, so adding 10 seconds to be on the safe side. ''' time.sleep(total_sleep) self.query_routers(count, showall) for router in self.routers: if router.state == "Running": hosts = list_hosts( self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list host returns a valid list" ) host = hosts[0] for _ in range(5): host.user, host.passwd = get_host_credentials(self.config, host.name) result = str(get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "sh /opt/cosmic/router/scripts/checkrouter.sh " )) self.logger.debug('check_routers_state router: %s, result: %s' % (router.name, result)) if result.count(status_to_check) == 1: cnts[vals.index(status_to_check)] += 1 break elif result.count("UNKNOWN") == 1: time.sleep(5) else: break if cnts[vals.index(status_to_check)] != expected_count: self.logger.debug("Investigate! not MASTER/BACKUP") while True: time.sleep(1) self.fail("Expected '%s' router[s] at state '%s', but found '%s'! Result: %s" % (expected_count, status_to_check, cnts[vals.index(status_to_check)], result))
def test_01_Multiple_RemoteAccessVPN_Connections_To_VPC_Ping_Guest_VM_Multiple_Times(self): """ Test case no : Test Multiple VPN Connections to a VPN Server on VPC # Validate the following for Each VPN VM Client # 1. Create VPN User on the VPC # 2. Configure the VPN Client VM with the required Information # 3. Initialize the VPN Client Services on the VPN Client # 4. Start the VPN Client Services on the VPN Client # 5. Wait for 30 seconds before attempting to ping # 6. Conduct the Ping Test on the VM # After the deployment VPN Client VMs and the post deployment steps, do the following steps: # 7. Wait for 60 seconds # 8. Check Routers pppX NICs Information """ for vm in xrange(0,int(TestMultipleVPNAccessonVPC.services["vpnclient_count"])): vpn_user_name = ''.join((str(vm),"-user")) vpn_password = ''.join((str(vm),"-pass")) self.debug("VPN User Name created with %s " % vpn_user_name) self.debug("VPN Password created with %s " % vpn_password) self.debug("Create new VPN User to use the Remote Access Service enabled on the VPC") newVPNUser = VpnUser.create( TestMultipleVPNAccessonVPC.api_client_vpn_server_reg_user, vpn_user_name, vpn_password, rand_name=False ) self.debug("VPN User %s got created Successfully " % vpn_user_name) self.debug("Configure the VPN Client Services on the VM deployed for VPN client purpose.") TestMultipleVPNAccessonVPC.configureVPNClientServicesFile( TestMultipleVPNAccessonVPC.vpnclientvms[vm], "/tmp/vpnclient_services.sh", TestMultipleVPNAccessonVPC.listFirstVPCPublicIpAddress[0].ipaddress, TestMultipleVPNAccessonVPC.listfirstNetworkTier[0].cidr, TestMultipleVPNAccessonVPC.FirstVPNonFirstVPC.presharedkey, vpn_user_name, vpn_password ) self.debug("Configuration of VPN Client VM %d Done " % (vm)) self.debug("Initialize the VPN Client Services on the VPN Client") TestMultipleVPNAccessonVPC.vpnClientServicesInit( TestMultipleVPNAccessonVPC.vpnclientvms[vm], "/tmp/vpnclient_services.sh" ) self.debug("Initiation of VPN Client Services on VM %d Done " % (vm)) self.debug("Start the VPN Client Services on the VPN Client") TestMultipleVPNAccessonVPC.vpnClientServicesStart( TestMultipleVPNAccessonVPC.vpnclientvms[vm], "/tmp/vpnclient_services.sh" ) self.debug("VPN Client Services on VM %d Started Successfully " % (vm)) self.debug("Wait for 30 seconds before attempting to ping") time.sleep(30) self.debug("Conduct the Ping Test on the VM %d" % (vm)) thread.start_new_thread(self.ping_vm,( TestMultipleVPNAccessonVPC.vpnclientvms[vm], TestMultipleVPNAccessonVPC.vm1.nic[0].ipaddress, 25000, 15, "Thread-{0}".format(vm) )) self.debug("Waiting for 60 seconds.........") time.sleep(60) self.debug("End of 60 seconds.........") # Find router associated with user account list_router_response = list_routers( self.apiclient, vpcid= TestMultipleVPNAccessonVPC.firstvpc.id, listall=True ) self.assertEqual( isinstance(list_router_response, list), True, "Check list response returns a valid list" ) router = list_router_response[0] hosts = list_hosts( self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list response returns a valid list" ) host = hosts[0] self.debug("Router ID: %s, state: %s" % (router.id, router.state)) self.assertEqual( router.state, 'Running', "Check list router response for router state" ) if self.hypervisor.lower() == 'vmware': result = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, router.linklocalip, "ifconfig | grep ppp", hypervisor=self.hypervisor ) else: try: host.user, host.passwd = get_host_credentials(self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "ifconfig | grep ppp" ) self.debug("Routers pppX NICs Information : %s" % str(result)) except KeyError: self.skipTest("Marvin configuration has no host credentials to check router services")
def test_03_RVR_Network_check_router_state(self): """ Test redundant router internals """ self.logger.debug("Starting test_03_RVR_Network_check_router_state...") hypervisor = self.testClient.getHypervisorInfo() self.logger.debug("Creating Network Offering with default egress FALSE") network_offering_egress_false = NetworkOffering.create( self.apiclient, self.services["nw_off_persistent_RVR_egress_false"], conservemode=True ) network_offering_egress_false.update(self.apiclient, state='Enabled') self.logger.debug("Creating network with network offering: %s" % network_offering_egress_false.id) network = Network.create( self.apiclient, self.services["network"], accountid=self.account.name, domainid=self.account.domainid, networkofferingid=network_offering_egress_false.id, zoneid=self.zone.id ) self.logger.debug("Created network with ID: %s" % network.id) networks = Network.list( self.apiclient, id=network.id, listall=True ) self.assertEqual( isinstance(networks, list), True, "List networks should return a valid response for created network" ) nw_response = networks[0] self.logger.debug("Deploying VM in account: %s" % self.account.name) virtual_machine = VirtualMachine.create( self.apiclient, self.services["virtual_machine"], templateid=self.template.id, accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering.id, networkids=[str(network.id)] ) self.logger.debug("Deployed VM in network: %s" % network.id) self.cleanup.insert(0, network_offering_egress_false) self.cleanup.insert(0, network) self.cleanup.insert(0, virtual_machine) vms = VirtualMachine.list( self.apiclient, id=virtual_machine.id, listall=True ) self.assertEqual( isinstance(vms, list), True, "List Vms should return a valid list" ) vm = vms[0] self.assertEqual( vm.state, "Running", "VM should be in running state after deployment" ) self.logger.debug("Listing routers for network: %s" % network.name) routers = Router.list( self.apiclient, networkid=network.id, listall=True ) self.assertEqual( isinstance(routers, list), True, "list router should return Master and backup routers" ) self.assertEqual( len(routers), 2, "Length of the list router should be 2 (Backup & master)" ) vals = ["MASTER", "BACKUP", "UNKNOWN"] cnts = [0, 0, 0] result = "UNKNOWN" for router in routers: if router.state == "Running": hosts = list_hosts( self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list host returns a valid list" ) host = hosts[0] if hypervisor.lower() in ('vmware', 'hyperv'): result = str(get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, router.linklocalip, "sh /opt/cloud/bin/checkrouter.sh ", hypervisor=hypervisor )) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = str(get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "sh /opt/cloud/bin/checkrouter.sh " )) except KeyError: self.skipTest( "Marvin configuration has no host credentials to\ check router services") if result.count(vals[0]) == 1: cnts[vals.index(vals[0])] += 1 if cnts[vals.index('MASTER')] != 1: self.fail("No Master or too many master routers found %s" % cnts[vals.index('MASTER')]) return
def test_06_create_virtual_machine(self): # Validate the following # # 1. Create a security group # 2. Create a virtual machine # 3. Try to add a new ingress rule # 4. Check if ingress rule is applied successfully on host # 5. Throw exception if it's not applied # 6. Try to add a new egress rule # 7. Check if egress rule is applied successfully on host # 8. Throw exception if it's not applied self.security_group = SecurityGroup.create( self.apiclient, self.testdata["security_group"], account=self.account.name, domainid=self.account.domainid) self.virtual_machine = VirtualMachine.create( self.apiclient, self.testdata["virtual_machine_userdata"], accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering.id, securitygroupids=[self.security_group.id]) # Get the virtual machine virtial_machine = VirtualMachine.list(self.apiclient, id=self.virtual_machine.id) vm = virtial_machine[0] # get the host on which the vm is running hosts = list_hosts(self.apiclient, id=vm.hostid) host = hosts[0] if host.hypervisor.lower() not in "kvm": return host.user, host.passwd = get_host_credentials(self.config, host.ipaddress) # Add ingress rule self.createIngressRule("tcp", "1.1.1.1/32") # verify number of ingress rules and egress rules self.verify_security_group_rules(self.security_group.id, 1, 0) # Check if the ingress rule if applied successfully on host rule = "-A %s -s 1.1.1.1/32 -p tcp -m tcp --dport 1:65535 -m state --state NEW -j ACCEPT" % vm.instancename self.verify_rule_on_host(host.ipaddress, host.user, host.passwd, rule) # Add ingress rule self.createIngressRule("udp", "2.2.2.2/32") # verify number of ingress rules and egress rules self.verify_security_group_rules(self.security_group.id, 2, 0) # Check if the ingress rule if applied successfully on host rule = "-A %s -s 2.2.2.2/32 -p udp -m udp --dport 1:65535 -m state --state NEW -j ACCEPT" % vm.instancename self.verify_rule_on_host(host.ipaddress, host.user, host.passwd, rule) # Add ingress rule self.createIngressRule("icmp", "3.3.3.3/32") # verify number of ingress rules and egress rules self.verify_security_group_rules(self.security_group.id, 3, 0) # Check if the ingress rule if applied successfully on host rule = "-A %s -s 3.3.3.3/32 -p icmp -m icmp --icmp-type any -j ACCEPT" % vm.instancename self.verify_rule_on_host(host.ipaddress, host.user, host.passwd, rule) # Add ingress rule self.createIngressRule("all", "4.4.4.4/32") # verify number of ingress rules and egress rules self.verify_security_group_rules(self.security_group.id, 4, 0) # Check if the ingress rule if applied successfully on host rule = "-A %s -s 4.4.4.4/32 -m state --state NEW -j ACCEPT" % vm.instancename self.verify_rule_on_host(host.ipaddress, host.user, host.passwd, rule) # Add ingress rule self.createIngressRule("47", "5.5.5.5/32") # verify number of ingress rules and egress rules self.verify_security_group_rules(self.security_group.id, 5, 0) # Check if the ingress rule if applied successfully on host rule = "-A %s -s 5.5.5.5/32 -p gre -j ACCEPT" % vm.instancename self.verify_rule_on_host(host.ipaddress, host.user, host.passwd, rule) # Add egress rule self.createEgressRule("tcp", "11.11.11.11/32") # verify number of ingress rules and egress rules self.verify_security_group_rules(self.security_group.id, 5, 1) # Check if the egress rule if applied successfully on host rule = "-A %s-eg -d 11.11.11.11/32 -p tcp -m tcp --dport 1:65535 -m state --state NEW -j RETURN" % vm.instancename self.verify_rule_on_host(host.ipaddress, host.user, host.passwd, rule) # Add egress rule self.createEgressRule("udp", "12.12.12.12/32") # verify number of ingress rules and egress rules self.verify_security_group_rules(self.security_group.id, 5, 2) # Check if the egress rule if applied successfully on host rule = "-A %s-eg -d 12.12.12.12/32 -p udp -m udp --dport 1:65535 -m state --state NEW -j RETURN" % vm.instancename self.verify_rule_on_host(host.ipaddress, host.user, host.passwd, rule) # Add egress rule self.createEgressRule("icmp", "13.13.13.13/32") # verify number of ingress rules and egress rules self.verify_security_group_rules(self.security_group.id, 5, 3) # Check if the egress rule if applied successfully on host rule = "-A %s-eg -d 13.13.13.13/32 -p icmp -m icmp --icmp-type any -j RETURN" % vm.instancename self.verify_rule_on_host(host.ipaddress, host.user, host.passwd, rule) # Add egress rule self.createEgressRule("all", "14.14.14.14/32") # verify number of ingress rules and egress rules self.verify_security_group_rules(self.security_group.id, 5, 4) # Check if the egress rule if applied successfully on host rule = "-A %s-eg -d 14.14.14.14/32 -m state --state NEW -j RETURN" % vm.instancename self.verify_rule_on_host(host.ipaddress, host.user, host.passwd, rule) # Add egress rule self.createEgressRule("47", "15.15.15.15/32") # verify number of ingress rules and egress rules self.verify_security_group_rules(self.security_group.id, 5, 5) # Check if the egress rule if applied successfully on host rule = "-A %s-eg -d 15.15.15.15/32 -p gre -j RETURN" % vm.instancename self.verify_rule_on_host(host.ipaddress, host.user, host.passwd, rule)
def test_03_ssvm_internals(self): """Test SSVM Internals""" # Validate the following # 1. The SSVM check script should not return any # WARN|ERROR|FAIL messages # 2. If you are unable to login to the SSVM with the signed key # then test is deemed a failure # 3. There should be only one ""cloud"" process running within the SSVM # 4. If no process is running/multiple process are running # then the test is a failure list_ssvm_response = list_ssvms( self.apiclient, systemvmtype='secondarystoragevm', state='Running', zoneid=self.zone.id ) self.assertEqual( isinstance(list_ssvm_response, list), True, "Check list response returns a valid list" ) ssvm = list_ssvm_response[0] hosts = list_hosts( self.apiclient, id=ssvm.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list response returns a valid list" ) host = hosts[0] self.debug("Running SSVM check script") if self.hypervisor.lower() in ('vmware', 'hyperv'): # SSH into SSVMs is done via management server for Vmware and # Hyper-V result = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, ssvm.privateip, "/usr/local/cloud/systemvm/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL", hypervisor=self.hypervisor) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, ssvm.linklocalip, "/usr/local/cloud/systemvm/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL") except KeyError: self.skipTest( "Marvin configuration has no host\ credentials to check router services") res = str(result) self.debug("SSVM script output: %s" % res) self.assertEqual( res.count("ERROR"), 1, "Check for Errors in tests" ) self.assertEqual( res.count("WARNING"), 1, "Check for warnings in tests" ) # Check status of cloud service if self.hypervisor.lower() in ('vmware', 'hyperv'): # SSH into SSVMs is done via management server for Vmware and # Hyper-V result = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, ssvm.privateip, "service cloud status", hypervisor=self.hypervisor ) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, ssvm.linklocalip, "service cloud status" ) except KeyError: self.skipTest( "Marvin configuration has no host\ credentials to check router services") res = str(result) self.debug("Cloud Process status: %s" % res) # cloud.com service (type=secstorage) is running: process id: 2346 self.assertEqual( res.count("is running"), 1, "Check cloud service is running or not" ) return
def test_03_ssvm_internals(self): """Test SSVM Internals""" # Validate the following # 1. The SSVM check script should not return any # WARN|ERROR|FAIL messages # 2. If you are unable to login to the SSVM with the signed key # then test is deemed a failure # 3. There should be only one ""cloud"" process running within the SSVM # 4. If no process is running/multiple process are running # then the test is a failure list_ssvm_response = list_ssvms(self.apiclient, systemvmtype='secondarystoragevm', state='Running', zoneid=self.zone.id) self.assertEqual(isinstance(list_ssvm_response, list), True, "Check list response returns a valid list") ssvm = list_ssvm_response[0] hosts = list_hosts(self.apiclient, id=ssvm.hostid) self.assertEqual(isinstance(hosts, list), True, "Check list response returns a valid list") host = hosts[0] self.logger.debug("Running SSVM check script") try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, ssvm.linklocalip, "/opt/cosmic/agent/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL" ) except KeyError: self.skipTest( "Marvin configuration has no host credentials to check router services" ) res = str(result) self.logger.debug("SSVM script output: %s" % res) self.assertEqual(res.count("ERROR"), 1, "Check for Errors in tests") self.assertEqual(res.count("WARNING"), 1, "Check for warnings in tests") # Check status of cosmic-agent service try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status(host.ipaddress, 22, host.user, host.passwd, ssvm.linklocalip, "systemctl status cosmic-agent") except KeyError: self.skipTest( "Marvin configuration has no host credentials to check router services" ) res = str(result) self.logger.debug("cosmic-agent Process status: %s" % res) # cloud.com service (type=secstorage) is running: process id: 2346 self.assertEqual(res.count("active (running)"), 1, "Check cosmic-agent service is running or not") linklocal_ip = None # Check status of cloud service try: linklocal_ip = ssvm.linklocalip host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, ssvm.linklocalip, "cat /etc/cosmic/agent/agent.properties | grep eth0ip= | cut -d= -f2" ) except KeyError: self.skipTest( "Marvin configuration has no host credentials to check router services" ) res = result[0] self.logger.debug("Cached Link Local IP: %s" % res) self.assertEqual( linklocal_ip, res, "The cached Link Local should be the same as the current Link Local IP, but they are different! Current ==> %s; Cached ==> %s " % (linklocal_ip, res)) return
def test_04_cpvm_internals(self): """Test CPVM Internals""" # Validate the following # 1. test that telnet access on 8250 is available to # the management server for the CPVM # 2. No telnet access, test FAIL # 3. Service cloud status should report cloud agent status to be # running list_cpvm_response = list_ssvms(self.apiclient, systemvmtype='consoleproxy', state='Running', zoneid=self.zone.id) self.assertEqual(isinstance(list_cpvm_response, list), True, "Check list response returns a valid list") cpvm = list_cpvm_response[0] hosts = list_hosts(self.apiclient, id=cpvm.hostid) self.assertEqual(isinstance(hosts, list), True, "Check list response returns a valid list") host = hosts[0] try: telnetlib.Telnet(str(self.apiclient.connection.mgtSvr), '8250') self.debug("Telnet management server (IP: %s)" % self.apiclient.connection.mgtSvr) except Exception as e: self.fail("Telnet Access failed for %s: %s" % (self.apiclient.connection.mgtSvr, e)) self.debug("Checking cloud process status") if self.hypervisor.lower() in ('vmware', 'hyperv'): # SSH into SSVMs is done via management server for Vmware and # Hyper-V result = get_process_status(self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, cpvm.privateip, "service cloud status", hypervisor=self.hypervisor) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status(host.ipaddress, 22, host.user, host.passwd, cpvm.linklocalip, "service cloud status") except KeyError: self.skipTest("Marvin configuration has no host\ credentials to check router services") res = str(result) self.debug("Cloud Process status: %s" % res) self.assertEqual(res.count("is running"), 1, "Check cloud service is running or not") linklocal_ip = None # Check status of cloud service if self.hypervisor.lower() in ('vmware', 'hyperv'): # SSH into SSVMs is done via management server for Vmware and # Hyper-V linklocal_ip = cpvm.privateip result = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, cpvm.privateip, "cat /var/cache/cloud/cmdline | xargs | sed \"s/ /\\n/g\" | grep eth1ip= | sed \"s/\=/ /g\" | awk '{print $2}'", hypervisor=self.hypervisor) else: try: linklocal_ip = cpvm.linklocalip host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, cpvm.linklocalip, "cat /var/cache/cloud/cmdline | xargs | sed \"s/ /\\n/g\" | grep eth0ip= | sed \"s/\=/ /g\" | awk '{print $2}'" ) except KeyError: self.skipTest("Marvin configuration has no host\ credentials to check router services") res = result[0] self.debug("Cached Link Local IP: %s" % res) self.assertEqual( linklocal_ip, res, "The cached Link Local should be the same as the current Link Local IP, but they are different! Current ==> %s; Cached ==> %s " % (linklocal_ip, res)) return
def test_02_router_internal_adv(self): """Test router internal advanced zone """ # Validate the following # 1. Router does dhcp, dns, gateway, LB, PF, FW # 2. verify that dhcp, dns ports are open on UDP # 3. dnsmasq, haproxy processes should be running # Find router associated with user account list_router_response = list_routers( self.apiclient, account=self.account.name, domainid=self.account.domainid ) self.assertEqual( isinstance(list_router_response, list), True, "Check list response returns a valid list" ) router = list_router_response[0] hosts = list_hosts( self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list response returns a valid list" ) host = hosts[0] self.debug("Router ID: %s, state: %s" % (router.id, router.state)) self.assertEqual( router.state, 'Running', "Check list router response for router state" ) if self.hypervisor.lower() in ('vmware', 'hyperv'): result = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, router.linklocalip, "service dnsmasq status", hypervisor=self.hypervisor ) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "service dnsmasq status" ) except KeyError: self.skipTest( "Marvin configuration has no host credentials\ to check router services") res = str(result) self.debug("Dnsmasq process status: %s" % res) self.assertEqual( res.count("running"), 1, "Check dnsmasq service is running or not" ) if self.hypervisor.lower() in ('vmware', 'hyperv'): result = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, router.linklocalip, "service haproxy status", hypervisor=self.hypervisor ) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "service haproxy status" ) except KeyError: self.skipTest( "Marvin configuration has no host credentials\ to check router services") res = str(result) self.assertEqual( res.count("running"), 1, "Check haproxy service is running or not" ) self.debug("Haproxy process status: %s" % res) return
def test_03_RVR_Network_check_router_state(self): """ Test redundant router internals """ self.logger.debug("Starting test_03_RVR_Network_check_router_state...") network_offering_egress_false = get_default_redundant_isolated_network_offering( self.apiclient) self.logger.debug("Creating network with network offering: %s" % network_offering_egress_false.id) network = Network.create( self.apiclient, self.services["network"], accountid=self.account.name, domainid=self.account.domainid, networkofferingid=network_offering_egress_false.id, zoneid=self.zone.id) self.logger.debug("Created network with ID: %s" % network.id) networks = Network.list(self.apiclient, id=network.id, listall=True) self.assertEqual( isinstance(networks, list), True, "List networks should return a valid response for created network") self.logger.debug("Deploying VM in account: %s" % self.account.name) virtual_machine = VirtualMachine.create( self.apiclient, self.services["virtual_machine"], templateid=self.template.id, accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.service_offering.id, networkids=[str(network.id)]) self.logger.debug("Deployed VM in network: %s" % network.id) self.cleanup.insert(0, network) self.cleanup.insert(0, virtual_machine) vms = VirtualMachine.list(self.apiclient, id=virtual_machine.id, listall=True) self.assertEqual(isinstance(vms, list), True, "List Vms should return a valid list") vm = vms[0] self.assertEqual(vm.state, "Running", "VM should be in running state after deployment") self.logger.debug("Listing routers for network: %s" % network.name) routers = Router.list(self.apiclient, networkid=network.id, listall=True) self.assertEqual( isinstance(routers, list), True, "list router should return Master and backup routers") self.assertEqual( len(routers), 2, "Length of the list router should be 2 (Backup & master)") vals = ["MASTER", "BACKUP", "UNKNOWN"] cnts = [0, 0, 0] result = "UNKNOWN" for router in routers: if router.state == "Running": hosts = list_hosts(self.apiclient, zoneid=router.zoneid, type='Routing', state='Up', id=router.hostid) self.assertEqual(isinstance(hosts, list), True, "Check list host returns a valid list") host = hosts[0] try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = str( get_process_status( host.ipaddress, 22, host.user, host.passwd, router.linklocalip, "sh /opt/cosmic/router/scripts/checkrouter.sh ")) except KeyError: self.skipTest( "Marvin configuration has no host credentials to\ check router services") if result.count(vals[0]) == 1: cnts[vals.index(vals[0])] += 1 if cnts[vals.index('MASTER')] != 1: self.fail("No Master or too many master routers found %s" % cnts[vals.index('MASTER')]) return
def test_04_cpvm_internals(self): """Test CPVM Internals""" # Validate the following # 1. test that telnet access on 8250 is available to # the management server for the CPVM # 2. No telnet access, test FAIL # 3. Service cloud status should report cloud agent status to be # running list_cpvm_response = list_ssvms( self.apiclient, systemvmtype='consoleproxy', state='Running', zoneid=self.zone.id ) self.assertEqual( isinstance(list_cpvm_response, list), True, "Check list response returns a valid list" ) cpvm = list_cpvm_response[0] hosts = list_hosts( self.apiclient, id=cpvm.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list response returns a valid list" ) host = hosts[0] try: telnetlib.Telnet( str(self.apiclient.connection.mgtSvr), '8250' ) self.debug("Telnet management server (IP: %s)" % self.apiclient.connection.mgtSvr) except Exception as e: self.fail( "Telnet Access failed for %s: %s" % (self.apiclient.connection.mgtSvr, e) ) self.debug("Checking cloud process status") if self.hypervisor.lower() in ('vmware', 'hyperv'): # SSH into SSVMs is done via management server for Vmware and # Hyper-V result = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, cpvm.privateip, "systemctl is-active cloud", hypervisor=self.hypervisor ) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, cpvm.linklocalip, "systemctl is-active cloud" ) except KeyError: self.skipTest( "Marvin configuration has no host\ credentials to check router services") res = str(result) self.debug("Cloud Process status: %s" % res) self.assertEqual( res.count("active"), 1, "Check cloud service is running or not" ) linklocal_ip = None # Check status of cloud service if self.hypervisor.lower() in ('vmware', 'hyperv'): # SSH into SSVMs is done via management server for Vmware and # Hyper-V linklocal_ip = cpvm.privateip result = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, cpvm.privateip, "cat /var/cache/cloud/cmdline | xargs | sed \"s/ /\\n/g\" | grep eth1ip= | sed \"s/\=/ /g\" | awk '{print $2}'", hypervisor=self.hypervisor ) else: try: linklocal_ip = cpvm.linklocalip host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, cpvm.linklocalip, "cat /var/cache/cloud/cmdline | xargs | sed \"s/ /\\n/g\" | grep eth0ip= | sed \"s/\=/ /g\" | awk '{print $2}'" ) except KeyError: self.skipTest( "Marvin configuration has no host\ credentials to check router services") res = result[0] self.debug("Cached Link Local IP: %s" % res) self.assertEqual( linklocal_ip, res, "The cached Link Local should be the same as the current Link Local IP, but they are different! Current ==> %s; Cached ==> %s " % (linklocal_ip, res) )
def test_04_cpvm_internals(self): """Test CPVM Internals""" # Validate the following # 1. test that telnet access on 8250 is available to # the management server for the CPVM # 2. No telnet access, test FAIL # 3. Service cloud status should report cloud agent status to be # running list_cpvm_response = list_ssvms( self.apiclient, systemvmtype='consoleproxy', state='Running', zoneid=self.zone.id ) self.assertEqual( isinstance(list_cpvm_response, list), True, "Check list response returns a valid list" ) cpvm = list_cpvm_response[0] hosts = list_hosts( self.apiclient, id=cpvm.hostid ) self.assertEqual( isinstance(hosts, list), True, "Check list response returns a valid list" ) host = hosts[0] try: telnetlib.Telnet( str(self.apiclient.connection.mgtSvr), '8250' ) self.debug("Telnet management server (IP: %s)" % self.apiclient.connection.mgtSvr) except Exception as e: self.fail( "Telnet Access failed for %s: %s" % (self.apiclient.connection.mgtSvr, e) ) self.debug("Checking cloud process status") if self.hypervisor.lower() in ('vmware', 'hyperv'): # SSH into SSVMs is done via management server for Vmware and # Hyper-V result = get_process_status( self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, cpvm.privateip, "service cloud status", hypervisor=self.hypervisor ) else: try: host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) result = get_process_status( host.ipaddress, 22, host.user, host.passwd, cpvm.linklocalip, "service cloud status" ) except KeyError: self.skipTest( "Marvin configuration has no host\ credentials to check router services") res = str(result) self.debug("Cloud Process status: %s" % res) self.assertEqual( res.count("is running"), 1, "Check cloud service is running or not" ) return
def test_01_RouterStopCreateFW(self): """Test router stop create Firewall rule """ # validate the following # 1. 1. listFirewallRules (filter by ipaddressid of sourcenat) # 2. rule should be for ports 1-600 and in state=Active # (optional backend) # 3. verify on router using iptables -t nat -nvx if rules are applied # Get the router details associated with account routers = list_routers( self.apiclient, account=self.account.name, domainid=self.account.domainid, ) self.assertEqual(isinstance(routers, list), True, "Check for list routers response return valid data") self.assertNotEqual(len(routers), 0, "Check list router response") router = routers[0] self.debug("Stopping the router: %s" % router.id) # Stop the router cmd = stopRouter.stopRouterCmd() cmd.id = router.id self.apiclient.stopRouter(cmd) routers = list_routers( self.apiclient, account=self.account.name, domainid=self.account.domainid, ) self.assertEqual(isinstance(routers, list), True, "Check for list routers response return valid data") router = routers[0] self.assertEqual(router.state, 'Stopped', "Check list router response for router state") public_ips = list_publicIP(self.apiclient, account=self.account.name, domainid=self.account.domainid) self.assertEqual( isinstance(public_ips, list), True, "Check for list public IP response return valid data") public_ip = public_ips[0] # Create Firewall rule with configurations from settings file fw_rule = FireWallRule.create( self.apiclient, ipaddressid=public_ip.id, protocol='TCP', cidrlist=[self.services["fw_rule"]["cidr"]], startport=self.services["fw_rule"]["startport"], endport=self.services["fw_rule"]["endport"]) self.debug("Created firewall rule: %s" % fw_rule.id) self.debug("Starting the router: %s" % router.id) # Start the router cmd = startRouter.startRouterCmd() cmd.id = router.id self.apiclient.startRouter(cmd) routers = list_routers( self.apiclient, account=self.account.name, domainid=self.account.domainid, ) self.assertEqual(isinstance(routers, list), True, "Check for list routers response return valid data") router = routers[0] self.assertEqual(router.state, 'Running', "Check list router response for router state") # After Router start, FW rule should be in Active state fw_rules = list_firewall_rules( self.apiclient, id=fw_rule.id, ) self.assertEqual(isinstance(fw_rules, list), True, "Check for list FW rules response return valid data") self.assertEqual(fw_rules[0].state, 'Active', "Check list load balancing rules") self.assertEqual(fw_rules[0].startport, str(self.services["fw_rule"]["startport"]), "Check start port of firewall rule") self.assertEqual(fw_rules[0].endport, str(self.services["fw_rule"]["endport"]), "Check end port of firewall rule") # For DNS and DHCP check 'dnsmasq' process status if (self.hypervisor.lower() == 'vmware' or self.hypervisor.lower() == 'hyperv'): result = get_process_status(self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, self.apiclient.connection.passwd, router.linklocalip, 'iptables -t nat -L', hypervisor=self.hypervisor) else: hosts = list_hosts( self.apiclient, id=router.hostid, ) self.assertEqual( isinstance(hosts, list), True, "Check for list hosts response return valid data") host = hosts[0] host.user, host.passwd = get_host_credentials( self.config, host.ipaddress) try: result = get_process_status(host.ipaddress, 22, host.user, host.passwd, router.linklocalip, 'iptables -t nat -L') except KeyError: self.skipTest("Provide a marvin config file with host\ credentials to run %s" % self._testMethodName) self.debug("iptables -t nat -L: %s" % result) self.debug("Public IP: %s" % public_ip.ipaddress) res = str(result) self.assertEqual(res.count(str(public_ip.ipaddress)), 1, "Check public IP address") return