def __init__(self, host, port, user, passwd, retries=60, delay=10, log_lvl=logging.INFO, keyPairFiles=None, timeout=10.0): self.host = None self.port = 22 self.user = user self.passwd = passwd self.keyPairFiles = keyPairFiles self.ssh = SSHClient() self.ssh.set_missing_host_key_policy(AutoAddPolicy()) self.retryCnt = 0 self.delay = 0 self.timeout = 3.0 self.logger = MarvinLog('ssh').getLogger() # Check invalid host value and raise exception # Atleast host is required for connection if host is not None and host != '': self.host = host if retries is not None and retries > 0: self.retryCnt = retries if delay is not None and delay > 0: self.delay = delay if timeout is not None and timeout > 0: self.timeout = timeout if port is not None and port >= 0: self.port = port if self.createConnection() == FAILED: raise internalError("Connection Failed")
def __init__(self, mgmt_details, dbsvr_details, async_timeout=3600, test_data_filepath=None, zone=None, hypervisor_type=None): self.__mgmtDetails = mgmt_details self.__dbSvrDetails = dbsvr_details self.__csConnection = None self.__dbConnection = None self.__testClient = None self.__asyncTimeOut = async_timeout self.__logger = MarvinLog('marvin').getLogger() self.__apiClient = None self.__userApiClient = None self.__asyncJobMgr = None self.__id = None self.__hypervisor = hypervisor_type self.__testDataFilePath = test_data_filepath self.__parsedTestDataConfig = None self.__zone = zone self.__setHypervisorInfo()
def __init__(self, host, port, user, passwd, retries=60, delay=10, log_lvl=logging.INFO, keyPairFiles=None, timeout=10.0): self.host = None self.port = 22 self.user = user self.passwd = passwd self.keyPairFiles = keyPairFiles self.ssh = SSHClient() self.ssh.set_missing_host_key_policy(AutoAddPolicy()) self.retryCnt = 0 self.delay = 0 self.timeout = 3.0 self.logger = MarvinLog('ssh').getLogger() # Check invalid host value and raise exception # Atleast host is required for connection if host is not None and host != '': self.host = host if retries is not None and retries > 0: self.retryCnt = retries if delay is not None and delay > 0: self.delay = delay if timeout is not None and timeout > 0: self.timeout = timeout if port is not None and port >= 0: self.port = port if self.createConnection() == FAILED: raise internalError("Connection Failed")
def __init__(self, mgmt_details, dbsvr_details, async_timeout=3600, test_data_filepath=None, zone=None, hypervisor_type=None): self.__mgmtDetails = mgmt_details self.__dbSvrDetails = dbsvr_details self.__csConnection = None self.__dbConnection = None self.__testClient = None self.__asyncTimeOut = async_timeout self.__logger = MarvinLog('marvin').getLogger() self.__apiClient = None self.__userApiClient = None self.__asyncJobMgr = None self.__id = None self.__hypervisor = hypervisor_type self.__testDataFilePath = test_data_filepath self.__parsedTestDataConfig = None self.__zone = zone self.__setHypervisorInfo()
def __init__(self, test_client, cfg): self.__cfg = cfg self.__test_client = test_client self.__logger = MarvinLog('marvin').getLogger() self.__apiClient = None
class DeleteDataCenters: """ @Desc : Deletes the Data Center using the settings provided. test_client :Client for deleting the DC. dc_cfg_file : obj file exported by DeployDataCenter when successfully created DC. This file is serialized one containing entries with successful DC. dc_cfg: If dc_cfg_file, is not available, we can use the dictionary of elements to delete. tc_run_logger: Logger to dump log messages. """ def __init__(self, test_client, cfg): self.__cfg = cfg self.__test_client = test_client self.__logger = MarvinLog('marvin').getLogger() self.__apiClient = None def __deleteCmds(self, cmd_name, cmd_obj): """ @Name : __deleteCmds @Desc : Deletes the entities provided by cmd """ if cmd_name.lower() == "deletehostcmd": cmd_obj.forcedestroylocalstorage = "true" cmd_obj.force = "true" ''' Step1 : Prepare Host For Maintenance ''' host_maint_cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd() host_maint_cmd.id = cmd_obj.id host_maint_resp = self.__apiClient.prepareHostForMaintenance(host_maint_cmd) if host_maint_resp: ''' Step2 : List Hosts for Resource State ''' list_host_cmd = listHosts.listHostsCmd() list_host_cmd.id = cmd_obj.id retries = 3 for i in xrange(retries): list_host_resp = self.__apiClient.listHosts(list_host_cmd) if list_host_resp and (list_host_resp[0].resourcestate == 'Maintenance'): break sleep(30) if cmd_name.lower() == "deletestoragepoolcmd": cmd_obj.forced = "true" store_maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() store_maint_cmd.id = cmd_obj.id store_maint_resp = self.__apiClient.enableStorageMaintenance(store_maint_cmd) if store_maint_resp: list_store_cmd = listStoragePools.listStoragePoolsCmd() list_store_cmd.id = cmd_obj.id retries = 3 for i in xrange(retries): store_maint_resp = self.__apiClient.listStoragePools(list_store_cmd) if store_maint_resp and (store_maint_resp[0].state == 'Maintenance'): break sleep(30) return cmd_obj def __setClient(self): self.__apiClient = self.__test_client.getApiClient() def __cleanEntries(self): """ @Name : __cleanAndEntries @Description: Cleans up the created DC in order of creation """ try: ret = FAILED if "order" in self.__cfg.keys() and len(self.__cfg["order"]): self.__cfg["order"].reverse() self.__logger.info("=== Clean Up Entries: %s ===" % self.__cfg) for type in self.__cfg["order"]: self.__logger.info("=== CleanUp Started For Type: %s ===" % type) if type: temp_ids = self.__cfg[type] ids = [items for items in temp_ids if items] for id in ids: del_mod = "delete" + type del_cmd = getattr(globals()[del_mod], del_mod + "Cmd") del_cmd_obj = del_cmd() del_cmd_obj.id = id del_cmd_obj = self.__deleteCmds(del_mod + "Cmd", del_cmd_obj) del_func = getattr(self.__apiClient, del_mod) del_cmd_resp = del_func(del_cmd_obj) if del_cmd_resp: self.__logger.error("=== %s CleanUp Failed. ID: %s ===" % (type, id)) else: self.__logger.info("=== %s CleanUp Successful. ID : %s ===" % (type, id)) ret = SUCCESS except Exception as e: self.__logger.exception("=== Clean Up Entries failed: %s ===" % e) finally: return ret def removeDataCenter(self): """ @Name : removeDataCenter @Desc : Removes the Data Center provided by Configuration If Input dc file configuration is None, uses the cfg provided else uses the dc file to get the configuration """ try: self.__setClient() self.__logger.info("=== DeployDC: CleanUp Started ===") ret = self.__cleanEntries() except Exception as e: self.__logger.exception("=== DeployDC: CleanUp failed: %s ===" % e) finally: return ret
class DeployDataCenters(object): """ @Desc : Deploys the Data Center with information provided. Once the Deployment is successful, it will export the DataCenter settings to an obj file ( can be used if wanted to delete the created DC) """ def __init__(self, test_client, cfg): self.__test_client = test_client self.__config = cfg self.__logger = MarvinLog('marvin').getLogger() self.__apiClient = None self.__cleanUp = {} def __persistDcConfig(self): try: ts = strftime("%b_%d_%Y_%H_%M_%S", localtime()) dc_file_path = "dc_entries_" + str(ts) + ".obj" file_to_write = open(dc_file_path, 'w') if file_to_write: pickle.dump(self.__cleanUp, file_to_write) self.__logger.info("=== Data Center Settings are dumped to %s ===" % dc_file_path) except Exception as e: self.__logger.exception("=== Persisting DataCenter config failed: %s ===" % e) def __cleanAndExit(self): try: self.__logger.info("=== deploy dc failed, so cleaning the created entries ===") if not test_data.get("deleteDC", None): self.__logger.info("=== Deploy DC Clean Up flag not set. So, exiting ===") exit(1) self.__logger.info("=== Deploy DC Failed, So Cleaning to Exit ===") remove_dc_obj = DeleteDataCenters(self.__test_client, self.__cleanUp) if remove_dc_obj: if remove_dc_obj.removeDataCenter() == FAILED: self.__logger.error("=== Removing DataCenter Failed ===") else: self.__logger.info("=== Removing DataCenter Successful ===") except Exception as e: self.__logger.exception("=== Clean up failed: %s ===" % e) finally: exit(1) def __addToCleanUp(self, type, id): if type not in self.__cleanUp.keys(): self.__cleanUp[type] = [] self.__cleanUp[type].append(id) if "order" not in self.__cleanUp.keys(): self.__cleanUp["order"] = [] if type not in self.__cleanUp["order"]: self.__cleanUp["order"].append(type) def addHosts(self, hosts, zoneId, podId, clusterId, hypervisor): if hosts is None: self.__logger.warn("=== Invalid Hosts Information ===") return failed_cnt = 0 for host in hosts: try: hostcmd = addHost.addHostCmd() hostcmd.clusterid = clusterId hostcmd.hosttags = host.hosttags hostcmd.hypervisor = host.hypervisor hostcmd.password = host.password hostcmd.podid = podId hostcmd.url = host.url hostcmd.username = host.username hostcmd.zoneid = zoneId hostcmd.hypervisor = hypervisor ret = self.__apiClient.addHost(hostcmd) if ret: self.__logger.info("=== Add Host Successful ===") self.__addToCleanUp("Host", ret[0].id) except Exception as e: failed_cnt += 1 self.__logger.exception("=== Adding Host (%s) Failed: %s ===" % (str(host.url), e)) if failed_cnt == len(hosts): self.__cleanAndExit() def createClusters(self, clusters, zoneId, podId, vmwareDc=None): try: if clusters is None: return for cluster in clusters: clustercmd = addCluster.addClusterCmd() clustercmd.clustername = cluster.clustername clustercmd.clustertype = cluster.clustertype clustercmd.hypervisor = cluster.hypervisor clustercmd.password = cluster.password clustercmd.podid = podId clustercmd.url = cluster.url clustercmd.username = cluster.username clustercmd.zoneid = zoneId clusterresponse = self.__apiClient.addCluster(clustercmd) if clusterresponse[0].id: clusterId = clusterresponse[0].id self.__logger.debug( "Cluster Name : %s Id : %s Created Successfully" % (str(cluster.clustername), str(clusterId))) self.__addToCleanUp("Cluster", clusterId) self.addHosts(cluster.hosts, zoneId, podId, clusterId, cluster.hypervisor) self.waitForHost(zoneId, clusterId) self.createPrimaryStorages(cluster.primaryStorages, zoneId, podId, clusterId) except Exception as e: self.__logger.exception("=== Cluster %s Creation Failed: %s ===" % (str(cluster.clustername), e)) self.__cleanAndExit() def waitForHost(self, zoneId, clusterId): """ Wait for the hosts in the zoneid, clusterid to be up 2 retries with 30s delay """ try: retry, timeout = 2, 30 cmd = listHosts.listHostsCmd() cmd.clusterid, cmd.zoneid = clusterId, zoneId hosts = self.__apiClient.listHosts(cmd) while retry != 0: for host in hosts: if host.state != 'Up': break sleep(timeout) retry -= 1 except Exception as e: self.__logger.exception("=== List Hosts Failed: %s ===" % e) self.__cleanAndExit() def createPrimaryStorages(self, primaryStorages, zoneId, podId=None, clusterId=None): try: if primaryStorages is None: return for primary in primaryStorages: primarycmd = createStoragePool.createStoragePoolCmd() if primary.details: for key, value in vars(primary.details).iteritems(): primarycmd.details.append({key: value}) primarycmd.name = primary.name primarycmd.tags = primary.tags primarycmd.url = primary.url if primary.scope == 'zone' or clusterId is None: primarycmd.scope = 'zone' primarycmd.hypervisor = primary.hypervisor else: primarycmd.podid = podId primarycmd.clusterid = clusterId primarycmd.zoneid = zoneId ret = self.__apiClient.createStoragePool(primarycmd) if ret.id: self.__logger.info("=== Creating Storage Pool Successful ===") self.__addToCleanUp("StoragePool", ret.id) except Exception as e: self.__logger.exception("=== Create Storage Pool Failed: %s ===" % e) self.__cleanAndExit() def createPods(self, pods, zoneId, networkId=None): try: if pods is None: return for pod in pods: createpod = createPod.createPodCmd() createpod.name = pod.name createpod.gateway = pod.gateway createpod.netmask = pod.netmask createpod.startip = pod.startip createpod.endip = pod.endip createpod.zoneid = zoneId createpodResponse = self.__apiClient.createPod(createpod) if createpodResponse.id: podId = createpodResponse.id self.__logger.debug("Pod Name : %s Id : %s Created Successfully" % (str(pod.name), str(podId))) self.__addToCleanUp("Pod", podId) if pod.guestIpRanges is not None and networkId is not None: self.createVlanIpRanges("Basic", pod.guestIpRanges, zoneId, podId, networkId) self.createClusters(pod.clusters, zoneId, podId, vmwareDc=pod.vmwaredc) except Exception as e: self.__logger.exception("=== Pod: %s Creation Failed: %s ===" % (str(pod.name), e)) self.__cleanAndExit() def createVlanIpRanges(self, mode, ipranges, zoneId, podId=None, networkId=None, forvirtualnetwork=None): try: if ipranges is None: return for iprange in ipranges: vlanipcmd = createVlanIpRange.createVlanIpRangeCmd() vlanipcmd.account = iprange.account vlanipcmd.domainid = iprange.domainid vlanipcmd.endip = iprange.endip vlanipcmd.gateway = iprange.gateway vlanipcmd.netmask = iprange.netmask vlanipcmd.networkid = networkId vlanipcmd.podid = podId vlanipcmd.startip = iprange.startip vlanipcmd.zoneid = zoneId vlanipcmd.vlan = iprange.vlan if mode == "Basic": if forvirtualnetwork: vlanipcmd.forvirtualnetwork = "true" else: vlanipcmd.forvirtualnetwork = "false" else: vlanipcmd.forvirtualnetwork = "true" ret = self.__apiClient.createVlanIpRange(vlanipcmd) if ret.id: self.__logger.info("=== Creating Vlan Ip Range Successful ===") self.__addToCleanUp("VlanIpRange", ret.id) except Exception as e: self.__logger.exception("=== Create Vlan Ip Range Failed: %s ===" % e) self.__cleanAndExit() def createSecondaryStorages(self, secondaryStorages, zoneId): try: if secondaryStorages is None: return for secondary in secondaryStorages: secondarycmd = addImageStore.addImageStoreCmd() secondarycmd.url = secondary.url secondarycmd.provider = secondary.provider secondarycmd.details = [] if secondarycmd.provider.lower() in ('s3', "swift", "smb"): for key, value in vars(secondary.details).iteritems(): secondarycmd.details.append({ 'key': key, 'value': value }) if secondarycmd.provider.lower() in ("nfs", "smb"): secondarycmd.zoneid = zoneId ret = self.__apiClient.addImageStore(secondarycmd) if ret.id: self.__logger.info("=== Add Image Store Successful ===") self.__addToCleanUp("ImageStore", ret.id) except Exception as e: self.__logger.exception("=== Add Image Store Failed: %s ===" % e) self.__cleanAndExit() def createCacheStorages(self, cacheStorages, zoneId): try: if cacheStorages is None: return for cache in cacheStorages: cachecmd = createSecondaryStagingStore.createSecondaryStagingStoreCmd() cachecmd.url = cache.url cachecmd.provider = cache.provider cachecmd.zoneid = zoneId cachecmd.details = [] if cache.details: for key, value in vars(cache.details).iteritems(): cachecmd.details.append({ 'key': key, 'value': value }) ret = self.__apiClient.createSecondaryStagingStore(cachecmd) if ret.id: self.__logger.info("=== Creating Secondary StagingStore Successful ===") self.__addToCleanUp("SecondaryStagingStore", ret.id) except Exception as e: self.__logger.exception("=== Creating Secondary Staging Storage Failed: %s ===" % e) self.__cleanAndExit() def createNetworks(self, networks, zoneId): try: if networks is None: return for network in networks: networkcmd = createNetwork.createNetworkCmd() networkcmd.displaytext = network.displaytext networkcmd.name = network.name networkcmd.networkofferingid = network.networkofferingid networkcmd.zoneid = zoneId ipranges = network.ipranges if ipranges: iprange = ipranges.pop() networkcmd.startip = iprange.startip networkcmd.endip = iprange.endip networkcmd.gateway = iprange.gateway networkcmd.netmask = iprange.netmask networkcmdresponse = self.__apiClient.createNetwork(networkcmd) if networkcmdresponse.id: networkId = networkcmdresponse.id self.__logger.info( "=== Creating Network Name : %s Id : %s Successful ===" % (str(network.name), str(networkId))) self.__addToCleanUp("Network", networkId) return networkId except Exception as e: self.__logger.exception("=== Network : %s Creation Failed: %s ===" % (str(network.name), e)) self.__cleanAndExit() def createPhysicalNetwork(self, net, zoneid): try: phynet = createPhysicalNetwork.createPhysicalNetworkCmd() phynet.zoneid = zoneid phynet.name = net.name phynet.isolationmethods = net.isolationmethods phynetwrk = self.__apiClient.createPhysicalNetwork(phynet) if phynetwrk.id: self.__logger.info("=== Creating Physical Network Name : %s Id : %s Successful ===" % ( str(phynet.name), str(phynetwrk.id))) self.__addToCleanUp("PhysicalNetwork", phynetwrk.id) self.addTrafficTypes(phynetwrk.id, net.traffictypes) return phynetwrk except Exception as e: self.__logger.exception("=== Physical Network Creation Failed: %s ===" % e) self.__cleanAndExit() def updatePhysicalNetwork(self, networkid, state="Enabled", vlan=None): try: upnet = updatePhysicalNetwork.updatePhysicalNetworkCmd() upnet.id = networkid upnet.state = state if vlan: upnet.vlan = vlan ret = self.__apiClient.updatePhysicalNetwork(upnet) return ret except Exception as e: self.__logger.exception("=== Update Physical Network Failed: %s ===" % e) self.__cleanAndExit() def enableProvider(self, provider_id): try: upnetprov = \ updateNetworkServiceProvider.updateNetworkServiceProviderCmd() upnetprov.id = provider_id upnetprov.state = "Enabled" ret = self.__apiClient.updateNetworkServiceProvider(upnetprov) if ret.id: self.__logger.info("=== Update Network Service Provider Successfull ===") except Exception as e: self.__logger.exception("=== Update Network Service Provider Failed: %s ===" % e) self.__cleanAndExit() def configureProviders(self, phynetwrk, providers): """ We will enable the virtualrouter elements for all zones. """ try: for provider in providers: pnetprov = listNetworkServiceProviders.listNetworkServiceProvidersCmd() pnetprov.physicalnetworkid = phynetwrk.id pnetprov.state = "Disabled" pnetprov.name = provider.name pnetprovres = self.__apiClient.listNetworkServiceProviders( pnetprov) if pnetprovres and len(pnetprovres) > 0: if provider.name == 'VirtualRouter' or provider.name == 'VpcVirtualRouter': vrprov = listVirtualRouterElements.listVirtualRouterElementsCmd() vrprov.nspid = pnetprovres[0].id vrprovresponse = self.__apiClient.listVirtualRouterElements(vrprov) vrprovid = vrprovresponse[0].id vrconfig = configureVirtualRouterElement.configureVirtualRouterElementCmd() vrconfig.enabled = "true" vrconfig.id = vrprovid self.__apiClient.configureVirtualRouterElement(vrconfig) self.enableProvider(pnetprovres[0].id) elif provider.name == 'InternalLbVm': internallbprov = listInternalLoadBalancerElements.listInternalLoadBalancerElementsCmd() internallbprov.nspid = pnetprovres[0].id internallbresponse = self.__apiClient.listInternalLoadBalancerElements(internallbprov) internallbid = internallbresponse[0].id internallbconfig = configureInternalLoadBalancerElement.configureInternalLoadBalancerElementCmd() internallbconfig.enabled = "true" internallbconfig.id = internallbid self.__apiClient.configureInternalLoadBalancerElement(internallbconfig) self.enableProvider(pnetprovres[0].id) elif provider.name == 'SecurityGroupProvider': self.enableProvider(pnetprovres[0].id) elif provider.name in ['NiciraNvp']: netprov = addNetworkServiceProvider.addNetworkServiceProviderCmd() netprov.name = provider.name netprov.physicalnetworkid = phynetwrk.id result = self.__apiClient.addNetworkServiceProvider( netprov) if result.id: self.__logger.info("=== AddNetworkServiceProvider Successful ===") self.__addToCleanUp("NetworkServiceProvider", result.id) if provider.devices is not None: for device in provider.devices: if provider.name == 'NiciraNvp': cmd = addNiciraNvpDevice.addNiciraNvpDeviceCmd() cmd.hostname = device.hostname cmd.username = device.username cmd.password = device.password cmd.transportzoneuuid = device.transportzoneuuid cmd.physicalnetworkid = phynetwrk.id ret = self.__apiClient.addNiciraNvpDevice(cmd) self.__logger.info("=== Add NiciraNvp Successful ===") self.__addToCleanUp("NiciraNvp", ret.id) else: raise InvalidParameterException( "Device %s doesn't match any know provider type" % device) self.enableProvider(result.id) except Exception as e: self.__logger.exception("=== List Network Service Providers Failed: %s ===" % e) self.__cleanAndExit() def addTrafficTypes(self, physical_network_id, traffictypes): [self.addTrafficType(physical_network_id, traffic_type) for traffic_type in traffictypes] def addTrafficType(self, physical_network_id, traffictype): try: traffic_type = addTrafficType.addTrafficTypeCmd() traffic_type.physicalnetworkid = physical_network_id traffic_type.traffictype = traffictype.typ traffic_type.kvmnetworklabel = traffictype.kvm \ if traffictype.kvm is not None else None traffic_type.xennetworklabel = traffictype.xen \ if traffictype.xen is not None else None traffic_type.vmwarenetworklabel = traffictype.vmware \ if traffictype.vmware is not None else None traffic_type.simulatorlabel = traffictype.simulator \ if traffictype.simulator is not None else None ret = self.__apiClient.addTrafficType(traffic_type) if ret.id: self.__logger.info("=== Add TrafficType Successful ===") self.__addToCleanUp("TrafficType", ret.id) return ret except Exception as e: self.__logger.exception("=== Add TrafficType Failed: %s ===" % e) self.__cleanAndExit() def enableZone(self, zoneid, allocation_state="Enabled"): try: zoneCmd = updateZone.updateZoneCmd() zoneCmd.id = zoneid zoneCmd.allocationstate = allocation_state ret = self.__apiClient.updateZone(zoneCmd) if ret.id: self.__logger.info("=== Enable Zone Successful ===") return ret except Exception as e: self.__logger.exception("=== Enable Zone Failed: %s ===" % e) self.__cleanAndExit() def updateZoneDetails(self, zoneid, details): try: zoneCmd = updateZone.updateZoneCmd() zoneCmd.id = zoneid zoneCmd.details = details ret = self.__apiClient.updateZone(zoneCmd) if ret.id: self.__logger.info("=== Update Zone Successful ===") return ret except Exception as e: self.__logger.exception("=== Update Zone Failed: %s ===" % e) self.__cleanAndExit() def createZone(self, zone, rec=0): try: zoneresponse = self.__apiClient.createZone(zone) if zoneresponse.id: self.__logger.info("=== Create Zone Successful ===") self.__logger.debug("Zone Name : %s Id : %s " % (str(zone.name), str(zoneresponse.id))) self.__addToCleanUp("Zone", zoneresponse.id) return zoneresponse.id else: self.__logger.exception("=== Zone : %s Creation Failed ===" % str(zone.name)) if not rec: zone.name = zone.name + "_" + random_gen() self.__logger.info("=== Recreating Zone With New Name : %s ===" % zone.name) return self.createZone(zone, 1) except Exception as e: self.__logger.exception("=== Create Zone Failed: %s ===" % e) return FAILED def createZones(self, zones): try: for zone in zones: zonecmd = createZone.createZoneCmd() zonecmd.dns1 = zone.dns1 zonecmd.dns2 = zone.dns2 zonecmd.internaldns1 = zone.internaldns1 zonecmd.internaldns2 = zone.internaldns2 zonecmd.name = zone.name zonecmd.securitygroupenabled = zone.securitygroupenabled zonecmd.localstorageenabled = zone.localstorageenabled zonecmd.networktype = zone.networktype zonecmd.domain = zone.domain if zone.securitygroupenabled != "true": zonecmd.guestcidraddress = zone.guestcidraddress zoneId = self.createZone(zonecmd) if zoneId == FAILED: self.__logger.error("=== Zone: %s Creation Failed. So Exiting ===" % str(zone.name)) self.__cleanAndExit() for pnet in zone.physical_networks: phynetwrk = self.createPhysicalNetwork(pnet, zoneId) self.configureProviders(phynetwrk, pnet.providers) self.updatePhysicalNetwork(phynetwrk.id, "Enabled", vlan=pnet.vlan) if zone.networktype == "Basic": listnetworkoffering = listNetworkOfferings.listNetworkOfferingsCmd() if len(filter(lambda x: x.typ == 'Public', zone.physical_networks[0].traffictypes)) > 0: listnetworkoffering.name = "DefaultSharedNetscalerEIPandELBNetworkOffering" else: listnetworkoffering.name = "DefaultSharedNetworkOfferingWithSGService" if zone.networkofferingname is not None: listnetworkoffering.name = zone.networkofferingname listnetworkofferingresponse = self.__apiClient.listNetworkOfferings(listnetworkoffering) guestntwrk = configGenerator.network() guestntwrk.displaytext = "guestNetworkForBasicZone" guestntwrk.name = "guestNetworkForBasicZone" guestntwrk.zoneid = zoneId guestntwrk.networkofferingid = listnetworkofferingresponse[0].id networkid = self.createNetworks([guestntwrk], zoneId) self.createPods(zone.pods, zoneId, networkid) if self.isEipElbZone(zone): self.createVlanIpRanges(zone.networktype, zone.ipranges, zoneId, forvirtualnetwork=True) isPureAdvancedZone = (zone.networktype == "Advanced" and zone.securitygroupenabled != "true") if isPureAdvancedZone: self.createPods(zone.pods, zoneId) self.createVlanIpRanges(zone.networktype, zone.ipranges, zoneId) elif zone.networktype == "Advanced" and zone.securitygroupenabled == "true": listnetworkoffering = listNetworkOfferings.listNetworkOfferingsCmd() listnetworkoffering.name = "DefaultSharedNetworkOfferingWithSGService" if zone.networkofferingname is not None: listnetworkoffering.name = zone.networkofferingname listnetworkofferingresponse = self.__apiClient.listNetworkOfferings(listnetworkoffering) networkcmd = createNetwork.createNetworkCmd() networkcmd.displaytext = "Shared SG enabled network" networkcmd.name = "Shared SG enabled network" networkcmd.networkofferingid = listnetworkofferingresponse[0].id networkcmd.zoneid = zoneId ipranges = zone.ipranges if ipranges: iprange = ipranges.pop() networkcmd.startip = iprange.startip networkcmd.endip = iprange.endip networkcmd.gateway = iprange.gateway networkcmd.netmask = iprange.netmask networkcmd.vlan = iprange.vlan networkcmdresponse = self.__apiClient.createNetwork(networkcmd) if networkcmdresponse.id: self.__addToCleanUp("Network", networkcmdresponse.id) self.__logger.debug("create Network Successful. NetworkId : %s " % str(networkcmdresponse.id)) self.createPods(zone.pods, zoneId, networkcmdresponse.id) '''Note: Swift needs cache storage first''' self.createCacheStorages(zone.cacheStorages, zoneId) self.createSecondaryStorages(zone.secondaryStorages, zoneId) if zone.primaryStorages: self.createPrimaryStorages(zone.primaryStorages, zoneId) enabled = getattr(zone, 'enabled', 'True') if enabled == 'True' or enabled is None: self.enableZone(zoneId, "Enabled") details = getattr(zone, 'details') if details is not None: det = [d.__dict__ for d in details] self.updateZoneDetails(zoneId, det) return except Exception as e: self.__logger.exception("=== Create Zones Failed: %e ===" % e) @staticmethod @staticmethod def isEipElbZone(): if zone.networktype == "Basic" and len( filter(lambda x: x.typ == 'Public', zone.physical_networks[0].traffictypes)) > 0: return True return False def setClient(self): """ @Name : setClient @Desc : Sets the API Client retrieved from test client """ self.__apiClient = self.__test_client.getApiClient() def updateConfiguration(self, globalCfg): try: if globalCfg is None or self.__apiClient is None: return None for config in globalCfg: updateCfg = updateConfiguration.updateConfigurationCmd() updateCfg.name = config.name updateCfg.value = config.value ret = self.__apiClient.updateConfiguration(updateCfg) if ret.id: self.__logger.info("=== Update Configuration Successfull ===") except Exception as e: self.__logger.exception("=== Update Configuration Failed: %s ===" % e) self.__cleanAndExit() @staticmethod def copyAttributesToCommand(source, command): map(lambda attr: setattr(command, attr, getattr(source, attr, None)), filter(lambda attr: not attr.startswith("__") and attr not in ["required", "isAsync"], dir(command))) def deploy(self): try: self.__logger.info("=== Deploy DC Started ===") ''' Step1 : Set the Client ''' self.setClient() ''' Step2: Update the Configuration ''' self.updateConfiguration(self.__config.globalConfig) ''' Step3 :Deploy the Zone ''' self.createZones(self.__config.zones) ''' Persist the Configuration to an external file post DC creation ''' self.__persistDcConfig() self.__logger.info("=== Deploy DC Successful ===") return SUCCESS except Exception as e: self.__logger.exception("=== Deploy DC Failed: %s ===" % e) self.__cleanAndExit() return FAILED
return self.__option_is_set(options.remove) and self.__path_points_to_file(options.remove) def __options_for_remove_data_center_are_invalid(self, options): return self.__option_is_set(options.remove) and not self.__path_points_to_file(options.remove) @staticmethod def __deploy_and_remove_data_center(options): return options.input and options.remove @staticmethod def __nothing_to_do(options): return options.input is None and options.remove is None @staticmethod def __option_is_set(flag): return flag is not None @staticmethod def __path_points_to_file(path): return os.path.isfile(path) if __name__ == "__main__": logger = MarvinLog('marvin').getLogger() try: Application().main(sys.argv[1:]) except Exception as e: logger.exception("Aborting run due to: %s" % e) exit(1) exit(0)
class SshClient(object): ''' @Desc : SSH Library for Marvin. Facilitates SSH,SCP services to marvin users @Input: host: Host to connect port: port on host to connect user: Username to be used for connecting passwd: Password for connection retries and delay applies for establishing connection timeout : Applies while executing command ''' def __init__(self, host, port, user, passwd, retries=60, delay=10, log_lvl=logging.INFO, keyPairFiles=None, timeout=10.0): self.host = None self.port = 22 self.user = user self.passwd = passwd self.keyPairFiles = keyPairFiles self.ssh = SSHClient() self.ssh.set_missing_host_key_policy(AutoAddPolicy()) self.retryCnt = 0 self.delay = 0 self.timeout = 3.0 self.logger = MarvinLog('ssh').getLogger() # Check invalid host value and raise exception # Atleast host is required for connection if host is not None and host != '': self.host = host if retries is not None and retries > 0: self.retryCnt = retries if delay is not None and delay > 0: self.delay = delay if timeout is not None and timeout > 0: self.timeout = timeout if port is not None and port >= 0: self.port = port if self.createConnection() == FAILED: raise internalError("Connection Failed") def execute(self, command): stdin, stdout, stderr = self.ssh.exec_command(command) output = stdout.readlines() errors = stderr.readlines() results = [] if output is not None and len(output) == 0: if errors is not None and len(errors) > 0: for error in errors: results.append(error.rstrip()) else: for strOut in output: results.append(strOut.rstrip()) self.logger.debug("Executing command via host %s: %s Output: %s" % (str(self.host), command, results)) return results def createConnection(self): ''' @Name: createConnection @Desc: Creates an ssh connection for retries mentioned,along with sleep mentioned @Output: SUCCESS on successful connection FAILED If connection through ssh failed ''' ret = FAILED while self.retryCnt >= 0: try: self.logger.debug( "Trying SSH Connection to host %s on port %s as user %s. RetryCount: %s" % (self.host, str(self.port), self.user, str(self.retryCnt))) if self.keyPairFiles is None: self.ssh.connect(hostname=self.host, port=self.port, username=self.user, password=self.passwd, timeout=self.timeout) else: self.ssh.connect(hostname=self.host, port=self.port, username=self.user, password=self.passwd, key_filename=self.keyPairFiles, timeout=self.timeout, look_for_keys=False) self.logger.debug( "Connection to host %s on port %s is SUCCESSFUL" % (str(self.host), str(self.port))) ret = SUCCESS break except BadHostKeyException as e: self.logger.debug("Failed to create connection: %s" % e) except AuthenticationException as e: self.logger.debug("Failed to create connection: %s" % e) except SSHException as e: self.logger.debug("Failed to create connection: %s" % e) except socket.error as e: self.logger.debug("Failed to create connection: %s" % e) except Exception as e: self.logger.debug("Failed to create connection: %s" % e) finally: if self.retryCnt == 0 or ret == SUCCESS: break self.retryCnt -= 1 time.sleep(self.delay) return ret def runCommand(self, command): ''' @Name: runCommand @Desc: Runs a command over ssh and returns the result along with status code @Input: command to execute @Output: 1: status of command executed. SUCCESS : If command execution is successful FAILED : If command execution has failed 2: stdin,stdout,stderr values of command output ''' ret = { "status": FAILED, "stdin": None, "stdout": None, "stderr": INVALID_INPUT } if command is None or command == '': return ret try: stdin, stdout, stderr = self.ssh.exec_command(command, timeout=self.timeout) if stdout is not None: status_check = stdout.channel.recv_exit_status() if status_check == 0: ret["status"] = SUCCESS ret["stdout"] = stdout.readlines() if stderr is not None: ret["stderr"] = stderr.readlines() except Exception as e: self.logger.debug("Failed to run command: %s" % e) finally: self.logger.debug( "Connection to host %s on port %s is SUCCESSFUL" % (str(self.host), command)) return ret def scp(self, srcFile, destPath): transport = Transport((self.host, int(self.port))) transport.connect(username=self.user, password=self.passwd) sftp = SFTPClient.from_transport(transport) try: sftp.put(srcFile, destPath) except IOError as e: raise e def __del__(self): self.close() def close(self): if self.ssh is not None: self.ssh.close() self.ssh = None
def __init__(self, test_client, cfg): self.__cfg = cfg self.__test_client = test_client self.__logger = MarvinLog('marvin').getLogger() self.__apiClient = None
class DeleteDataCenters: """ @Desc : Deletes the Data Center using the settings provided. test_client :Client for deleting the DC. dc_cfg_file : obj file exported by DeployDataCenter when successfully created DC. This file is serialized one containing entries with successful DC. dc_cfg: If dc_cfg_file, is not available, we can use the dictionary of elements to delete. tc_run_logger: Logger to dump log messages. """ def __init__(self, test_client, cfg): self.__cfg = cfg self.__test_client = test_client self.__logger = MarvinLog('marvin').getLogger() self.__apiClient = None def __deleteCmds(self, cmd_name, cmd_obj): """ @Name : __deleteCmds @Desc : Deletes the entities provided by cmd """ if cmd_name.lower() == "deletehostcmd": cmd_obj.forcedestroylocalstorage = "true" cmd_obj.force = "true" ''' Step1 : Prepare Host For Maintenance ''' host_maint_cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd() host_maint_cmd.id = cmd_obj.id host_maint_resp = self.__apiClient.prepareHostForMaintenance(host_maint_cmd) if host_maint_resp: ''' Step2 : List Hosts for Resource State ''' list_host_cmd = listHosts.listHostsCmd() list_host_cmd.id = cmd_obj.id retries = 3 for i in xrange(retries): list_host_resp = self.__apiClient.listHosts(list_host_cmd) if list_host_resp and (list_host_resp[0].resourcestate == 'Maintenance'): break sleep(30) if cmd_name.lower() == "deletestoragepoolcmd": cmd_obj.forced = "true" store_maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() store_maint_cmd.id = cmd_obj.id store_maint_resp = self.__apiClient.enableStorageMaintenance(store_maint_cmd) if store_maint_resp: list_store_cmd = listStoragePools.listStoragePoolsCmd() list_store_cmd.id = cmd_obj.id retries = 3 for i in xrange(retries): store_maint_resp = self.__apiClient.listStoragePools(list_store_cmd) if store_maint_resp and (store_maint_resp[0].state == 'Maintenance'): break sleep(30) return cmd_obj def __setClient(self): self.__apiClient = self.__test_client.getApiClient() def __cleanEntries(self): """ @Name : __cleanAndEntries @Description: Cleans up the created DC in order of creation """ try: ret = FAILED if "order" in self.__cfg.keys() and len(self.__cfg["order"]): self.__cfg["order"].reverse() self.__logger.info("=== Clean Up Entries: %s ===" % self.__cfg) for type in self.__cfg["order"]: self.__logger.info("=== CleanUp Started For Type: %s ===" % type) if type: temp_ids = self.__cfg[type] ids = [items for items in temp_ids if items] for id in ids: del_mod = "delete" + type del_cmd = getattr(globals()[del_mod], del_mod + "Cmd") del_cmd_obj = del_cmd() del_cmd_obj.id = id del_cmd_obj = self.__deleteCmds(del_mod + "Cmd", del_cmd_obj) del_func = getattr(self.__apiClient, del_mod) del_cmd_resp = del_func(del_cmd_obj) if del_cmd_resp: self.__logger.error("=== %s CleanUp Failed. ID: %s ===" % (type, id)) else: self.__logger.info("=== %s CleanUp Successful. ID : %s ===" % (type, id)) ret = SUCCESS except Exception as e: self.__logger.exception("=== Clean Up Entries failed: %s ===" % e) finally: return ret def removeDataCenter(self): """ @Name : removeDataCenter @Desc : Removes the Data Center provided by Configuration If Input dc file configuration is None, uses the cfg provided else uses the dc file to get the configuration """ try: self.__setClient() self.__logger.info("=== DeployDC: CleanUp Started ===") ret = self.__cleanEntries() except Exception as e: self.__logger.exception("=== DeployDC: CleanUp failed: %s ===" % e) finally: return ret
class DeployDataCenters(object): """ @Desc : Deploys the Data Center with information provided. Once the Deployment is successful, it will export the DataCenter settings to an obj file ( can be used if wanted to delete the created DC) """ def __init__(self, test_client, cfg): self.__test_client = test_client self.__config = cfg self.__logger = MarvinLog('marvin').getLogger() self.__apiClient = None self.__cleanUp = {} def __persistDcConfig(self): try: ts = strftime("%b_%d_%Y_%H_%M_%S", localtime()) dc_file_path = "dc_entries_" + str(ts) + ".obj" file_to_write = open(dc_file_path, 'w') if file_to_write: pickle.dump(self.__cleanUp, file_to_write) self.__logger.info("=== Data Center Settings are dumped to %s ===" % dc_file_path) except Exception as e: self.__logger.exception("=== Persisting DataCenter config failed: %s ===" % e) def __cleanAndExit(self): try: self.__logger.info("=== deploy dc failed, so cleaning the created entries ===") if not test_data.get("deleteDC", None): self.__logger.info("=== Deploy DC Clean Up flag not set. So, exiting ===") exit(1) self.__logger.info("=== Deploy DC Failed, So Cleaning to Exit ===") remove_dc_obj = DeleteDataCenters(self.__test_client, self.__cleanUp) if remove_dc_obj: if remove_dc_obj.removeDataCenter() == FAILED: self.__logger.error("=== Removing DataCenter Failed ===") else: self.__logger.info("=== Removing DataCenter Successful ===") except Exception as e: self.__logger.exception("=== Clean up failed: %s ===" % e) finally: exit(1) def __addToCleanUp(self, type, id): if type not in self.__cleanUp.keys(): self.__cleanUp[type] = [] self.__cleanUp[type].append(id) if "order" not in self.__cleanUp.keys(): self.__cleanUp["order"] = [] if type not in self.__cleanUp["order"]: self.__cleanUp["order"].append(type) def addHosts(self, hosts, zoneId, podId, clusterId, hypervisor): if hosts is None: self.__logger.warn("=== Invalid Hosts Information ===") return failed_cnt = 0 for host in hosts: try: hostcmd = addHost.addHostCmd() hostcmd.clusterid = clusterId hostcmd.hosttags = host.hosttags hostcmd.hypervisor = host.hypervisor hostcmd.password = host.password hostcmd.podid = podId hostcmd.url = host.url hostcmd.username = host.username hostcmd.zoneid = zoneId hostcmd.hypervisor = hypervisor ret = self.__apiClient.addHost(hostcmd) if ret: self.__logger.info("=== Add Host Successful ===") self.__addToCleanUp("Host", ret[0].id) except Exception as e: failed_cnt += 1 self.__logger.exception("=== Adding Host (%s) Failed: %s ===" % (str(host.url), e)) if failed_cnt == len(hosts): self.__cleanAndExit() def createClusters(self, clusters, zoneId, podId, vmwareDc=None): try: if clusters is None: return for cluster in clusters: clustercmd = addCluster.addClusterCmd() clustercmd.clustername = cluster.clustername clustercmd.clustertype = cluster.clustertype clustercmd.hypervisor = cluster.hypervisor clustercmd.password = cluster.password clustercmd.podid = podId clustercmd.url = cluster.url clustercmd.username = cluster.username clustercmd.zoneid = zoneId clusterresponse = self.__apiClient.addCluster(clustercmd) if clusterresponse[0].id: clusterId = clusterresponse[0].id self.__logger.debug( "Cluster Name : %s Id : %s Created Successfully" % (str(cluster.clustername), str(clusterId))) self.__addToCleanUp("Cluster", clusterId) self.addHosts(cluster.hosts, zoneId, podId, clusterId, cluster.hypervisor) self.waitForHost(zoneId, clusterId) self.createPrimaryStorages(cluster.primaryStorages, zoneId, podId, clusterId) except Exception as e: self.__logger.exception("=== Cluster %s Creation Failed: %s ===" % (str(cluster.clustername), e)) self.__cleanAndExit() def waitForHost(self, zoneId, clusterId): """ Wait for the hosts in the zoneid, clusterid to be up 2 retries with 30s delay """ try: retry, timeout = 2, 30 cmd = listHosts.listHostsCmd() cmd.clusterid, cmd.zoneid = clusterId, zoneId hosts = self.__apiClient.listHosts(cmd) while retry != 0: for host in hosts: if host.state != 'Up': break sleep(timeout) retry -= 1 except Exception as e: self.__logger.exception("=== List Hosts Failed: %s ===" % e) self.__cleanAndExit() def createPrimaryStorages(self, primaryStorages, zoneId, podId=None, clusterId=None): try: if primaryStorages is None: return for primary in primaryStorages: primarycmd = createStoragePool.createStoragePoolCmd() if primary.details: for key, value in vars(primary.details).iteritems(): primarycmd.details.append({key: value}) primarycmd.name = primary.name primarycmd.tags = primary.tags primarycmd.url = primary.url if primary.scope == 'zone' or clusterId is None: primarycmd.scope = 'zone' primarycmd.hypervisor = primary.hypervisor else: primarycmd.podid = podId primarycmd.clusterid = clusterId primarycmd.zoneid = zoneId ret = self.__apiClient.createStoragePool(primarycmd) if ret.id: self.__logger.info("=== Creating Storage Pool Successful ===") self.__addToCleanUp("StoragePool", ret.id) except Exception as e: self.__logger.exception("=== Create Storage Pool Failed: %s ===" % e) self.__cleanAndExit() def createPods(self, pods, zoneId, networkId=None): try: if pods is None: return for pod in pods: createpod = createPod.createPodCmd() createpod.name = pod.name createpod.gateway = pod.gateway createpod.netmask = pod.netmask createpod.startip = pod.startip createpod.endip = pod.endip createpod.zoneid = zoneId createpodResponse = self.__apiClient.createPod(createpod) if createpodResponse.id: podId = createpodResponse.id self.__logger.debug("Pod Name : %s Id : %s Created Successfully" % (str(pod.name), str(podId))) self.__addToCleanUp("Pod", podId) if pod.guestIpRanges is not None and networkId is not None: self.createVlanIpRanges("Basic", pod.guestIpRanges, zoneId, podId, networkId) self.createClusters(pod.clusters, zoneId, podId, vmwareDc=pod.vmwaredc) except Exception as e: self.__logger.exception("=== Pod: %s Creation Failed: %s ===" % (str(pod.name), e)) self.__cleanAndExit() def createVlanIpRanges(self, mode, ipranges, zoneId, podId=None, networkId=None, forvirtualnetwork=None): try: if ipranges is None: return for iprange in ipranges: vlanipcmd = createVlanIpRange.createVlanIpRangeCmd() vlanipcmd.account = iprange.account vlanipcmd.domainid = iprange.domainid vlanipcmd.endip = iprange.endip vlanipcmd.gateway = iprange.gateway vlanipcmd.netmask = iprange.netmask vlanipcmd.networkid = networkId vlanipcmd.podid = podId vlanipcmd.startip = iprange.startip vlanipcmd.zoneid = zoneId vlanipcmd.vlan = iprange.vlan if mode == "Basic": if forvirtualnetwork: vlanipcmd.forvirtualnetwork = "true" else: vlanipcmd.forvirtualnetwork = "false" else: vlanipcmd.forvirtualnetwork = "true" ret = self.__apiClient.createVlanIpRange(vlanipcmd) if ret.id: self.__logger.info("=== Creating Vlan Ip Range Successful ===") self.__addToCleanUp("VlanIpRange", ret.id) except Exception as e: self.__logger.exception("=== Create Vlan Ip Range Failed: %s ===" % e) self.__cleanAndExit() def createSecondaryStorages(self, secondaryStorages, zoneId): try: if secondaryStorages is None: return for secondary in secondaryStorages: secondarycmd = addImageStore.addImageStoreCmd() secondarycmd.url = secondary.url secondarycmd.provider = secondary.provider secondarycmd.details = [] if secondarycmd.provider.lower() in ('s3', "swift", "smb"): for key, value in vars(secondary.details).iteritems(): secondarycmd.details.append({ 'key': key, 'value': value }) if secondarycmd.provider.lower() in ("nfs", "smb"): secondarycmd.zoneid = zoneId ret = self.__apiClient.addImageStore(secondarycmd) if ret.id: self.__logger.info("=== Add Image Store Successful ===") self.__addToCleanUp("ImageStore", ret.id) except Exception as e: self.__logger.exception("=== Add Image Store Failed: %s ===" % e) self.__cleanAndExit() def createCacheStorages(self, cacheStorages, zoneId): try: if cacheStorages is None: return for cache in cacheStorages: cachecmd = createSecondaryStagingStore.createSecondaryStagingStoreCmd() cachecmd.url = cache.url cachecmd.provider = cache.provider cachecmd.zoneid = zoneId cachecmd.details = [] if cache.details: for key, value in vars(cache.details).iteritems(): cachecmd.details.append({ 'key': key, 'value': value }) ret = self.__apiClient.createSecondaryStagingStore(cachecmd) if ret.id: self.__logger.info("=== Creating Secondary StagingStore Successful ===") self.__addToCleanUp("SecondaryStagingStore", ret.id) except Exception as e: self.__logger.exception("=== Creating Secondary Staging Storage Failed: %s ===" % e) self.__cleanAndExit() def createNetworks(self, networks, zoneId): try: if networks is None: return for network in networks: networkcmd = createNetwork.createNetworkCmd() networkcmd.displaytext = network.displaytext networkcmd.name = network.name networkcmd.networkofferingid = network.networkofferingid networkcmd.zoneid = zoneId ipranges = network.ipranges if ipranges: iprange = ipranges.pop() networkcmd.startip = iprange.startip networkcmd.endip = iprange.endip networkcmd.gateway = iprange.gateway networkcmd.netmask = iprange.netmask networkcmdresponse = self.__apiClient.createNetwork(networkcmd) if networkcmdresponse.id: networkId = networkcmdresponse.id self.__logger.info( "=== Creating Network Name : %s Id : %s Successful ===" % (str(network.name), str(networkId))) self.__addToCleanUp("Network", networkId) return networkId except Exception as e: self.__logger.exception("=== Network : %s Creation Failed: %s ===" % (str(network.name), e)) self.__cleanAndExit() def createPhysicalNetwork(self, net, zoneid): try: phynet = createPhysicalNetwork.createPhysicalNetworkCmd() phynet.zoneid = zoneid phynet.name = net.name phynet.isolationmethods = net.isolationmethods phynetwrk = self.__apiClient.createPhysicalNetwork(phynet) if phynetwrk.id: self.__logger.info("=== Creating Physical Network Name : %s Id : %s Successful ===" % ( str(phynet.name), str(phynetwrk.id))) self.__addToCleanUp("PhysicalNetwork", phynetwrk.id) self.addTrafficTypes(phynetwrk.id, net.traffictypes) return phynetwrk except Exception as e: self.__logger.exception("=== Physical Network Creation Failed: %s ===" % e) self.__cleanAndExit() def updatePhysicalNetwork(self, networkid, state="Enabled", vlan=None): try: upnet = updatePhysicalNetwork.updatePhysicalNetworkCmd() upnet.id = networkid upnet.state = state if vlan: upnet.vlan = vlan ret = self.__apiClient.updatePhysicalNetwork(upnet) return ret except Exception as e: self.__logger.exception("=== Update Physical Network Failed: %s ===" % e) self.__cleanAndExit() def enableProvider(self, provider_id): try: upnetprov = \ updateNetworkServiceProvider.updateNetworkServiceProviderCmd() upnetprov.id = provider_id upnetprov.state = "Enabled" ret = self.__apiClient.updateNetworkServiceProvider(upnetprov) if ret.id: self.__logger.info("=== Update Network Service Provider Successfull ===") except Exception as e: self.__logger.exception("=== Update Network Service Provider Failed: %s ===" % e) self.__cleanAndExit() def configureProviders(self, phynetwrk, providers): """ We will enable the virtualrouter elements for all zones. """ try: for provider in providers: pnetprov = listNetworkServiceProviders.listNetworkServiceProvidersCmd() pnetprov.physicalnetworkid = phynetwrk.id pnetprov.state = "Disabled" pnetprov.name = provider.name pnetprovres = self.__apiClient.listNetworkServiceProviders( pnetprov) if pnetprovres and len(pnetprovres) > 0: if provider.name == 'VirtualRouter' or provider.name == 'VpcVirtualRouter': vrprov = listVirtualRouterElements.listVirtualRouterElementsCmd() vrprov.nspid = pnetprovres[0].id vrprovresponse = self.__apiClient.listVirtualRouterElements(vrprov) vrprovid = vrprovresponse[0].id vrconfig = configureVirtualRouterElement.configureVirtualRouterElementCmd() vrconfig.enabled = "true" vrconfig.id = vrprovid self.__apiClient.configureVirtualRouterElement(vrconfig) self.enableProvider(pnetprovres[0].id) elif provider.name == 'InternalLbVm': internallbprov = listInternalLoadBalancerElements.listInternalLoadBalancerElementsCmd() internallbprov.nspid = pnetprovres[0].id internallbresponse = self.__apiClient.listInternalLoadBalancerElements(internallbprov) internallbid = internallbresponse[0].id internallbconfig = configureInternalLoadBalancerElement.configureInternalLoadBalancerElementCmd() internallbconfig.enabled = "true" internallbconfig.id = internallbid self.__apiClient.configureInternalLoadBalancerElement(internallbconfig) self.enableProvider(pnetprovres[0].id) elif provider.name == 'SecurityGroupProvider': self.enableProvider(pnetprovres[0].id) elif provider.name in ['NiciraNvp']: netprov = addNetworkServiceProvider.addNetworkServiceProviderCmd() netprov.name = provider.name netprov.physicalnetworkid = phynetwrk.id result = self.__apiClient.addNetworkServiceProvider( netprov) if result.id: self.__logger.info("=== AddNetworkServiceProvider Successful ===") self.__addToCleanUp("NetworkServiceProvider", result.id) if provider.devices is not None: for device in provider.devices: if provider.name == 'NiciraNvp': cmd = addNiciraNvpDevice.addNiciraNvpDeviceCmd() cmd.hostname = device.hostname cmd.username = device.username cmd.password = device.password cmd.transportzoneuuid = device.transportzoneuuid cmd.physicalnetworkid = phynetwrk.id ret = self.__apiClient.addNiciraNvpDevice(cmd) self.__logger.info("=== Add NiciraNvp Successful ===") self.__addToCleanUp("NiciraNvp", ret.id) else: raise InvalidParameterException( "Device %s doesn't match any know provider type" % device) self.enableProvider(result.id) except Exception as e: self.__logger.exception("=== List Network Service Providers Failed: %s ===" % e) self.__cleanAndExit() def addTrafficTypes(self, physical_network_id, traffictypes): [self.addTrafficType(physical_network_id, traffic_type) for traffic_type in traffictypes] def addTrafficType(self, physical_network_id, traffictype): try: traffic_type = addTrafficType.addTrafficTypeCmd() traffic_type.physicalnetworkid = physical_network_id traffic_type.traffictype = traffictype.typ traffic_type.kvmnetworklabel = traffictype.kvm \ if traffictype.kvm is not None else None traffic_type.xennetworklabel = traffictype.xen \ if traffictype.xen is not None else None traffic_type.vmwarenetworklabel = traffictype.vmware \ if traffictype.vmware is not None else None traffic_type.simulatorlabel = traffictype.simulator \ if traffictype.simulator is not None else None ret = self.__apiClient.addTrafficType(traffic_type) if ret.id: self.__logger.info("=== Add TrafficType Successful ===") self.__addToCleanUp("TrafficType", ret.id) return ret except Exception as e: self.__logger.exception("=== Add TrafficType Failed: %s ===" % e) self.__cleanAndExit() def enableZone(self, zoneid, allocation_state="Enabled"): try: zoneCmd = updateZone.updateZoneCmd() zoneCmd.id = zoneid zoneCmd.allocationstate = allocation_state ret = self.__apiClient.updateZone(zoneCmd) if ret.id: self.__logger.info("=== Enable Zone Successful ===") return ret except Exception as e: self.__logger.exception("=== Enable Zone Failed: %s ===" % e) self.__cleanAndExit() def updateZoneDetails(self, zoneid, details): try: zoneCmd = updateZone.updateZoneCmd() zoneCmd.id = zoneid zoneCmd.details = details ret = self.__apiClient.updateZone(zoneCmd) if ret.id: self.__logger.info("=== Update Zone Successful ===") return ret except Exception as e: self.__logger.exception("=== Update Zone Failed: %s ===" % e) self.__cleanAndExit() def createZone(self, zone, rec=0): try: zoneresponse = self.__apiClient.createZone(zone) if zoneresponse.id: self.__logger.info("=== Create Zone Successful ===") self.__logger.debug("Zone Name : %s Id : %s " % (str(zone.name), str(zoneresponse.id))) self.__addToCleanUp("Zone", zoneresponse.id) return zoneresponse.id else: self.__logger.exception("=== Zone : %s Creation Failed ===" % str(zone.name)) if not rec: zone.name = zone.name + "_" + random_gen() self.__logger.info("=== Recreating Zone With New Name : %s ===" % zone.name) return self.createZone(zone, 1) except Exception as e: self.__logger.exception("=== Create Zone Failed: %s ===" % e) return FAILED def createZones(self, zones): try: for zone in zones: zonecmd = createZone.createZoneCmd() zonecmd.dns1 = zone.dns1 zonecmd.dns2 = zone.dns2 zonecmd.internaldns1 = zone.internaldns1 zonecmd.internaldns2 = zone.internaldns2 zonecmd.name = zone.name zonecmd.securitygroupenabled = zone.securitygroupenabled zonecmd.localstorageenabled = zone.localstorageenabled zonecmd.networktype = zone.networktype zonecmd.domain = zone.domain if zone.securitygroupenabled != "true": zonecmd.guestcidraddress = zone.guestcidraddress zoneId = self.createZone(zonecmd) if zoneId == FAILED: self.__logger.error("=== Zone: %s Creation Failed. So Exiting ===" % str(zone.name)) self.__cleanAndExit() for pnet in zone.physical_networks: phynetwrk = self.createPhysicalNetwork(pnet, zoneId) self.configureProviders(phynetwrk, pnet.providers) self.updatePhysicalNetwork(phynetwrk.id, "Enabled", vlan=pnet.vlan) if zone.networktype == "Basic": listnetworkoffering = listNetworkOfferings.listNetworkOfferingsCmd() if len(filter(lambda x: x.typ == 'Public', zone.physical_networks[0].traffictypes)) > 0: listnetworkoffering.name = "DefaultSharedNetscalerEIPandELBNetworkOffering" else: listnetworkoffering.name = "DefaultSharedNetworkOfferingWithSGService" if zone.networkofferingname is not None: listnetworkoffering.name = zone.networkofferingname listnetworkofferingresponse = self.__apiClient.listNetworkOfferings(listnetworkoffering) guestntwrk = configGenerator.network() guestntwrk.displaytext = "guestNetworkForBasicZone" guestntwrk.name = "guestNetworkForBasicZone" guestntwrk.zoneid = zoneId guestntwrk.networkofferingid = listnetworkofferingresponse[0].id networkid = self.createNetworks([guestntwrk], zoneId) self.createPods(zone.pods, zoneId, networkid) if self.isEipElbZone(zone): self.createVlanIpRanges(zone.networktype, zone.ipranges, zoneId, forvirtualnetwork=True) isPureAdvancedZone = (zone.networktype == "Advanced" and zone.securitygroupenabled != "true") if isPureAdvancedZone: self.createPods(zone.pods, zoneId) self.createVlanIpRanges(zone.networktype, zone.ipranges, zoneId) elif zone.networktype == "Advanced" and zone.securitygroupenabled == "true": listnetworkoffering = listNetworkOfferings.listNetworkOfferingsCmd() listnetworkoffering.name = "DefaultSharedNetworkOfferingWithSGService" if zone.networkofferingname is not None: listnetworkoffering.name = zone.networkofferingname listnetworkofferingresponse = self.__apiClient.listNetworkOfferings(listnetworkoffering) networkcmd = createNetwork.createNetworkCmd() networkcmd.displaytext = "Shared SG enabled network" networkcmd.name = "Shared SG enabled network" networkcmd.networkofferingid = listnetworkofferingresponse[0].id networkcmd.zoneid = zoneId ipranges = zone.ipranges if ipranges: iprange = ipranges.pop() networkcmd.startip = iprange.startip networkcmd.endip = iprange.endip networkcmd.gateway = iprange.gateway networkcmd.netmask = iprange.netmask networkcmd.vlan = iprange.vlan networkcmdresponse = self.__apiClient.createNetwork(networkcmd) if networkcmdresponse.id: self.__addToCleanUp("Network", networkcmdresponse.id) self.__logger.debug("create Network Successful. NetworkId : %s " % str(networkcmdresponse.id)) self.createPods(zone.pods, zoneId, networkcmdresponse.id) '''Note: Swift needs cache storage first''' self.createCacheStorages(zone.cacheStorages, zoneId) self.createSecondaryStorages(zone.secondaryStorages, zoneId) if zone.primaryStorages: self.createPrimaryStorages(zone.primaryStorages, zoneId) enabled = getattr(zone, 'enabled', 'True') if enabled == 'True' or enabled is None: self.enableZone(zoneId, "Enabled") details = getattr(zone, 'details') if details is not None: det = [d.__dict__ for d in details] self.updateZoneDetails(zoneId, det) return except Exception as e: self.__logger.exception("=== Create Zones Failed: %e ===" % e) def isEipElbZone(self, zone): if zone.networktype == "Basic" and len( filter(lambda x: x.typ == 'Public', zone.physical_networks[0].traffictypes)) > 0: return True return False def setClient(self): """ @Name : setClient @Desc : Sets the API Client retrieved from test client """ self.__apiClient = self.__test_client.getApiClient() def updateConfiguration(self, globalCfg): try: if globalCfg is None or self.__apiClient is None: return None for config in globalCfg: updateCfg = updateConfiguration.updateConfigurationCmd() updateCfg.name = config.name updateCfg.value = config.value ret = self.__apiClient.updateConfiguration(updateCfg) if ret.id: self.__logger.info("=== Update Configuration Successfull ===") except Exception as e: self.__logger.exception("=== Update Configuration Failed: %s ===" % e) self.__cleanAndExit() @staticmethod def copyAttributesToCommand(source, command): map(lambda attr: setattr(command, attr, getattr(source, attr, None)), filter(lambda attr: not attr.startswith("__") and attr not in ["required", "isAsync"], dir(command))) def deploy(self): try: self.__logger.info("=== Deploy DC Started ===") ''' Step1 : Set the Client ''' self.setClient() ''' Step2: Update the Configuration ''' self.updateConfiguration(self.__config.globalConfig) ''' Step3 :Deploy the Zone ''' self.createZones(self.__config.zones) ''' Persist the Configuration to an external file post DC creation ''' self.__persistDcConfig() self.__logger.info("=== Deploy DC Successful ===") return SUCCESS except Exception as e: self.__logger.exception("=== Deploy DC Failed: %s ===" % e) self.__cleanAndExit() return FAILED
return self.__option_is_set(options.remove) and self.__path_points_to_file(options.remove) def __options_for_remove_data_center_are_invalid(self, options): return self.__option_is_set(options.remove) and not self.__path_points_to_file(options.remove) @staticmethod def __deploy_and_remove_data_center(options): return options.input and options.remove @staticmethod def __nothing_to_do(options): return options.input is None and options.remove is None @staticmethod def __option_is_set(flag): return flag is not None @staticmethod def __path_points_to_file(path): return os.path.isfile(path) if __name__ == "__main__": logger = MarvinLog('marvin').getLogger() try: Application().main(sys.argv[1:]) except Exception as e: logger.exception("Aborting run due to: %s" % e) exit(1) exit(0)
class CSTestClient(object): ''' @Desc : CloudStackTestClient is encapsulated entity for creating and getting various clients viz., apiclient, user api client, dbconnection, test Data parsed information etc @Input : mgmt_details : Management Server Details dbsvr_details: Database Server details of Management \ Server. Retrieved from configuration file. async_timeout : Timeout for Async queries default_worker_threads : Number of worker threads logger : provides logging facilities for this library zone : The zone on which test suites using this test client will run ''' def __init__(self, mgmt_details, dbsvr_details, async_timeout=3600, test_data_filepath=None, zone=None, hypervisor_type=None): self.__mgmtDetails = mgmt_details self.__dbSvrDetails = dbsvr_details self.__csConnection = None self.__dbConnection = None self.__testClient = None self.__asyncTimeOut = async_timeout self.__logger = MarvinLog('marvin').getLogger() self.__apiClient = None self.__userApiClient = None self.__asyncJobMgr = None self.__id = None self.__hypervisor = hypervisor_type self.__testDataFilePath = test_data_filepath self.__parsedTestDataConfig = None self.__zone = zone self.__setHypervisorInfo() @property def identifier(self): return self.__id @identifier.setter def identifier(self, id): self.__id = id def getParsedTestDataConfig(self): ''' @Name : getParsedTestDataConfig @Desc : Provides the TestData Config needed for Tests are to Run @Output : Returns the Parsed Test Data Dictionary ''' return copy.deepcopy(self.__parsedTestDataConfig) def getZoneForTests(self): ''' @Name : getZoneForTests @Desc : Provides the Zone against which Tests are to run If zone name provided to marvin plugin is none it will get it from Test Data Config File Even, if it is not available, return None @Output : Returns the Zone Name ''' return self.__zone def getHypervisorInfo(self): ''' @Name : getHypervisorInfo @Desc : Provides the hypervisor Information to test users @Output : Return Hypervisor Information ''' return self.__hypervisor def __setHypervisorInfo(self): ''' @Name : __setHypervisorInfo @Desc: Set the HyperVisor details; default to XenServer ''' try: if not self.__hypervisor: self.__hypervisor = XEN_SERVER return SUCCESS except Exception as e: self.__logger.exception("Set hypervisor failed: %s" % e) return FAILED def __createApiClient(self): try: ''' Step1 : Create a CS Connection Object ''' mgmt_details = self.__mgmtDetails self.__csConnection = CSConnection(mgmt_details, self.__asyncTimeOut, self.__logger) ''' Step2 : Create API Client with earlier created connection object ''' self.__apiClient = CloudStackAPIClient(self.__csConnection) ''' Step3: If API Key is not provided as part of Management Details, then verify and register ''' if mgmt_details.apiKey is None: list_user = listUsers.listUsersCmd() list_user.account = "admin" list_user_res = self.__apiClient.listUsers(list_user) if list_user_res is None or \ (validateList(list_user_res)[0] != PASS): self.__logger.error("API Client Creation Failed") return FAILED user_id = list_user_res[0].id api_key = list_user_res[0].apikey security_key = list_user_res[0].secretkey if api_key is None: ret = self.__getKeys(user_id) if ret != FAILED: mgmt_details.apiKey = ret[0] mgmt_details.securityKey = ret[1] else: self.__logger.error("API Client Creation Failed while Registering User") return FAILED else: mgmt_details.port = 8080 mgmt_details.apiKey = api_key mgmt_details.securityKey = security_key ''' Now Create the Connection objects and Api Client using new details ''' self.__csConnection = CSConnection(mgmt_details, self.__asyncTimeOut, self.__logger) self.__apiClient = CloudStackAPIClient(self.__csConnection) return SUCCESS except Exception as e: self.__logger.exception("Create API client failed: %s" % e) return FAILED def __createDbConnection(self): ''' @Name : ___createDbConnection @Desc : Creates the CloudStack DB Connection ''' host = "localhost" if self.__dbSvrDetails.dbSvr is None \ else self.__dbSvrDetails.dbSvr port = 3306 if self.__dbSvrDetails.port is None \ else self.__dbSvrDetails.port user = "******" if self.__dbSvrDetails.user is None \ else self.__dbSvrDetails.user passwd = 'cloud' if self.__dbSvrDetails.passwd is None \ else self.__dbSvrDetails.passwd db = 'cloud' if self.__dbSvrDetails.db is None \ else self.__dbSvrDetails.db self.__dbConnection = DbConnection(host, port, user, passwd, db) def __getKeys(self, userid): ''' @Name : ___getKeys @Desc : Retrieves the API and Secret Key for the provided Userid @Input: userid: Userid to register @Output: FAILED or tuple with apikey and secretkey ''' try: register_user = registerUserKeys.registerUserKeysCmd() register_user.id = userid register_user_res = \ self.__apiClient.registerUserKeys(register_user) if not register_user_res: return FAILED return (register_user_res.apikey, register_user_res.secretkey) except Exception as e: self.__logger.exception("Key retrival failed: %s" % e) return FAILED def createTestClient(self): ''' @Name : createTestClient @Desc : Creates the Test Client. The test Client is used by test suites Here we create ParsedTestData Config. Creates a DB Connection. Creates an API Client @Output : FAILED In case of an issue\Failure SUCCESS in case of Success of this function ''' try: ''' 1. Create Config Object Provides the Configuration Object to test suites through getConfigParser. The purpose of this config object is to parse the default config and provide dictionary of the config so users can use that configuration. Users can later call getConfig on this object and it will return the default parsed config dictionary from default configuration file. They can overwrite it with providing their own configuration file as well. ''' ''' 1. Check Config,Zone,Hypervisor Information ''' self.__configObj = ConfigManager(self.__testDataFilePath) if not self.__configObj or not self.__hypervisor: self.__logger.error("Either Hypervisor is None or Not able to create ConfigManager Object") return FAILED self.__parsedTestDataConfig = self.__configObj.getConfig() self.__logger.info("Parsing Test data successful") ''' 2. Create DB Connection ''' self.__createDbConnection() ''' 3. Creates API Client ''' ret = self.__createApiClient() if ret == FAILED: self.__logger.error("=== Test Client Creation Failed ===") else: self.__logger.info("=== Test Client Creation Successful ===") return ret except Exception as e: self.__logger.exception("Test Client creation failed: %s" % e) return FAILED def isAdminContext(self): """ @Name : isAdminContext @Desc:A user is a regular user if he fails to listDomains; if he is a domain-admin, he can list only domains that are non-ROOT; if he is an admin, he can list the ROOT domain successfully """ try: listdom = listDomains.listDomainsCmd() listdom.name = 'ROOT' listdomres = self.__apiClient.listDomains(listdom) if listdomres != FAILED: rootdom = listdomres[0].name if rootdom == 'ROOT': return ADMIN else: return DOMAIN_ADMIN return USER except: return USER def __createUserApiClient(self, UserName, DomainName, acctType=0): ''' @Name : ___createUserApiClient @Desc : Creates a User API Client with given UserName\DomainName Parameters @Input: UserName: Username to be created in cloudstack DomainName: Domain under which the above account be created accType: Type of Account EX: Root,Non Root etc @Output: Return the API client for the user ''' try: if not self.isAdminContext(): return self.__apiClient mgmt_details = self.__mgmtDetails listDomain = listDomains.listDomainsCmd() listDomain.listall = True listDomain.name = DomainName try: domains = self.__apiClient.listDomains(listDomain) domId = domains[0].id except: cdomain = createDomain.createDomainCmd() cdomain.name = DomainName domain = self.__apiClient.createDomain(cdomain) domId = domain.id cmd = listAccounts.listAccountsCmd() cmd.name = UserName cmd.domainid = domId cmd.listall = True try: accounts = self.__apiClient.listAccounts(cmd) acctId = accounts[0].id except: createAcctCmd = createAccount.createAccountCmd() createAcctCmd.accounttype = acctType createAcctCmd.domainid = domId createAcctCmd.email = "test-" + random_gen() \ + "@cloudstack.org" createAcctCmd.firstname = UserName createAcctCmd.lastname = UserName createAcctCmd.password = '******' createAcctCmd.username = UserName acct = self.__apiClient.createAccount(createAcctCmd) acctId = acct.id listuser = listUsers.listUsersCmd() listuser.username = UserName listuser.domainid = domId listuser.listall = True listuserRes = self.__apiClient.listUsers(listuser) userId = listuserRes[0].id apiKey = listuserRes[0].apikey securityKey = listuserRes[0].secretkey if apiKey is None: ret = self.__getKeys(userId) if ret != FAILED: mgmt_details.apiKey = ret[0] mgmt_details.securityKey = ret[1] else: self.__logger.error("User API Client Creation While Registering User Failed") return FAILED else: mgmt_details.port = 8080 mgmt_details.apiKey = apiKey mgmt_details.securityKey = securityKey newUserConnection = \ CSConnection(mgmt_details, self.__csConnection.asyncTimeout, self.__csConnection.logger) self.__userApiClient = CloudStackAPIClient(newUserConnection) self.__userApiClient.connection = newUserConnection self.__userApiClient.hypervisor = self.__hypervisor return self.__userApiClient except Exception as e: self.__logger.exception("API user creation failed: %s" % e) return FAILED def close(self): if self.__csConnection is not None: self.__csConnection.close() def getDbConnection(self): ''' @Name : getDbConnection @Desc : Retrieves the DB Connection Handle ''' return self.__dbConnection def getConfigParser(self): ''' @Name : getConfigParser @Desc : Provides the ConfigManager Interface to TestClients ''' return self.__configObj def getApiClient(self): if self.__apiClient: self.__apiClient.id = self.identifier return self.__apiClient return None def getUserApiClient(self, UserName=None, DomainName=None, type=0): """ @Name : getUserApiClient @Desc : Provides the User API Client to test Users 0 - user ; 1 - admin;2 - domain admin @OutPut : FAILED In case of an issue else User API Client """ if UserName is None or DomainName is None: return FAILED return self.__createUserApiClient(UserName, DomainName, type) def submitCmdsAndWait(self, cmds, workers=1, apiclient=None): ''' @Desc : FixME, httplib has issue if more than one thread submitted ''' if not apiclient: apiclient = self.__apiClient if self.__asyncJobMgr is None: self.__asyncJobMgr = asyncJobMgr(apiclient, self.__dbConnection) return self.__asyncJobMgr.submitCmdsAndWait(cmds, workers) def submitJob(self, job, ntimes=1, nums_threads=10, interval=1): ''' @Desc : submit one job and execute the same job ntimes, with nums_threads of threads ''' if self.__asyncJobMgr is None: self.__asyncJobMgr = asyncJobMgr(self.__apiClient, self.__dbConnection) self.__asyncJobMgr.submitJobExecuteNtimes(job, ntimes, nums_threads, interval) def submitJobs(self, jobs, nums_threads=10, interval=1): ''' @Desc :submit n jobs, execute them with nums_threads of threads ''' if self.__asyncJobMgr is None: self.__asyncJobMgr = asyncJobMgr(self.__apiClient, self.__dbConnection) self.__asyncJobMgr.submitJobs(jobs, nums_threads, interval)
class CSTestClient(object): ''' @Desc : CloudStackTestClient is encapsulated entity for creating and getting various clients viz., apiclient, user api client, dbconnection, test Data parsed information etc @Input : mgmt_details : Management Server Details dbsvr_details: Database Server details of Management \ Server. Retrieved from configuration file. async_timeout : Timeout for Async queries default_worker_threads : Number of worker threads logger : provides logging facilities for this library zone : The zone on which test suites using this test client will run ''' def __init__(self, mgmt_details, dbsvr_details, async_timeout=3600, test_data_filepath=None, zone=None, hypervisor_type=None): self.__mgmtDetails = mgmt_details self.__dbSvrDetails = dbsvr_details self.__csConnection = None self.__dbConnection = None self.__testClient = None self.__asyncTimeOut = async_timeout self.__logger = MarvinLog('marvin').getLogger() self.__apiClient = None self.__userApiClient = None self.__asyncJobMgr = None self.__id = None self.__hypervisor = hypervisor_type self.__testDataFilePath = test_data_filepath self.__parsedTestDataConfig = None self.__zone = zone self.__setHypervisorInfo() @property def identifier(self): return self.__id @identifier.setter def identifier(self, id): self.__id = id def getParsedTestDataConfig(self): ''' @Name : getParsedTestDataConfig @Desc : Provides the TestData Config needed for Tests are to Run @Output : Returns the Parsed Test Data Dictionary ''' return copy.deepcopy(self.__parsedTestDataConfig) def getZoneForTests(self): ''' @Name : getZoneForTests @Desc : Provides the Zone against which Tests are to run If zone name provided to marvin plugin is none it will get it from Test Data Config File Even, if it is not available, return None @Output : Returns the Zone Name ''' return self.__zone def getHypervisorInfo(self): ''' @Name : getHypervisorInfo @Desc : Provides the hypervisor Information to test users @Output : Return Hypervisor Information ''' return self.__hypervisor def __setHypervisorInfo(self): ''' @Name : __setHypervisorInfo @Desc: Set the HyperVisor details; default to XenServer ''' try: if not self.__hypervisor: self.__hypervisor = XEN_SERVER return SUCCESS except Exception as e: self.__logger.exception("Set hypervisor failed: %s" % e) return FAILED def __createApiClient(self): try: ''' Step1 : Create a CS Connection Object ''' mgmt_details = self.__mgmtDetails self.__csConnection = CSConnection(mgmt_details, self.__asyncTimeOut, self.__logger) ''' Step2 : Create API Client with earlier created connection object ''' self.__apiClient = CloudStackAPIClient(self.__csConnection) ''' Step3: If API Key is not provided as part of Management Details, then verify and register ''' if mgmt_details.apiKey is None: list_user = listUsers.listUsersCmd() list_user.account = "admin" list_user_res = self.__apiClient.listUsers(list_user) if list_user_res is None or \ (validateList(list_user_res)[0] != PASS): self.__logger.error("API Client Creation Failed") return FAILED user_id = list_user_res[0].id api_key = list_user_res[0].apikey security_key = list_user_res[0].secretkey if api_key is None: ret = self.__getKeys(user_id) if ret != FAILED: mgmt_details.apiKey = ret[0] mgmt_details.securityKey = ret[1] else: self.__logger.error( "API Client Creation Failed while Registering User" ) return FAILED else: mgmt_details.port = 8080 mgmt_details.apiKey = api_key mgmt_details.securityKey = security_key ''' Now Create the Connection objects and Api Client using new details ''' self.__csConnection = CSConnection(mgmt_details, self.__asyncTimeOut, self.__logger) self.__apiClient = CloudStackAPIClient(self.__csConnection) return SUCCESS except Exception as e: self.__logger.exception("Create API client failed: %s" % e) return FAILED def __createDbConnection(self): ''' @Name : ___createDbConnection @Desc : Creates the CloudStack DB Connection ''' host = "localhost" if self.__dbSvrDetails.dbSvr is None \ else self.__dbSvrDetails.dbSvr port = 3306 if self.__dbSvrDetails.port is None \ else self.__dbSvrDetails.port user = "******" if self.__dbSvrDetails.user is None \ else self.__dbSvrDetails.user passwd = 'cloud' if self.__dbSvrDetails.passwd is None \ else self.__dbSvrDetails.passwd db = 'cloud' if self.__dbSvrDetails.db is None \ else self.__dbSvrDetails.db self.__dbConnection = DbConnection(host, port, user, passwd, db) def __getKeys(self, userid): ''' @Name : ___getKeys @Desc : Retrieves the API and Secret Key for the provided Userid @Input: userid: Userid to register @Output: FAILED or tuple with apikey and secretkey ''' try: register_user = registerUserKeys.registerUserKeysCmd() register_user.id = userid register_user_res = \ self.__apiClient.registerUserKeys(register_user) if not register_user_res: return FAILED return (register_user_res.apikey, register_user_res.secretkey) except Exception as e: self.__logger.exception("Key retrival failed: %s" % e) return FAILED def createTestClient(self): ''' @Name : createTestClient @Desc : Creates the Test Client. The test Client is used by test suites Here we create ParsedTestData Config. Creates a DB Connection. Creates an API Client @Output : FAILED In case of an issue\Failure SUCCESS in case of Success of this function ''' try: ''' 1. Create Config Object Provides the Configuration Object to test suites through getConfigParser. The purpose of this config object is to parse the default config and provide dictionary of the config so users can use that configuration. Users can later call getConfig on this object and it will return the default parsed config dictionary from default configuration file. They can overwrite it with providing their own configuration file as well. ''' ''' 1. Check Config,Zone,Hypervisor Information ''' self.__configObj = ConfigManager(self.__testDataFilePath) if not self.__configObj or not self.__hypervisor: self.__logger.error( "Either Hypervisor is None or Not able to create ConfigManager Object" ) return FAILED self.__parsedTestDataConfig = self.__configObj.getConfig() self.__logger.info("Parsing Test data successful") ''' 2. Create DB Connection ''' self.__createDbConnection() ''' 3. Creates API Client ''' ret = self.__createApiClient() if ret == FAILED: self.__logger.error("=== Test Client Creation Failed ===") else: self.__logger.info("=== Test Client Creation Successful ===") return ret except Exception as e: self.__logger.exception("Test Client creation failed: %s" % e) return FAILED def isAdminContext(self): """ @Name : isAdminContext @Desc:A user is a regular user if he fails to listDomains; if he is a domain-admin, he can list only domains that are non-ROOT; if he is an admin, he can list the ROOT domain successfully """ try: listdom = listDomains.listDomainsCmd() listdom.name = 'ROOT' listdomres = self.__apiClient.listDomains(listdom) if listdomres != FAILED: rootdom = listdomres[0].name if rootdom == 'ROOT': return ADMIN else: return DOMAIN_ADMIN return USER except: return USER def __createUserApiClient(self, UserName, DomainName, acctType=0): ''' @Name : ___createUserApiClient @Desc : Creates a User API Client with given UserName\DomainName Parameters @Input: UserName: Username to be created in cloudstack DomainName: Domain under which the above account be created accType: Type of Account EX: Root,Non Root etc @Output: Return the API client for the user ''' try: if not self.isAdminContext(): return self.__apiClient mgmt_details = self.__mgmtDetails listDomain = listDomains.listDomainsCmd() listDomain.listall = True listDomain.name = DomainName try: domains = self.__apiClient.listDomains(listDomain) domId = domains[0].id except: cdomain = createDomain.createDomainCmd() cdomain.name = DomainName domain = self.__apiClient.createDomain(cdomain) domId = domain.id cmd = listAccounts.listAccountsCmd() cmd.name = UserName cmd.domainid = domId cmd.listall = True try: accounts = self.__apiClient.listAccounts(cmd) acctId = accounts[0].id except: createAcctCmd = createAccount.createAccountCmd() createAcctCmd.accounttype = acctType createAcctCmd.domainid = domId createAcctCmd.email = "test-" + random_gen() \ + "@cloudstack.org" createAcctCmd.firstname = UserName createAcctCmd.lastname = UserName createAcctCmd.password = '******' createAcctCmd.username = UserName acct = self.__apiClient.createAccount(createAcctCmd) acctId = acct.id listuser = listUsers.listUsersCmd() listuser.username = UserName listuser.domainid = domId listuser.listall = True listuserRes = self.__apiClient.listUsers(listuser) userId = listuserRes[0].id apiKey = listuserRes[0].apikey securityKey = listuserRes[0].secretkey if apiKey is None: ret = self.__getKeys(userId) if ret != FAILED: mgmt_details.apiKey = ret[0] mgmt_details.securityKey = ret[1] else: self.__logger.error( "User API Client Creation While Registering User Failed" ) return FAILED else: mgmt_details.port = 8080 mgmt_details.apiKey = apiKey mgmt_details.securityKey = securityKey newUserConnection = \ CSConnection(mgmt_details, self.__csConnection.asyncTimeout, self.__csConnection.logger) self.__userApiClient = CloudStackAPIClient(newUserConnection) self.__userApiClient.connection = newUserConnection self.__userApiClient.hypervisor = self.__hypervisor return self.__userApiClient except Exception as e: self.__logger.exception("API user creation failed: %s" % e) return FAILED def close(self): if self.__csConnection is not None: self.__csConnection.close() def getDbConnection(self): ''' @Name : getDbConnection @Desc : Retrieves the DB Connection Handle ''' return self.__dbConnection def getConfigParser(self): ''' @Name : getConfigParser @Desc : Provides the ConfigManager Interface to TestClients ''' return self.__configObj def getApiClient(self): if self.__apiClient: self.__apiClient.id = self.identifier return self.__apiClient return None def getUserApiClient(self, UserName=None, DomainName=None, type=0): """ @Name : getUserApiClient @Desc : Provides the User API Client to test Users 0 - user ; 1 - admin;2 - domain admin @OutPut : FAILED In case of an issue else User API Client """ if UserName is None or DomainName is None: return FAILED return self.__createUserApiClient(UserName, DomainName, type) def submitCmdsAndWait(self, cmds, workers=1, apiclient=None): ''' @Desc : FixME, httplib has issue if more than one thread submitted ''' if not apiclient: apiclient = self.__apiClient if self.__asyncJobMgr is None: self.__asyncJobMgr = asyncJobMgr(apiclient, self.__dbConnection) return self.__asyncJobMgr.submitCmdsAndWait(cmds, workers) def submitJob(self, job, ntimes=1, nums_threads=10, interval=1): ''' @Desc : submit one job and execute the same job ntimes, with nums_threads of threads ''' if self.__asyncJobMgr is None: self.__asyncJobMgr = asyncJobMgr(self.__apiClient, self.__dbConnection) self.__asyncJobMgr.submitJobExecuteNtimes(job, ntimes, nums_threads, interval) def submitJobs(self, jobs, nums_threads=10, interval=1): ''' @Desc :submit n jobs, execute them with nums_threads of threads ''' if self.__asyncJobMgr is None: self.__asyncJobMgr = asyncJobMgr(self.__apiClient, self.__dbConnection) self.__asyncJobMgr.submitJobs(jobs, nums_threads, interval)
class SshClient(object): ''' @Desc : SSH Library for Marvin. Facilitates SSH,SCP services to marvin users @Input: host: Host to connect port: port on host to connect user: Username to be used for connecting passwd: Password for connection retries and delay applies for establishing connection timeout : Applies while executing command ''' def __init__(self, host, port, user, passwd, retries=60, delay=10, log_lvl=logging.INFO, keyPairFiles=None, timeout=10.0): self.host = None self.port = 22 self.user = user self.passwd = passwd self.keyPairFiles = keyPairFiles self.ssh = SSHClient() self.ssh.set_missing_host_key_policy(AutoAddPolicy()) self.retryCnt = 0 self.delay = 0 self.timeout = 3.0 self.logger = MarvinLog('ssh').getLogger() # Check invalid host value and raise exception # Atleast host is required for connection if host is not None and host != '': self.host = host if retries is not None and retries > 0: self.retryCnt = retries if delay is not None and delay > 0: self.delay = delay if timeout is not None and timeout > 0: self.timeout = timeout if port is not None and port >= 0: self.port = port if self.createConnection() == FAILED: raise internalError("Connection Failed") def execute(self, command): stdin, stdout, stderr = self.ssh.exec_command(command) output = stdout.readlines() errors = stderr.readlines() results = [] if output is not None and len(output) == 0: if errors is not None and len(errors) > 0: for error in errors: results.append(error.rstrip()) else: for strOut in output: results.append(strOut.rstrip()) self.logger.debug("Executing command via host %s: %s Output: %s" % (str(self.host), command, results)) return results def createConnection(self): ''' @Name: createConnection @Desc: Creates an ssh connection for retries mentioned,along with sleep mentioned @Output: SUCCESS on successful connection FAILED If connection through ssh failed ''' ret = FAILED while self.retryCnt >= 0: try: self.logger.debug("Trying SSH Connection to host %s on port %s as user %s. RetryCount: %s" % (self.host, str(self.port), self.user, str(self.retryCnt))) if self.keyPairFiles is None: self.ssh.connect(hostname=self.host, port=self.port, username=self.user, password=self.passwd, timeout=self.timeout) else: self.ssh.connect(hostname=self.host, port=self.port, username=self.user, password=self.passwd, key_filename=self.keyPairFiles, timeout=self.timeout, look_for_keys=False ) self.logger.debug("Connection to host %s on port %s is SUCCESSFUL" % (str(self.host), str(self.port))) ret = SUCCESS break except BadHostKeyException as e: self.logger.debug("Failed to create connection: %s" % e) except AuthenticationException as e: self.logger.debug("Failed to create connection: %s" % e) except SSHException as e: self.logger.debug("Failed to create connection: %s" % e) except socket.error as e: self.logger.debug("Failed to create connection: %s" % e) except Exception as e: self.logger.debug("Failed to create connection: %s" % e) finally: if self.retryCnt == 0 or ret == SUCCESS: break self.retryCnt -= 1 time.sleep(self.delay) return ret def runCommand(self, command): ''' @Name: runCommand @Desc: Runs a command over ssh and returns the result along with status code @Input: command to execute @Output: 1: status of command executed. SUCCESS : If command execution is successful FAILED : If command execution has failed 2: stdin,stdout,stderr values of command output ''' ret = {"status": FAILED, "stdin": None, "stdout": None, "stderr": INVALID_INPUT} if command is None or command == '': return ret try: stdin, stdout, stderr = self.ssh.exec_command(command, timeout=self.timeout) if stdout is not None: status_check = stdout.channel.recv_exit_status() if status_check == 0: ret["status"] = SUCCESS ret["stdout"] = stdout.readlines() if stderr is not None: ret["stderr"] = stderr.readlines() except Exception as e: self.logger.debug("Failed to run command: %s" % e) finally: self.logger.debug("Connection to host %s on port %s is SUCCESSFUL" % (str(self.host), command)) return ret def scp(self, srcFile, destPath): transport = Transport((self.host, int(self.port))) transport.connect(username=self.user, password=self.passwd) sftp = SFTPClient.from_transport(transport) try: sftp.put(srcFile, destPath) except IOError as e: raise e def __del__(self): self.close() def close(self): if self.ssh is not None: self.ssh.close() self.ssh = None