def create(self,
server=None,
path=None,
physical_size=0,
content_type="",
nosubdir=False):
if not (server or path):
if xenrt.TEC().lookup("FORCE_NFSSR_ON_CTRL", False, boolean=True):
# Create an SR using an NFS export from the XenRT controller.
# This should only be used for small and low I/O throughput
# activities - VMs should never be installed on this.
nfsexport = xenrt.NFSDirectory()
server, path = nfsexport.getHostAndPath("")
else:
# Create an SR on an external NFS file server
share = xenrt.ExternalNFSShare()
nfs = share.getMount()
r = re.search(r"([0-9\.]+):(\S+)", nfs)
server = r.group(1)
path = r.group(2)
self.server = server
self.path = path
self._create("netfs",
"<host name='%s'/><dir path='%s'/>" % (server, path), "")
def getPrimaryStorageUrl(self, key, ref):
if not ref.has_key("XRT_PriStorageType"):
storageType = "NFS"
else:
storageType = ref['XRT_PriStorageType']
if storageType == "NFS":
if ref.has_key('XRT_Guest_NFS'):
ssGuest = xenrt.TEC().registry.guestGet(ref['XRT_Guest_NFS'])
xenrt.TEC().logverbose(
'Using guest %s for primary NFS storage' % (ssGuest.name))
shareName = 'PS-%s-%s' % (self.currentClusterName, ''.join(
random.sample(
string.ascii_lowercase + string.ascii_uppercase, 6)))
storagePath = ssGuest.createLinuxNfsShare(shareName)
url = 'nfs://%s' % (storagePath.replace(':', ''))
else:
primaryStorage = xenrt.ExternalNFSShare()
url = 'nfs://%s' % (primaryStorage.getMount().replace(':', ''))
elif storageType == "SMB":
h = xenrt.GEC().registry.hostGet("RESOURCE_HOST_%s" %
ref['XRT_SMBHostId'])
ip = h.getFQDN()
url = "cifs://%s/storage/primary" % (ip)
ad = xenrt.getADConfig()
return url
def prepare(self, arglist=None):
# Set up a pool of two machines, and install a VM.
self.master = self.getHost("RESOURCE_HOST_0")
self.master.resetToFreshInstall()
if self.NEED_SLAVE:
self.slave = self.getHost("RESOURCE_HOST_1")
self.slave.resetToFreshInstall()
self.pool = xenrt.lib.xenserver.poolFactory(
self.master.productVersion)(self.master)
if self.NEED_SLAVE:
self.pool.addHost(self.slave)
# Enable any FIST points and restart xapi...
for fp in self.FIST_POINTS:
for h in self.pool.getHosts():
h.execdom0("touch /tmp/%s" % (fp))
if len(self.FIST_POINTS) > 0:
self.pool.master.restartToolstack()
time.sleep(60)
for h in self.pool.getSlaves():
h.restartToolstack()
time.sleep(30)
time.sleep(30)
sr = None
try:
sr = self.master.getLocalSR()
except:
pass
if self.USE_NFS or not sr:
# Create an NFS SR
nfs = xenrt.ExternalNFSShare()
nfsMount = nfs.getMount()
r = re.search(r"([0-9\.]+):(\S+)", nfsMount)
if not r:
raise xenrt.XRTError("Unable to parse NFS paths %s" %
(nfsMount))
nfsSR = xenrt.lib.xenserver.NFSStorageRepository(
self.master, "NFS")
nfsSR.create(r.group(1), r.group(2))
sr = nfsSR.uuid
if self.VM_ON_SLAVE:
self.guest = self.slave.createGenericLinuxGuest(start=False, sr=sr)
else:
self.guest = self.master.createGenericLinuxGuest(start=False,
sr=sr)
# Remove any existing RRDs for this guest
self.master.execdom0("rm -f /var/xapi/blobs/rrds/%s.gz || true" %
(self.guest.getUUID()))
self.startTime = xenrt.util.timenow()
self.guest.start()
def getnfs(self):
rshare = xenrt.ExternalNFSShare()
rpath = rshare.getMount()
lshare = xenrt.rootops.MountNFS(rpath)
lpath = lshare.getMount()
self._nfs = {
'rshare': rshare,
'rpath': rpath,
'lshare': lshare,
'lpath': lpath
}
return self._nfs
def getSecondaryStorageUrl(self, key, ref):
if not ref.has_key('provider'):
provider = "NFS"
else:
provider = ref['provider']
if provider == "NFS":
if ref.has_key("XRT_Guest_NFS"):
ssGuest = xenrt.TEC().registry.guestGet(ref['XRT_Guest_NFS'])
xenrt.TEC().logverbose(
'Using guest %s for secondary NFS storage' %
(ssGuest.name))
shareName = 'SS-%s-%s' % (self.currentZoneName, ''.join(
random.sample(
string.ascii_lowercase + string.ascii_uppercase, 6)))
storagePath = ssGuest.createLinuxNfsShare(shareName)
self.marvin.copySystemTemplatesToSecondaryStorage(
storagePath, 'NFS')
url = 'nfs://%s' % (storagePath.replace(':', ''))
elif self.initialNFSSecStorageUrl:
url = self.initialNFSSecStorageUrl
self.initialNFSSecStorageUrl = None
else:
secondaryStorage = xenrt.ExternalNFSShare()
storagePath = secondaryStorage.getMount()
url = 'nfs://%s' % (secondaryStorage.getMount().replace(
':', ''))
self.marvin.copySystemTemplatesToSecondaryStorage(
storagePath, 'NFS')
elif provider == "SMB":
if self.initialSMBSecStorageUrl:
url = self.initialSMBSecStorageUrl
self.initialSMBSecStorageUrl = None
else:
if xenrt.TEC().lookup("EXTERNAL_SMB", False, boolean=True):
secondaryStorage = xenrt.ExternalSMBShare()
url = 'cifs://%s' % (secondaryStorage.getMount().replace(
':', ''))
storagePath = secondaryStorage.getMount()
else:
h = xenrt.GEC().registry.hostGet("RESOURCE_HOST_%s" %
ref['XRT_SMBHostId'])
ip = h.getIP()
url = "cifs://%s/storage/secondary" % ip
storagePath = "%s:/storage/secondary" % ip
self.marvin.copySystemTemplatesToSecondaryStorage(
storagePath, 'SMB')
return url
def run(self, arglist):
self.nfs = xenrt.ExternalNFSShare()
resultName = None
try:
resultName = self.__runTestScript()
finally:
if not resultName:
raise xenrt.XRTFailure(
"Could not get the result files from the powershell tests")
xenrt.TEC().logverbose("Reading file %s...." % resultName)
result = self.guest.xmlrpcReadFile(resultName)
resultFile = resultName.split('\\')[-1]
file("%s/%s" % (xenrt.TEC().getLogdir(), resultFile),
"w").write(result)
try:
self.parseResults(result)
except:
raise xenrt.XRTError("Error parsing XML results file.")
for x in self.myresults:
if x["state"] == "Fail":
raise xenrt.XRTFailure("One or more subcases failed.")
def setupManagementServer(self):
self.primaryManagementServer.execcmd(
'iptables -I INPUT -p tcp --dport 8096 -j ACCEPT')
setupMsLoc = self.primaryManagementServer.execcmd(
'find /usr/bin -name %s-setup-management' %
(self.cmdPrefix)).strip()
self.primaryManagementServer.execcmd(setupMsLoc)
self.primaryManagementServer.execcmd(
'mysql -u cloud --password=cloud -h %s --execute="UPDATE cloud.configuration SET value=8096 WHERE name=\'integration.api.port\'"'
% self.dbServer.getIP())
if xenrt.TEC().lookup("USE_CCP_SIMULATOR", False,
boolean=True) or self._simDbServer:
# For some reason the cloud user doesn't seem to have access to the simulator DB
self.primaryManagementServer.execcmd(
"""sed -i s/db.simulator.username=cloud/db.simulator.username=root/ /usr/share/cloudstack-management/conf/db.properties"""
)
self.primaryManagementServer.execcmd(
"""sed -i s/db.simulator.password=cloud/db.simulator.password=xensource/ /usr/share/cloudstack-management/conf/db.properties"""
)
self.primaryManagementServer.execcmd(
"""sed -i s/db.simulator.host=localhost/db.simulator.host=%s/ /usr/share/cloudstack-management/conf/db.properties"""
% self.simDbServer.getIP())
self.restart(checkHealth=False)
self.checkManagementServerHealth(timeout=300)
# We have to update templates *after* starting the management server as some templates are not introduced until DB schema updates are applied
templateSubsts = {
"http://download.cloud.com/templates/builtin/centos56-x86_64.vhd.bz2":
"%s/cloudTemplates/centos56-x86_64.vhd.bz2" %
xenrt.TEC().lookup("EXPORT_DISTFILES_HTTP"),
"http://download.cloud.com/releases/4.3/centos6_4_64bit.vhd.bz2":
"%s/cloudTemplates/centos6_4_64bit.vhd.bz2" %
xenrt.TEC().lookup("EXPORT_DISTFILES_HTTP"),
"http://nfs1.lab.vmops.com/templates/centos53-x86_64/latest/f59f18fb-ae94-4f97-afd2-f84755767aca.vhd.bz2":
"%s/cloudTemplates/f59f18fb-ae94-4f97-afd2-f84755767aca.vhd.bz2" %
xenrt.TEC().lookup("EXPORT_DISTFILES_HTTP"),
"http://download.cloud.com/templates/builtin/f59f18fb-ae94-4f97-afd2-f84755767aca.vhd.bz2":
"%s/cloudTemplates/f59f18fb-ae94-4f97-afd2-f84755767aca.vhd.bz2" %
xenrt.TEC().lookup("EXPORT_DISTFILES_HTTP"),
"http://download.cloud.com/releases/2.2.0/CentOS5.3-x86_64.ova":
"%s/cloudTemplates/CentOS5.3-x86_64.ova" %
xenrt.TEC().lookup("EXPORT_DISTFILES_HTTP"),
"http://download.cloud.com/releases/2.2.0/eec2209b-9875-3c8d-92be-c001bd8a0faf.qcow2.bz2":
"%s/cloudTemplates/eec2209b-9875-3c8d-92be-c001bd8a0faf.qcow2.bz2"
% xenrt.TEC().lookup("EXPORT_DISTFILES_HTTP"),
"http://download.cloud.com/templates/builtin/centos-7-x86_64.tar.gz":
"%s/cloudTemplates/centos-7-x86_64.tar.gz" %
xenrt.TEC().lookup("EXPORT_DISTFILES_HTTP")
}
if xenrt.TEC().lookup("MARVIN_BUILTIN_TEMPLATES", False, boolean=True):
templateSubsts["http://download.cloud.com/templates/builtin/centos56-x86_64.vhd.bz2"] = \
"%s/cloudTemplates/centos56-httpd-64bit.vhd.bz2" % xenrt.TEC().lookup("EXPORT_DISTFILES_HTTP")
templateSubsts["http://download.cloud.com/releases/2.2.0/CentOS5.3-x86_64.ova"] = \
"%s/cloudTemplates/centos53-httpd-64bit.ova" % xenrt.TEC().lookup("EXPORT_DISTFILES_HTTP")
templateSubsts["http://download.cloud.com/releases/2.2.0/eec2209b-9875-3c8d-92be-c001bd8a0faf.qcow2.bz2"] = \
"%s/cloudTemplates/centos55-httpd-64bit.qcow2" % xenrt.TEC().lookup("EXPORT_DISTFILES_HTTP")
for t in templateSubsts.keys():
self.primaryManagementServer.execcmd(
"""mysql -u cloud --password=cloud -h %s --execute="UPDATE cloud.vm_template SET url='%s' WHERE url='%s'" """
% (self.dbServer.getIP(), templateSubsts[t], t))
self.restart()
marvinApi = xenrt.lib.cloud.MarvinApi(self)
internalMask = IPy.IP("%s/%s" % (xenrt.getNetworkParam(
"NPRI", "SUBNET"), xenrt.getNetworkParam("NPRI", "SUBNETMASK")))
if xenrt.TEC().lookup("USE_CCP_SIMULATOR", False,
boolean=True) or self._simDbServer:
self.primaryManagementServer.execcmd(
'mysql -u root --password=xensource -h %s < /usr/share/cloudstack-management/setup/hypervisor_capabilities.simulator.sql'
% self.dbServer.getIP())
self.primaryManagementServer.execcmd(
'mysql -u root --password=xensource -h %s < /usr/share/cloudstack-management/setup/templates.simulator.sql'
% self.dbServer.getIP())
marvinApi.setCloudGlobalConfig("secstorage.allowed.internal.sites",
internalMask.strNormal())
if not xenrt.TEC().lookup("MARVIN_SETUP", False, boolean=True):
marvinApi.setCloudGlobalConfig("use.external.dns", "true")
endpoint_url = "http://%s:8096/client/api" % marvinApi.mgtSvrDetails.mgtSvrIp
if self.additionalManagementServers:
marvinApi.setCloudGlobalConfig("agent.lb.enabled", "true")
marvinApi.setCloudGlobalConfig("endpointe.url", endpoint_url)
marvinApi.setCloudGlobalConfig("check.pod.cidrs",
"false",
restartManagementServer=True)
xenrt.GEC().dbconnect.jobUpdate("CLOUD_MGMT_SVR_IP",
self.primaryManagementServer.getIP())
xenrt.TEC().registry.toolstackPut(
"cloud",
xenrt.lib.cloud.CloudStack(place=self.primaryManagementServer))
# Create one secondary storage, to speed up deployment.
# Additional locations will need to be created during deployment
hvlist = xenrt.TEC().lookup("CLOUD_REQ_SYS_TMPLS", None)
if hvlist:
hvlist = hvlist.split(",")
else:
hvlist = []
if any(
map(lambda hv: hv in hvlist,
["kvm", "xenserver", "vmware", "lxc"])):
secondaryStorage = xenrt.ExternalNFSShare()
storagePath = secondaryStorage.getMount()
url = 'nfs://%s' % (secondaryStorage.getMount().replace(':', ''))
marvinApi.copySystemTemplatesToSecondaryStorage(storagePath, "NFS")
self.primaryManagementServer.special[
'initialNFSSecStorageUrl'] = url
elif "hyperv" in hvlist:
if xenrt.TEC().lookup("EXTERNAL_SMB", False, boolean=True):
secondaryStorage = xenrt.ExternalSMBShare()
storagePath = secondaryStorage.getMount()
url = 'cifs://%s' % (secondaryStorage.getMount().replace(
':', ''))
marvinApi.copySystemTemplatesToSecondaryStorage(
storagePath, "SMB")
self.primaryManagementServer.special[
'initialSMBSecStorageUrl'] = url
if xenrt.TEC().lookup("CCP_CODE_COVERAGE", False, boolean=True):
xenrt.TEC().logverbose("Enabling code coverage collection...")
for m in self.allManagementServers:
if m.execcmd("ls %s/setup_codecoverage.sh" % self.installDir,
retval="code") != 0:
raise xenrt.XRTError(
"CCP_CODE_COVERAGE set but setup_codecoverage.sh not found in build"
)
m.execcmd("cd %s && ./setup_codecoverage.sh" % self.installDir)
self.restart()
xenrt.TEC().logverbose("...done")
commit = None
try:
commit = self.primaryManagementServer.execcmd(
"cloudstack-sccs").strip()
xenrt.TEC().logverbose(
"Management server was built from commit %s" % commit)
except:
xenrt.TEC().warning(
"Error when trying to identify management server version")
if commit:
expectedCommit = xenrt.getCCPCommit(
self.primaryManagementServer.distro)
if expectedCommit and commit != expectedCommit:
raise xenrt.XRTError(
"Management server commit %s does not match expected commit %s"
% (commit, expectedCommit))
def run(self, arglist):
sftp = self.runner.sftpClient()
sftp.copyTreeTo("%s/data/tests/marvin" % xenrt.TEC().lookup("XENRT_BASE"), "/root/marvin-scripts")
try:
testData = json.loads(self.runner.execguest("/root/marvin-scripts/getDefaultConfig.py"))
except:
xenrt.TEC().logverbose("test_data not supported")
else:
# Older versions of marvin don't have configurableData.
# Initialise it to an empty dict here so we don't get exceptions
# later on
if not testData.has_key('configurableData'):
testData['configurableData'] = {}
if self.args.has_key("resources"):
resources = self.args['resources'].split(",")
else:
resources = []
if "nfs" in resources:
nfspath = xenrt.ExternalNFSShare().getMount()
testData['nfs'] = {"url": "nfs://%s" % nfspath, "name": "Test NFS Storage"}
if "iscsi" in resources:
lun = xenrt.ISCSITemporaryLun(100)
testData['iscsi'] = {"url": "iscsi://%s/%s/%d" % (lun.getServer(), lun.getTargetName(), lun.getLunID()), "name": "Test iSCSI Storage"}
testData['configurableData']['iscsi'] = {"url": "iscsi://%s/%s/%d" % (lun.getServer(), lun.getTargetName(), lun.getLunID()), "name": "Test iSCSI Storage"}
if "portableip" in resources:
range = xenrt.StaticIP4Addr().getIPRange(4)
testData['configurableData']['portableIpRange']['startip'] = range[0].getAddr()
testData['configurableData']['portableIpRange']['endip'] = range[-1].getAddr()
testData['configurableData']['portableIpRange']['gateway'] = xenrt.getNetworkParam("NPRI", "GATEWAY")
testData['configurableData']['portableIpRange']['netmask'] = xenrt.getNetworkParam("NPRI", "SUBNETMASK")
testData['configurableData']['portableIpRange']['vlan'] = 1000
if "netscaler" in resources:
netscaler = NetScaler.setupNetScalerVpx('NetScaler-VPX')
netscaler.applyLicense(netscaler.getLicenseFileFromXenRT())
testData['configurableData']['netscaler']['ipaddress'] = netscaler.managementIp
testData['configurableData']['netscaler']['username'] = '******'
testData['configurableData']['netscaler']['password'] = '******'
testData['configurableData']['netscaler']['networkdevicetype'] = 'NetscalerVPXLoadBalancer'
testData['configurableData']['netscaler']['publicinterface'] = '1/1'
testData['configurableData']['netscaler']['privateinterface'] = '1/1'
testData['configurableData']['netscaler']['numretries'] = '2'
if self.args['hypervisor'].lower() == "hyperv":
testData['service_offering']['memory'] = 512
if "service_offerings" in testData:
testData['service_offerings']['memory'] = 512
testData['service_offerings']['cpuspeed'] = 500
testData['service_offerings']['tiny']['memory'] = 512
testData['service_offerings']['small']['memory'] = 512
testData['service_offerings']['medium']['memory'] = 512
testData['service_offerings']['big']['memory'] = 512
testData['service_offering']['cpuspeed'] = 500
testData['service_offerings']['tiny']['cpuspeed'] = 500
testData['service_offerings']['small']['cpuspeed'] = 500
testData['service_offerings']['medium']['cpuspeed'] = 500
testData['service_offerings']['big']['cpuspeed'] = 500
testData['hypervisor'] = self.args['hypervisor']
testData['small']['hypervisor'] = self.args['hypervisor']
if 'medium' in testData:
testData['medium']['hypervisor'] = self.args['hypervisor']
if 'server' in testData:
testData['server']['hypervisor'] = self.args['hypervisor']
if 'server_without_disk' in testData:
testData['server_without_disk']['hypervisor'] = self.args['hypervisor']
testData['host_password'] = xenrt.TEC().lookup("ROOT_PASSWORD")
testData['configurableData']['host']['password'] = xenrt.TEC().lookup("ROOT_PASSWORD")
# ISO replacements
if testData['configurableData'].has_key("bootableIso"):
testData['configurableData']['bootableIso']['url'] = "%s/centos63_x86-64_xenrtinst.iso" % xenrt.TEC().lookup("EXPORT_ISO_HTTP_STATIC")
for i in ["iso", "iso1", "iso2"]:
if testData.has_key(i):
testData[i]['url'] = "%s/dummy.iso" % xenrt.TEC().lookup("EXPORT_ISO_HTTP_STATIC")
with open("%s/testdata.cfg" % xenrt.TEC().getLogdir(), "w") as f:
f.write(json.dumps(testData, indent=2))
sftp.copyTo("%s/testdata.cfg" % xenrt.TEC().getLogdir(), "/root/testdata.cfg")
self.toolstack.marvinCfg['TestData'] = {'Path': "/root/testdata.cfg"}
with open("%s/marvin.cfg" % xenrt.TEC().getLogdir(), "w") as f:
f.write(json.dumps(self.toolstack.marvinCfg, indent=2))
sftp.copyTo("%s/marvin.cfg" % xenrt.TEC().getLogdir(), "/root/marvin.cfg")
def run(self, arglist=None):
if not self.HA:
if not arglist or len(arglist) < 2:
raise xenrt.XRTError("Need at least a master and a pool name.")
poolname = arglist[0]
mastername = arglist[1]
host = xenrt.TEC().registry.hostGet(mastername)
if not host:
raise xenrt.XRTError("Unable to find host %s in registry." %
(mastername))
self.getLogsFrom(host)
# Create the pool object with the master host.
xenrt.TEC().logverbose("Creating standalone pool...")
pool = xenrt.lib.xenserver.poolFactory(host.productVersion)(host)
xenrt.TEC().logverbose("...pool created")
# Set the crashdump and suspend default SRs to be the shared storage.
if not xenrt.TEC().lookup(
"POOL_NO_DEFAULT_SR", False, boolean=True):
sruuid = pool.master.parseListForUUID("sr-list", "name-label",
pool.master.defaultsr)
pool.setPoolParam("default-SR", sruuid)
pool.setPoolParam("crash-dump-SR", sruuid)
pool.setPoolParam("suspend-image-SR", sruuid)
else:
# This is really to work around an annoying OEM trait...
pool.clearPoolParam("crash-dump-SR")
pool.clearPoolParam("suspend-image-SR")
pool.setPoolParam("name-label", poolname)
if xenrt.TEC().lookup("POOL_SHARED_DB", False, boolean=True):
# Use shared DB on this pool
pool.setupSharedDB()
hostlist = xenrt.TEC().registry.hostList()
hostlist.remove(mastername)
xenrt.TEC().logverbose("Adding hosts to pool...")
# Add other hosts to this pool.
for slavename in hostlist:
slave = xenrt.TEC().registry.hostGet(slavename)
if not slave:
raise xenrt.XRTError(
"Unable to find host %s in registry." % (slavename))
self.getLogsFrom(slave)
pool.addHost(slave, force=True)
pool.check()
hostsAdded = pool.master.minimalList("host-list")
if not len(hostsAdded) == self.MAX:
raise xenrt.XRTFailure("Didnt find 16 hosts")
xenrt.TEC().registry.poolPut(poolname, pool)
elif self.HA:
#Get the pool and the host
pool = xenrt.TEC().registry.poolGet("mypool")
host = xenrt.TEC().registry.hostGet("RESOURCE_HOST_0")
#Attach an external nfs SR
if self.SR == "nfs":
xenrt.TEC().logverbose("Attaching the nfs SR to host 0")
self.nfs = xenrt.ExternalNFSShare()
srs = host.getSRs(type=self.SR)
if len(srs) < 1:
raise xenrt.XRTError("Couldn't find an %s SR" % (self.SR))
sruuid = srs[0]
#Enable HA on a pool
pool.enableHA(srs=[sruuid])
#Check HA is enabled on each host
pool.checkHA()
#Disable HA
pool.disableHA()