def listVms(self, showVmsFromAllUsers=False):
fromAllUsers = -2
currentUserOnly = -3
if showVmsFromAllUsers:
visibilitySwitch = fromAllUsers
else:
visibilitySwitch = currentUserOnly
# Hack to retry on SSL errors.
maxRetries = 3
retries = 0
while True:
try:
ret, info, _ = self._rpc.one.vmpool.info(self._sessionString, visibilitySwitch, -1, -1, -1)
break
except ssl.SSLError as e:
retries += 1
t = strftime("%Y-%m-%d %H:%M:%S", gmtime())
Util.printDetail('SSL ERROR ENCOUNTERED (%s): %s' % (t, str(e)))
if retries >= maxRetries:
raise e
if not ret:
raise OneException(info)
vmlist = Util.etree_from_text(info)
for xml in vmlist.findall('VM'):
self._addStateSummary(xml)
return etree.tostring(vmlist)
def _generateOpensslConfig(self):
config = """
[ req ]
distinguished_name = req_distinguished_name
x509_extensions = v3_ca
prompt = no
input_password = %(certPassword)s
output_password = %(certPassword)s
dirstring_type = nobmp
[ req_distinguished_name ]
C = EU
O = StratusLab Project
OU = Testing Department
CN = %(commonName)s
[ v3_ca ]
basicConstraints = CA:false
nsCertType=client, email, objsign
keyUsage=critical, digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment, keyAgreement
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid:always,issuer:always
subjectAltName=email:%(subjectEmail)s
""" % self.configHolder.options
conf_filename = os.path.join(self.tmp_dir, 'openssl.cfg')
open(conf_filename, 'w').write(config)
Util.printDetail("Generated openssl configuration in: %s" % conf_filename,
self.configHolder.verboseLevel)
Util.printDetail("Openssl configuration: %s" % open(conf_filename).read(),
self.configHolder.verboseLevel,
Util.VERBOSE_LEVEL_DETAILED)
def addNetworkAcl(self, users, net_id_int, rights):
"""
users - hex
net_id_int - integer, network ID
rights - hex
"""
# "magic" number
_magic = self.ACL_USERS['UID']
net_resource = hex(self.ACL_RESOURCES['NET'] + _magic + net_id_int)
# Hack to retry on SSL errors
maxRetries = 3
retries = 0
while True:
try:
ret, info, _ = self._rpc.one.acl.addrule(self._sessionString,
users,
net_resource,
rights)
break
except ssl.SSLError as e:
retries += 1
t = strftime("%Y-%m-%d %H:%M:%S", gmtime())
Util.printDetail('SSL ERROR ENCOUNTERED (%s): %s' % (t, str(e)))
if retries >= maxRetries:
raise e
if not ret:
raise OneException(info)
return info
def _startDhcp(self):
Util.printDetail('(Re)Starting DHCP server.')
serviceName = self.packages['dhcp'].initdScriptName
rc = self.restartService(serviceName)
if rc != 0:
Util.printError('Failed to (re)start DHCP service.')
def _configureVirtualNetInterface(self, device, ip, netmask):
device = device + ":privlan"
Util.printDetail("Configuring network interface %s." % device)
self._configureNetworkInterface(device, ip, netmask)
Util.printDetail("Starting network interface %s." % device)
self.executeCmd(["ifup", device])
def _installDhcp(self):
Util.printDetail('Installing DHCP server.')
dhcpPackage = self.getPackageName('dhcp')
self.installPackages([dhcpPackage])
if not self.isPackageInstalled(dhcpPackage):
Util.printError('Failed to install %s.' % dhcpPackage)
def doWork(self):
configHolder = ConfigHolder(self.options.__dict__)
signator = Signator(self.manifestFile, configHolder)
isError = signator.sign()
if isError:
Util.printError('Error signing metadata file')
else:
Util.printDetail('Metadata file successfully signed: %s' % signator.outputManifestFile)
def _installDhcp(self):
Util.printDetail("Installing DHCP server.")
dhcpPackage = self.getPackageName("dhcp")
self.installPackages([dhcpPackage])
if not self.isPackageInstalled(dhcpPackage):
Util.printError("Failed to install %s." % dhcpPackage)
def _configureVirtualNetInterface(self, device, ip, netmask):
device = device + ':privlan'
Util.printDetail('Configuring network interface %s.' % device)
self._configureNetworkInterface(device, ip, netmask)
Util.printDetail('Starting network interface %s.' % device)
self.executeCmd(['ifup', device])
def _saveFirewallRules(self, filename):
# back-up
self.executeCmd(('cp -fp %s %s.LAST'%((filename,)*2)).split(' '))
_,output = self.executeCmdWithOutput(['iptables-save'])
Util.printDetail('Saving firewall rules to %s.' % filename)
filePutContent(filename, output)
os.chmod(filename, 0600)
def buildAndSave(self, filename=''):
manifestText = self.build()
if not filename:
filename = '%s-%s-%s-%s-%s%s' % (self.os, self.osversion,
self.arch, self.type,
self.version, Util.manifestExt)
Util.filePutContent(filename, manifestText)
Util.printDetail("Manifest: %s" % filename, self.verboseLevel,
Util.VERBOSE_LEVEL_DETAILED)
def _installFetchCrl(self):
"""fetch-crl 3:
http://www.nikhef.nl/grid/fetchcrl3
http://dist.eugridpma.info/distribution/util/fetch-crl3/
"""
Util.printDetail('NB! Installation of fetch-crl is not implemented for Ubuntu.')
Util.printDetail("""For manual installation see:
http://www.nikhef.nl/grid/fetchcrl3
http://dist.eugridpma.info/distribution/util/fetch-crl3/""")
def _installCAs(self):
"""CA:
https://wiki.egi.eu/wiki/EGI_IGTF_Release
http://repository.egi.eu/sw/production/cas/1/current/tgz/
"""
Util.printDetail('NB! Installation of CA is not implemented for Ubuntu.')
Util.printDetail("""For manual installation see:
https://wiki.egi.eu/wiki/EGI_IGTF_Release
http://repository.egi.eu/sw/production/cas/1/current/tgz/""")
self._installFetchCrl()
def isPackageInstalled(self, package):
cmd = self.getIsPackageInstalledCommand(package)
rc, output = self._executeWithOutput(cmd, shell=True)
if rc != 0:
Util.printDetail(output)
return False
return True
def installCAs(self):
def _isCertificateAuthority():
return Util.isTrueConfVal(getattr(self, 'certificateAuthority', False))
if not _isCertificateAuthority():
Util.printDetail('Requested not to install CAs.')
else:
self._installCAs()
self._installFetchCrl()
self._enableFetchCrl()
self._installVomsFiles()
def _configureCloudAdminSudo(self, commands):
Util.printDetail("Configuring sudo rights for '%s'" % self.oneUsername)
for cmd in commands:
replace = '%s ALL = NOPASSWD: %s' % (self.oneUsername, cmd)
self.appendOrReplaceInFileCmd('/etc/sudoers', '%s' % replace, replace)
replace = 'Defaults:%s !requiretty' % self.oneUsername
self.appendOrReplaceInFileCmd('/etc/sudoers', '%s' % replace, replace)
replace = 'Defaults:%s !requiretty' % 'root'
self.appendOrReplaceInFileCmd('/etc/sudoers', '%s' % replace, replace)
def configureCloudAdminPdiskNode(self):
pdiskAttach = "/usr/sbin/attach-persistent-disk.sh"
pdiskDetach = "/usr/sbin/detach-persistent-disk.sh"
if Util.isFalseConfVal(getattr(self, "persistentDisks", False)):
self.executeCmd('"[ -f %(pd)s ] || { touch %(pd)s; chmod +x %(pd)s; }"' % {"pd": pdiskDetach}, shell=True)
return
Util.printDetail("Configuring persistent disks management for " "'%s' user." % self.oneUsername)
line = "oneadmin ALL = NOPASSWD: %s, %s" % (pdiskAttach, pdiskDetach)
self.appendOrReplaceInFileCmd("/etc/sudoers", "^%s.*persistent-disk.*$" % self.oneUsername, line)
def _configureQemuUserOnFrontend(self):
"""Add qemu user on Fronted with the same UID and GID as on the node
being configured. Add qemu user to 'cloud' group both on Frontend
and the node.
"""
if self.shareType != 'nfs':
return
user = group = 'qemu'
getUidGidCmd = "getent passwd %s"
Util.printDetail("Configuring '%s' user on Frontend as shared filesystem setup requested." % user)
def getUidGidFromNode(user):
rc, output = self._nodeShell(getUidGidCmd % user,
withOutput=True)
if rc != 0:
Util.printError("Error getting '%s' user UID/GID from Node.\n%s" %
(user,output))
return _extractUidGidFromGetentPasswdOutput(output)
def _extractUidGidFromGetentPasswdOutput(output):
uid, gid = output.split(':')[2:4] # uid, gid
if not all([uid, gid]):
Util.printError("Error extracting '%s' user UID/GID from output.\n%s" %
(user,output))
return uid, gid
uidNode, gidNode = getUidGidFromNode(user)
rc, output = self._executeWithOutput((getUidGidCmd % uidNode).split())
if rc == 0:
uidLocal, gidLocal = _extractUidGidFromGetentPasswdOutput(output)
Util.printDetail("User with UID:%s/GID:%s already configured on Frontend." %
(uidLocal, gidLocal), verboseLevel=self.verboseLevel)
if gidNode != gidLocal:
Util.printError("Frontend user '%s' GID:%s doesn't match GID:%s on Node %s." %
(gidLocal, user, gidNode, self.nodeAddr))
else:
self._execute(['groupadd', '-g', gidNode, '-r', group])
self._execute(['useradd', '-r', '-u', uidNode, '-g', group,
'-d', '/', '-s', '/sbin/nologin',
'-c', '"%s user"'%user, user])
# Instruct libvirt to run VMs with GID of ONE group.
self.appendOrReplaceInFileCmd(self.qemuConf, '^group.*$',
'group = "%s"' % self.oneGroup)
def doWork(self):
configHolder = ConfigHolder(self.options.__dict__)
src = self._get_config_path()
dst = configHolder.configFile
if os.path.exists(dst) and not configHolder.force:
Util.printError(self.existingFileMsg % dst)
else:
try:
self._create_parent_dir(dst)
self._copy_config(src, dst)
Util.printDetail('wrote configuration file: %s' % dst)
except Exception as e:
Util.printError(e)
def installPackages(self, packages):
if len(packages) < 1:
return
self.addRepositories(packages)
packages_versioned = []
for package in packages:
packages_versioned.append(self.getPackageWithVersionForInstall(package))
cmd = "%s %s" % (self.installCmd, " ".join(packages_versioned))
rc, output = self._executeWithOutput(cmd, shell=True)
if rc != 0:
raise ExecutionException("Failed to install: %s\n%s" % (", ".join(packages_versioned), output))
Util.printDetail(output, self.verboseLevel, Util.VERBOSE_LEVEL_DETAILED)
def configureCloudAdminSshKeys(self):
keyFileName = "%s/.ssh/id_rsa" % self.oneHome
if os.path.exists(keyFileName):
Util.printDetail("Key file %s already exists, skipping this step" % keyFileName)
return
self.createDirsCmd(os.path.dirname(keyFileName))
self.setOwnerCmd(os.path.dirname(keyFileName))
self.executeCmd(['ssh-keygen -f %s -N "" -q' % keyFileName], shell=True)
self.setOwnerCmd(keyFileName)
self.setOwnerCmd("%s.pub" % keyFileName)
self.copyCmd("%s.pub" % keyFileName, "%s/.ssh/authorized_keys" % self.oneHome)
self.setOwnerCmd("%s/.ssh/authorized_keys" % self.oneHome)
self._configureCloudAdminSsh()
def generateP12(self):
self.tmp_dir = tempfile.mkdtemp()
Util.printDetail('Temporary directory for certificate generation: %s' %
self.tmp_dir, self.configHolder.verboseLevel,
Util.VERBOSE_LEVEL_DETAILED)
try:
self._generateOpensslConfig()
self._runCommandsP12()
finally:
if self.configHolder.noCleanup:
Util.printInfo('Intermediate files are in %s' % self.tmp_dir)
else:
try:
shutil.rmtree(self.tmp_dir, ignore_errors=True)
except:
pass
def _vmInfo(self, vmId):
# Hack to retry on SSL errors.
maxRetries = 3
retries = 0
while True:
try:
isSuccess, info, _ = self._rpc.one.vm.info(self._sessionString, vmId)
break
except ssl.SSLError as e:
retries += 1
t = strftime("%Y-%m-%d %H:%M:%S", gmtime())
Util.printDetail('SSL ERROR ENCOUNTERED (%s): %s' % (t, str(e)))
if retries >= maxRetries:
raise e
self._raiseIfError(isSuccess, info)
return info
def configureDatabase(self):
if self.oneDbHost in ['localhost', '127.0.0.1']:
Util.printDetail('Installing MySQL server.')
mysqlPackage = self.getPackageName('MySQLServer')
self.installPackages([mysqlPackage])
Util.printDetail('Starting MySQL server.')
mysqlService = self.getPackageInitdScriptName('MySQLServer')
self.startService(mysqlService)
Util.printDetail('Changing db root password')
self._configureRootDbUser(self.oneDbRootPassword)
Util.printDetail('Creating oneadmin db account')
self._configureDbUser(self.oneDbUsername, self.oneDbPassword)
else:
Util.printDetail('Skipping MySQL installation/configuration. It is assumed to be configured on %s' % self.oneDbHost)
def listHosts(self):
# Hack to retry on SSL errors
maxRetries = 3
retries = 0
while True:
try:
ret, info, _ = self._rpc.one.hostpool.info(self._sessionString)
break
except ssl.SSLError as e:
retries += 1
t = strftime("%Y-%m-%d %H:%M:%S", gmtime())
Util.printDetail('SSL ERROR ENCOUNTERED (%s): %s' % (t, str(e)))
if retries >= maxRetries:
raise e
if not ret:
raise OneException(info)
return info
def hostRemove(self, id):
# Hack to retry on SSL errors
maxRetries = 3
retries = 0
while True:
try:
ret = self._rpc.one.host.delete(self._sessionString, id)
break
except ssl.SSLError as e:
retries += 1
t = strftime("%Y-%m-%d %H:%M:%S", gmtime())
Util.printDetail('SSL ERROR ENCOUNTERED (%s): %s' % (t, str(e)))
if retries >= maxRetries:
raise e
if not ret:
raise OneException(id)
return id
def hostCreate(self, hostname, im, vmm, tm, vnm='dummy', inDomain=True):
# Hack to retry on SSL errors
maxRetries = 3
retries = 0
while True:
try:
ret, id, _ = self._rpc.one.host.allocate(self._sessionString, hostname, im, vmm, vnm, tm, inDomain)
break
except ssl.SSLError as e:
retries += 1
t = strftime("%Y-%m-%d %H:%M:%S", gmtime())
Util.printDetail('SSL ERROR ENCOUNTERED (%s): %s' % (t, str(e)))
if retries >= maxRetries:
raise e
if not ret:
raise OneException(id)
return id
def configureDatabase(self):
if self.oneDbHost in ["localhost", "127.0.0.1"]:
Util.printDetail("Installing MySQL server.")
mysqlPackage = self.getPackageName("MySQLServer")
self.installPackages([mysqlPackage])
Util.printDetail("Starting MySQL server.")
mysqlService = self.getPackageInitdScriptName("MySQLServer")
self.startService(mysqlService)
Util.printDetail("Changing db root password")
self._configureRootDbUser(self.oneDbRootPassword)
Util.printDetail("Creating oneadmin db account")
self._configureDbUser(self.oneDbUsername, self.oneDbPassword)
else:
Util.printDetail(
"Skipping MySQL installation/configuration. It is assumed to be configured on %s" % self.oneDbHost
)
def vmStart(self, vmTpl):
# Hack to retry on SSL errors
maxRetries = 3
retries = 0
while True:
try:
isSuccess, detail, _ = self._rpc.one.vm.allocate(self._sessionString, vmTpl)
break
except ssl.SSLError as e:
retries += 1
t = strftime("%Y-%m-%d %H:%M:%S", gmtime())
Util.printDetail('SSL ERROR ENCOUNTERED (%s): %s' % (t, str(e)))
if retries >= maxRetries:
raise e
self._raiseIfError(isSuccess, detail)
vmId = detail
return vmId
def networkCreate(self, vnetTpl):
# Hack to retry on SSL errors.
maxRetries = 3
retries = 0
while True:
try:
ret, vnetId, _ = self._rpc.one.vn.allocate(self._sessionString, vnetTpl)
break
except ssl.SSLError as e:
retries += 1
t = strftime("%Y-%m-%d %H:%M:%S", gmtime())
Util.printDetail('SSL ERROR ENCOUNTERED (%s): %s' % (t, str(e)))
if retries >= maxRetries:
raise e
if not ret:
error = vnetId
raise OneException('Error creating ONE network:\n%s' % error)
return vnetId