def _checkForMail(self):
# Lock is acquired in order to make sure that neither _numHosts nor incomingMail are changed during checkForMail
self._inLock.acquire()
try:
#self.log.debug("SPM_MailMonitor -_checking for mail")
cmd = self._inCmd + ['bs=' + str(self._outMailLen)]
#self.log.debug("SPM_MailMonitor - reading incoming mail, command: " + str(cmd))
(rc, in_mail, err) = misc.execCmd(cmd, sudo=False, raw=True)
if rc:
raise RuntimeError("_handleRequests._checkForMail - Could not read mailbox")
if (len(in_mail) != (self._outMailLen)):
self.log.error('SPM_MailMonitor: _checkForMail - dd succeeded but read %d bytes instead of %d, cannot check mail. Read mail contains: %s', len(in_mail), self._outMailLen, repr(in_mail[:80]))
raise RuntimeError("_handleRequests._checkForMail - Could not read mailbox")
#self.log.debug("Parsing inbox content: %s", in_mail)
if self._handleRequests(in_mail):
self._outLock.acquire()
try:
cmd = self._outCmd + ['bs=' + str(self._outMailLen)]
(rc, out, err) = misc.execCmd(cmd, sudo=False, data=self._outgoingMail)
if rc:
self.log.warning("SPM_MailMonitor couldn't write outgoing mail, dd failed")
finally:
self._outLock.release()
finally:
self._inLock.release()
def setupiSCSI():
"""
Set up the iSCSI daemon configuration to the known and
supported state. The original configuration, if any, is saved
"""
if os.path.exists(ISCSID_CONF):
backup = ISCSID_CONF + ".orig"
cmd = [constants.EXT_MV, ISCSID_CONF, backup]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.iSCSISetupError("Backup original iscsid.conf file")
f = tempfile.NamedTemporaryFile()
with open(ISCSID_CONF_TEMPLATE, "r") as tf:
f.write(tf)
f.flush()
cmd = [constants.EXT_CP, f.name, ISCSID_CONF]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.iSCSISetupError("Install new iscsid.conf file")
# f close also removes file - so close must be called after copy
f.close()
cmd = [constants.EXT_SERVICE, "iscsid", "stop"]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.iSCSISetupError("Stop iscsid service")
cmd = [constants.EXT_SERVICE, "iscsid", "force-start"]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.iSCSISetupError("Force-start iscsid service")
def _checkForMail(self):
# Lock is acquired in order to make sure that neither _numHosts nor incomingMail are changed during checkForMail
self._inLock.acquire()
try:
#self.log.debug("SPM_MailMonitor -_checking for mail")
cmd = self._inCmd + ['bs=' + str(self._outMailLen)]
#self.log.debug("SPM_MailMonitor - reading incoming mail, command: " + str(cmd))
(rc, in_mail, err) = misc.execCmd(cmd, sudo=False, raw=True)
if rc:
raise RuntimeError(
"_handleRequests._checkForMail - Could not read mailbox")
if (len(in_mail) != (self._outMailLen)):
self.log.error(
'SPM_MailMonitor: _checkForMail - dd succeeded but read %d bytes instead of %d, cannot check mail. Read mail contains: %s',
len(in_mail), self._outMailLen, repr(in_mail[:80]))
raise RuntimeError(
"_handleRequests._checkForMail - Could not read mailbox")
#self.log.debug("Parsing inbox content: %s", in_mail)
if self._handleRequests(in_mail):
self._outLock.acquire()
try:
cmd = self._outCmd + ['bs=' + str(self._outMailLen)]
(rc, out, err) = misc.execCmd(cmd,
sudo=False,
data=self._outgoingMail)
if rc:
self.log.warning(
"SPM_MailMonitor couldn't write outgoing mail, dd failed"
)
finally:
self._outLock.release()
finally:
self._inLock.release()
def setupMultipath():
"""
Set up the multipath daemon configuration to the known and
supported state. The original configuration, if any, is saved
"""
if os.path.exists(MPATH_CONF):
misc.rotateFiles(os.path.dirname(MPATH_CONF),
os.path.basename(MPATH_CONF),
MAX_CONF_COPIES,
cp=True,
persist=True)
with tempfile.NamedTemporaryFile() as f:
f.write(MPATH_CONF_TEMPLATE % {'scsi_id_path': _scsi_id.cmd})
f.flush()
cmd = [constants.EXT_CP, f.name, MPATH_CONF]
rc = misc.execCmd(cmd, sudo=True)[0]
if rc != 0:
raise se.MultipathSetupError()
misc.persistFile(MPATH_CONF)
# Flush all unused multipath device maps
misc.execCmd([constants.EXT_MULTIPATH, "-F"], sudo=True)
cmd = [constants.EXT_VDSM_TOOL, "service-reload", "multipathd"]
rc = misc.execCmd(cmd, sudo=True)[0]
if rc != 0:
raise se.MultipathReloadError()
def setupMultipath():
"""
Set up the multipath daemon configuration to the known and
supported state. The original configuration, if any, is saved
"""
if os.path.exists(MPATH_CONF):
misc.rotateFiles(
os.path.dirname(MPATH_CONF),
os.path.basename(MPATH_CONF), MAX_CONF_COPIES,
cp=True, persist=True)
with tempfile.NamedTemporaryFile() as f:
f.write(MPATH_CONF_TEMPLATE % {'scsi_id_path': _scsi_id.cmd})
f.flush()
cmd = [constants.EXT_CP, f.name, MPATH_CONF]
rc = misc.execCmd(cmd, sudo=True)[0]
if rc != 0:
raise se.MultipathSetupError()
misc.persistFile(MPATH_CONF)
# Flush all unused multipath device maps
misc.execCmd([constants.EXT_MULTIPATH, "-F"], sudo=True)
cmd = [constants.EXT_VDSM_TOOL, "service-reload", "multipathd"]
rc = misc.execCmd(cmd, sudo=True)[0]
if rc != 0:
raise se.MultipathReloadError()
def setupiSCSI():
"""
Set up the iSCSI daemon configuration to the known and
supported state. The original configuration, if any, is saved
"""
if os.path.exists(ISCSID_CONF):
backup = ISCSID_CONF + ".orig"
cmd = [constants.EXT_MV, ISCSID_CONF, backup]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.iSCSISetupError("Backup original iscsid.conf file")
f = tempfile.NamedTemporaryFile()
with open(ISCSID_CONF_TEMPLATE, "r") as tf:
f.write(tf)
f.flush()
cmd = [constants.EXT_CP, f.name, ISCSID_CONF]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.iSCSISetupError("Install new iscsid.conf file")
# f close also removes file - so close must be called after copy
f.close()
cmd = [constants.EXT_SERVICE, "iscsid", "stop"]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.iSCSISetupError("Stop iscsid service")
cmd = [constants.EXT_SERVICE, "iscsid", "force-start"]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.iSCSISetupError("Force-start iscsid service")
def setupMultipath():
"""
Set up the multipath daemon configuration to the known and
supported state. The original configuration, if any, is saved
"""
if os.path.exists(MPATH_CONF):
misc.rotateFiles(os.path.dirname(MPATH_CONF), os.path.basename(MPATH_CONF), MAX_CONF_COPIES, cp=True, persist=True)
f = tempfile.NamedTemporaryFile()
f.write(MPATH_CONF_TEMPLATE)
f.flush()
cmd = [constants.EXT_CP, f.name, MPATH_CONF]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.MultipathSetupError()
# f close removes file - must be after copy
f.close()
misc.persistFile(MPATH_CONF)
# Flush all unused multipath device maps
misc.execCmd([constants.EXT_MULTIPATH, "-F"])
cmd = [constants.EXT_SERVICE, "multipathd", "restart"]
rc = misc.execCmd(cmd)[0]
if rc != 0:
# No dice - try to reload instead of restart
cmd = [constants.EXT_SERVICE, "multipathd", "reload"]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.MultipathRestartError()
def setupMultipath():
"""
Set up the multipath daemon configuration to the known and
supported state. The original configuration, if any, is saved
"""
if os.path.exists(MPATH_CONF):
misc.rotateFiles(os.path.dirname(MPATH_CONF),
os.path.basename(MPATH_CONF),
MAX_CONF_COPIES,
cp=True,
persist=True)
f = tempfile.NamedTemporaryFile()
f.write(MPATH_CONF_TEMPLATE)
f.flush()
cmd = [constants.EXT_CP, f.name, MPATH_CONF]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.MultipathSetupError()
# f close removes file - must be after copy
f.close()
misc.persistFile(MPATH_CONF)
# Flush all unused multipath device maps
misc.execCmd([constants.EXT_MULTIPATH, "-F"])
cmd = [constants.EXT_SERVICE, "multipathd", "restart"]
rc = misc.execCmd(cmd)[0]
if rc != 0:
# No dice - try to reload instead of restart
cmd = [constants.EXT_SERVICE, "multipathd", "reload"]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.MultipathRestartError()
def remiSCSINode(ip,
port,
iqn,
tpgt,
username=None,
password=None,
logout=True):
"""
Remove a specific node/iSCSI target
"""
ip, port, username, password = validateiSCSIParams(ip, port, username,
password)
if port == "":
port = ISCSI_DEFAULT_PORT
portal = "%s:%s" % (ip, port)
if logout:
cmd = [
constants.EXT_ISCSIADM, "-m", "node", "-T", iqn, "-p", portal, "-u"
]
rc = misc.execCmd(cmd)[0]
if rc:
raise se.iSCSILogoutError(portal)
# FIXME: should we check if logout succeeds?
cmd = [
constants.EXT_ISCSIADM, "-m", "node", "-o", "delete", "-T", iqn, "-p",
portal
]
rc = misc.execCmd(cmd)[0]
if rc:
raise se.RemoveiSCSINodeError(portal)
def addiSCSINode(ip, port, iqn, tpgt, initiator, username=None, password=None):
"""
Add a specific node/iSCSI target
"""
ip, port, username, password = validateiSCSIParams(ip, port, username,
password)
if port == "":
port = ISCSI_DEFAULT_PORT
portal = "%s:%s" % (ip, port)
try:
addiSCSIPortal(ip, port, initiator, username, password)[0]
cmdt = [constants.EXT_ISCSIADM, "-m", "node", "-T", iqn]
if initiator:
cmdt += ["-I", initiator]
# If username or password exists assume CHAP authentication is required
if username or password:
# Set authentication type
cmd = cmdt + LOGIN_AUTH_CHAP
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.SetiSCSIAuthError(portal)
if username:
# Set username
cmd = cmdt + LOGIN_AUTH_USER + [username]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.SetiSCSIUsernameError(portal)
# Set password
cmd = cmdt + LOGIN_AUTH_PASS
rc = misc.execCmd(cmd + [password], printable=cmd + ["******"])[0]
if rc != 0:
raise se.SetiSCSIPasswdError(portal)
# Finally instruct the iscsi initiator to login to the target
cmd = cmdt + ["-l", "-p", portal]
rc = misc.execCmd(cmd)[0]
if rc == ISCSI_ERR_LOGIN_AUTH_FAILED:
raise se.iSCSILoginAuthError(portal)
elif rc not in (0, ISCSI_ERR_SESS_EXISTS):
raise se.iSCSILoginError(portal)
except se.StorageException:
exc_class, exc, tb = sys.exc_info()
# Do not try to disconnect - we may remove live node!
try:
remiSCSINode(ip, port, iqn, tpgt, username, password, logout=False)
except Exception:
log.error("Could not remove iscsi node", exc_info=True)
raise exc_class, exc, tb
def addiSCSINode(ip, port, iqn, tpgt, initiator, username=None, password=None):
"""
Add a specific node/iSCSI target
"""
ip, port, username, password = validateiSCSIParams(ip, port, username,
password)
if port == "":
port = ISCSI_DEFAULT_PORT
portal = "%s:%s" % (ip, port)
try:
addiSCSIPortal(ip, port, initiator, username, password)[0]
cmdt = [constants.EXT_ISCSIADM, "-m", "node", "-T", iqn]
if initiator:
cmdt += ["-I", initiator]
# If username or password exists assume CHAP authentication is required
if username or password:
# Set authentication type
cmd = cmdt + LOGIN_AUTH_CHAP
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.SetiSCSIAuthError(portal)
if username:
# Set username
cmd = cmdt + LOGIN_AUTH_USER + [username]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.SetiSCSIUsernameError(portal)
# Set password
cmd = cmdt + LOGIN_AUTH_PASS
rc = misc.execCmd(cmd + [password], printable=cmd + ["******"])[0]
if rc != 0:
raise se.SetiSCSIPasswdError(portal)
# Finally instruct the iscsi initiator to login to the target
cmd = cmdt + ["-l", "-p", portal]
rc = misc.execCmd(cmd)[0]
if rc == ISCSI_ERR_LOGIN_AUTH_FAILED:
raise se.iSCSILoginAuthError(portal)
elif rc not in (0, ISCSI_ERR_SESS_EXISTS):
raise se.iSCSILoginError(portal)
except se.StorageException:
exc_class, exc, tb = sys.exc_info()
# Do not try to disconnect - we may remove live node!
try:
remiSCSINode(ip, port, iqn, tpgt, username, password, logout=False)
except Exception:
log.error("Could not remove iscsi node", exc_info=True)
raise exc_class, exc, tb
def _sendMail(self):
self.log.info("HSM_MailMonitor sending mail to SPM - " +
str(self._outCmd))
chk = misc.checksum(
self._outgoingMail[0:MAILBOX_SIZE - CHECKSUM_BYTES],
CHECKSUM_BYTES)
pChk = struct.pack('<l', chk) # Assumes CHECKSUM_BYTES equals 4!!!
self._outgoingMail = self._outgoingMail[0:MAILBOX_SIZE -
CHECKSUM_BYTES] + pChk
misc.execCmd(self._outCmd, data=self._outgoingMail, sudo=False)
def _sendMail(self):
self.log.info("HSM_MailMonitor sending mail to SPM - " +
str(self._outCmd))
chk = misc.checksum(
self._outgoingMail[0:MAILBOX_SIZE - CHECKSUM_BYTES],
CHECKSUM_BYTES)
pChk = struct.pack('<l', chk) # Assumes CHECKSUM_BYTES equals 4!!!
self._outgoingMail = \
self._outgoingMail[0:MAILBOX_SIZE - CHECKSUM_BYTES] + pChk
misc.execCmd(self._outCmd, data=self._outgoingMail, sudo=False)
def mountMaster(self):
"""
Mount the master metadata file system. Should be called only by SPM.
"""
lvm.activateLVs(self.sdUUID, MASTERLV)
masterDir = os.path.join(self.domaindir, sd.MASTER_FS_DIR)
fileUtils.createdir(masterDir)
masterfsdev = lvm.lvPath(self.sdUUID, MASTERLV)
cmd = [constants.EXT_FSCK, "-p", masterfsdev]
(rc, out, err) = misc.execCmd(cmd)
# fsck exit codes
# 0 - No errors
# 1 - File system errors corrected
# 2 - File system errors corrected, system should
# be rebooted
# 4 - File system errors left uncorrected
# 8 - Operational error
# 16 - Usage or syntax error
# 32 - E2fsck canceled by user request
# 128 - Shared library error
if rc == 1 or rc == 2:
# rc is a number
self.log.info("fsck corrected fs errors (%s)", rc)
if rc >= 4:
raise se.BlockStorageDomainMasterFSCKError(masterfsdev, rc)
# TODO: Remove when upgrade is only from a version which creates ext3
# Try to add a journal - due to unfortunate circumstances we exposed
# to the public the code that created ext2 file system instead of ext3.
# In order to make up for it we are trying to add journal here, just
# to be sure (and we have fixed the file system creation).
# If there is a journal already tune2fs will do nothing, indicating this
# condition only with exit code. However, we do not really care.
cmd = [constants.EXT_TUNE2FS, "-j", masterfsdev]
misc.execCmd(cmd)
rc = fileUtils.mount(masterfsdev, masterDir, mountType=fileUtils.FSTYPE_EXT3)
# mount exit codes
# mount has the following return codes (the bits can be ORed):
# 0 success
# 1 incorrect invocation or permissions
# 2 system error (out of memory, cannot fork, no more loop devices)
# 4 internal mount bug or missing nfs support in mount
# 8 user interrupt
# 16 problems writing or locking /etc/mtab
# 32 mount failure
# 64 some mount succeeded
if rc != 0:
raise se.BlockStorageDomainMasterMountError(masterfsdev, rc, out)
cmd = [constants.EXT_CHOWN, "%s:%s" % (constants.METADATA_USER, constants.METADATA_GROUP), masterDir]
(rc, out, err) = misc.execCmd(cmd)
if rc != 0:
self.log.error("failed to chown %s", masterDir)
def mountMaster(self):
"""
Mount the master metadata file system. Should be called only by SPM.
"""
lvm.activateLVs(self.sdUUID, MASTERLV)
masterDir = os.path.join(self.domaindir, sd.MASTER_FS_DIR)
fileUtils.createdir(masterDir)
masterfsdev = lvm.lvPath(self.sdUUID, MASTERLV)
cmd = [constants.EXT_FSCK, "-p", masterfsdev]
(rc, out, err) = misc.execCmd(cmd,
sudo=True,
deathSignal=signal.SIGKILL)
# fsck exit codes
# 0 - No errors
# 1 - File system errors corrected
# 2 - File system errors corrected, system should
# be rebooted
# 4 - File system errors left uncorrected
# 8 - Operational error
# 16 - Usage or syntax error
# 32 - E2fsck canceled by user request
# 128 - Shared library error
if rc == 1 or rc == 2:
# rc is a number
self.log.info("fsck corrected fs errors (%s)", rc)
if rc >= 4:
raise se.BlockStorageDomainMasterFSCKError(masterfsdev, rc)
# TODO: Remove when upgrade is only from a version which creates ext3
# Try to add a journal - due to unfortunate circumstances we exposed
# to the public the code that created ext2 file system instead of ext3.
# In order to make up for it we are trying to add journal here, just
# to be sure (and we have fixed the file system creation).
# If there is a journal already tune2fs will do nothing, indicating
# this condition only with exit code. However, we do not really care.
cmd = [constants.EXT_TUNE2FS, "-j", masterfsdev]
misc.execCmd(cmd, sudo=True, deathSignal=signal.SIGKILL)
masterMount = mount.Mount(masterfsdev, masterDir)
try:
masterMount.mount(vfstype=mount.VFS_EXT3)
except mount.MountError as ex:
rc, out = ex
raise se.BlockStorageDomainMasterMountError(masterfsdev, rc, out)
cmd = [
constants.EXT_CHOWN,
"%s:%s" % (constants.METADATA_USER, constants.METADATA_GROUP),
masterDir
]
(rc, out, err) = misc.execCmd(cmd, sudo=True)
if rc != 0:
self.log.error("failed to chown %s", masterDir)
def rescan():
"""
Forces multipath daemon to rescan the list of available devices and
refresh the mapping table. New devices can be found under /dev/mapper
Should only be called from hsm._rescanDevices()
"""
# First ask iSCSI to rescan all its sessions
iscsi.rescan()
# Now let multipath daemon pick up new devices
misc.execCmd([constants.EXT_MULTIPATH], sudo=True)
def rescan():
"""
Forces multipath daemon to rescan the list of available devices and
refresh the mapping table. New devices can be found under /dev/mapper
Should only be called from hsm._rescanDevices()
"""
# First ask iSCSI to rescan all its sessions
iscsi.rescan()
# Now let multipath daemon pick up new devices
misc.execCmd([constants.EXT_MULTIPATH], sudo=True)
def mountMaster(self):
"""
Mount the master metadata file system. Should be called only by SPM.
"""
lvm.activateLVs(self.sdUUID, MASTERLV)
masterDir = os.path.join(self.domaindir, sd.MASTER_FS_DIR)
fileUtils.createdir(masterDir)
masterfsdev = lvm.lvPath(self.sdUUID, MASTERLV)
cmd = [constants.EXT_FSCK, "-p", masterfsdev]
(rc, out, err) = misc.execCmd(cmd, sudo=True,
deathSignal=signal.SIGKILL)
# fsck exit codes
# 0 - No errors
# 1 - File system errors corrected
# 2 - File system errors corrected, system should
# be rebooted
# 4 - File system errors left uncorrected
# 8 - Operational error
# 16 - Usage or syntax error
# 32 - E2fsck canceled by user request
# 128 - Shared library error
if rc == 1 or rc == 2:
# rc is a number
self.log.info("fsck corrected fs errors (%s)", rc)
if rc >= 4:
raise se.BlockStorageDomainMasterFSCKError(masterfsdev, rc)
# TODO: Remove when upgrade is only from a version which creates ext3
# Try to add a journal - due to unfortunate circumstances we exposed
# to the public the code that created ext2 file system instead of ext3.
# In order to make up for it we are trying to add journal here, just
# to be sure (and we have fixed the file system creation).
# If there is a journal already tune2fs will do nothing, indicating
# this condition only with exit code. However, we do not really care.
cmd = [constants.EXT_TUNE2FS, "-j", masterfsdev]
misc.execCmd(cmd, sudo=True, deathSignal=signal.SIGKILL)
masterMount = mount.Mount(masterfsdev, masterDir)
try:
masterMount.mount(vfstype=mount.VFS_EXT3)
except mount.MountError as ex:
rc, out = ex
raise se.BlockStorageDomainMasterMountError(masterfsdev, rc, out)
cmd = [constants.EXT_CHOWN, "%s:%s" %
(constants.METADATA_USER, constants.METADATA_GROUP), masterDir]
(rc, out, err) = misc.execCmd(cmd, sudo=True)
if rc != 0:
self.log.error("failed to chown %s", masterDir)
def rescan():
"""
Forces multipath daemon to rescan the list of available devices and
refresh the mapping table. New devices can be found under /dev/mapper
Should only be called from hsm._rescanDevices()
"""
# First rescan iSCSI and FCP connections
iscsi.rescan()
supervdsm.getProxy().hbaRescan()
# Now let multipath daemon pick up new devices
misc.execCmd([constants.EXT_MULTIPATH], sudo=True)
def cmd(self, cmd):
finalCmd = self._addExtraCfg(cmd)
rc, out, err = misc.execCmd(finalCmd)
if rc != 0:
#Filter might be stale
self.invalidateFilter()
newCmd = self._addExtraCfg(cmd)
# Before blindly trying again make sure
# that the commands are not identical, because
# the devlist is sorted there is no fear
# of two identical filters looking differently
if newCmd != finalCmd:
return misc.execCmd(newCmd)
return rc, out, err
def cmd(self, cmd, devices=tuple()):
finalCmd = self._addExtraCfg(cmd, devices)
rc, out, err = misc.execCmd(finalCmd, sudo=True)
if rc != 0:
# Filter might be stale
self.invalidateFilter()
newCmd = self._addExtraCfg(cmd)
# Before blindly trying again make sure
# that the commands are not identical, because
# the devlist is sorted there is no fear
# of two identical filters looking differently
if newCmd != finalCmd:
return misc.execCmd(newCmd, sudo=True)
return rc, out, err
def initLock(self):
lockUtil = constants.EXT_SAFELEASE
initCommand = [lockUtil, "release", "-f", self._leasesPath, "0"]
rc, out, err = misc.execCmd(initCommand, cwd=self.lockUtilPath)
if rc != 0:
self.log.warn("could not initialise spm lease (%s): %s", rc, out)
raise se.ClusterLockInitError()
def addiSCSIiface(initiator):
"""
Create the iSCSI iface with the given initiator name.
For the sake of simplicity the iface is created with the same name
as an initiator. It makes the bookkeeping trivial.
"""
cmd = ISCSIADM_IFACE + ["-o", "new", "-I", initiator]
rc, out, err = misc.execCmd(cmd)
if rc != 0:
raise se.iSCSIifaceError()
cmd = ISCSIADM_IFACE + ["-o", "update", "-I", initiator, "-n",
"iface.initiatorname", "-v", initiator]
rc, out, err = misc.execCmd(cmd)
if rc != 0:
raise se.iSCSIifaceError()
def addiSCSIPortal(ip, port, initiator, username=None, password=None):
"""
Attempts SendTarget discovery at the portal ip:port.
"""
if port == "":
port = ISCSI_DEFAULT_PORT
ip, port, username, password = validateiSCSIParams(ip, port, username,
password)
portal = "%s:%s" % (ip, port)
cmd = SENDTARGETS_DISCOVERY + ["-p", portal]
if initiator:
if initiator not in getiSCSIifaces():
addiSCSIiface(initiator)
cmd += ["-I", initiator]
if username or password:
_configureAuthInformation(cmd, username, password)
cmd.extend(AUTH_EXEC_DISCOVER)
(rc, out, err) = misc.execCmd(cmd)
if rc != 0:
raise se.iSCSIDiscoveryError(portal, err)
return rc, out
def initLock(self):
lockUtil = os.path.join(self.lockUtilPath, "safelease")
initCommand = [lockUtil, "release", "-f", self._leasesPath, "0"]
rc, out, err = misc.execCmd(initCommand, cwd=self.lockUtilPath)
if rc != 0:
self.log.warn("could not initialise spm lease (%s): %s", rc, out)
raise se.ClusterLockInitError()
def umount(resource=None,
mountPoint=None,
mountType=None,
force=True,
lazy=False):
"""
Unmount the requested resource from the associated mount point
"""
if mountPoint is None and resource is None:
raise ValueError("`mountPoint` or `resource` must be specified")
if not isMounted(resource, mountPoint):
if isMounted(mountPoint=mountPoint):
return -1
elif isMounted(resource=resource):
return -1
return 0
options = []
if mountType is not None:
options.extend(["-t", mountType])
if force:
options.append("-f")
if lazy:
options.append("-l")
cmd = [constants.EXT_UMOUNT
] + options + [mountPoint if mountPoint is not None else resource]
rc = misc.execCmd(cmd)[0]
return rc
def umount(resource=None, mountPoint=None, mountType=None, force=True, lazy=False):
"""
Unmount the requested resource from the associated mount point
"""
if mountPoint is None and resource is None:
raise ValueError("`mountPoint` or `resource` must be specified")
if not isMounted(resource, mountPoint):
if isMounted(mountPoint=mountPoint):
return -1
elif isMounted(resource=resource):
return -1
return 0
options = []
if mountType is not None:
options.extend(["-t", mountType])
if force:
options.append("-f")
if lazy:
options.append("-l")
cmd = [constants.EXT_UMOUNT] + options + [mountPoint if mountPoint is not None else resource]
rc = misc.execCmd(cmd)[0]
return rc
def mount(resource, mountPoint, mountType):
"""
Mount the requested resource
"""
if isStaleHandle(mountPoint):
rc = umount(resource, mountPoint, mountType)
if rc != 0:
return rc
if isMounted(resource, mountPoint, mountType):
return 0
if mountType == FSTYPE_NFS:
cmd = [constants.EXT_MOUNT, "-o", NFS_OPTIONS, "-t", FSTYPE_NFS, resource, mountPoint]
elif mountType in [FSTYPE_EXT3, FSTYPE_EXT4]:
options = []
if os.path.isdir(resource):
# We should perform bindmount here,
# because underlying resource is FS and not a block device
options.append("-B")
cmd = [constants.EXT_MOUNT] + options + [resource, mountPoint]
else:
raise se.MountTypeError()
rc = misc.execCmd(cmd)[0]
return rc
def acquire(self, hostID):
leaseTimeMs = self._leaseTimeSec * 1000
ioOpTimeoutMs = self._ioOpTimeoutSec * 1000
with self._lock:
self.log.debug("Acquiring cluster lock for domain %s" %
self._sdUUID)
lockUtil = os.path.join(self.lockUtilPath, self.lockCmd)
acquireLockCommand = subprocess.list2cmdline([
lockUtil, "start", self._sdUUID,
str(hostID),
str(self._lockRenewalIntervalSec),
str(self._leaseFile),
str(leaseTimeMs),
str(ioOpTimeoutMs),
str(self._leaseFailRetry)
])
cmd = [
constants.EXT_SETSID, constants.EXT_IONICE, '-c1', '-n0',
constants.EXT_SU, misc.IOUSER, '-s', constants.EXT_SH, '-c',
acquireLockCommand
]
(rc, out, err) = misc.execCmd(cmd, cwd=self.lockUtilPath)
if rc != 0:
raise se.AcquireLockFailure(self._sdUUID, rc, out, err)
self.__hostID = hostID
self.log.debug("Clustered lock acquired successfully")
def initLock(self):
lockUtil = os.path.join(self.lockUtilPath, "safelease")
initCommand = [lockUtil, "release", "-f", self._leasesPath, "0"]
rc, out, err = misc.execCmd(initCommand, cwd=self.lockUtilPath)
if rc != 0:
self.log.warn("could not initialise spm lease (%s): %s", rc, out)
raise se.ClusterLockInitError()
def mount(resource, mountPoint, mountType):
"""
Mount the requested resource
"""
if isStaleHandle(mountPoint):
rc = umount(resource, mountPoint, mountType)
if rc != 0:
return rc
if isMounted(resource, mountPoint, mountType):
return 0
if mountType == FSTYPE_NFS:
cmd = [
constants.EXT_MOUNT, "-o", NFS_OPTIONS, "-t", FSTYPE_NFS, resource,
mountPoint
]
elif mountType in [FSTYPE_EXT3, FSTYPE_EXT4]:
options = []
if os.path.isdir(resource):
# We should perform bindmount here,
# because underlying resource is FS and not a block device
options.append("-B")
cmd = [constants.EXT_MOUNT] + options + [resource, mountPoint]
else:
raise se.MountTypeError()
rc = misc.execCmd(cmd)[0]
return rc
def acquire(self, hostID):
leaseTimeMs = self._leaseTimeSec * 1000
ioOpTimeoutMs = self._ioOpTimeoutSec * 1000
with self._lock:
self.log.debug("Acquiring cluster lock for domain %s" %
self._sdUUID)
lockUtil = self.getLockUtilFullPath()
acquireLockCommand = subprocess.list2cmdline([
lockUtil, "start", self._sdUUID,
str(hostID),
str(self._lockRenewalIntervalSec),
str(self._leasesPath),
str(leaseTimeMs),
str(ioOpTimeoutMs),
str(self._leaseFailRetry)
])
cmd = [
constants.EXT_SU, misc.IOUSER, '-s', constants.EXT_SH, '-c',
acquireLockCommand
]
(rc, out, err) = misc.execCmd(cmd,
cwd=self.lockUtilPath,
sudo=True,
ioclass=utils.IOCLASS.REALTIME,
ioclassdata=0,
setsid=True)
if rc != 0:
raise se.AcquireLockFailure(self._sdUUID, rc, out, err)
self.log.debug("Clustered lock acquired successfully")
def addiSCSIPortal(ip, port, initiator, username=None, password=None):
"""
Attempts SendTarget discovery at the portal ip:port.
"""
if port == "":
port = ISCSI_DEFAULT_PORT
ip, port, username, password = validateiSCSIParams(ip, port, username,
password)
portal = "%s:%s" % (ip, port)
cmd = SENDTARGETS_DISCOVERY + ["-p", portal]
if initiator:
if initiator not in getiSCSIifaces():
addiSCSIiface(initiator)
cmd += ["-I", initiator]
if username or password:
_configureAuthInformation(cmd, username, password)
cmd.extend(AUTH_EXEC_DISCOVER)
(rc, out, err) = misc.execCmd(cmd)
if rc != 0:
raise se.iSCSIDiscoveryError(portal, err)
return rc, out
def initLock(cls, path):
lockUtil = os.path.join(cls.lockUtilPath, "safelease")
initCommand = [ lockUtil, "release", "-f", str(path), "0" ]
rc, out, err = misc.execCmd(initCommand, sudo=False, cwd=cls.lockUtilPath)
if rc != 0:
cls.log.warn("could not initialise spm lease (%s): %s", rc, out)
raise se.ClusterLockInitError()
def initLock(self):
lockUtil = constants.EXT_SAFELEASE
initCommand = [lockUtil, "release", "-f", self._leasesPath, "0"]
rc, out, err = misc.execCmd(initCommand, cwd=self.lockUtilPath)
if rc != 0:
self.log.warn("could not initialise spm lease (%s): %s", rc, out)
raise se.ClusterLockInitError()
def _createVMSfs(dev):
"""
Create a special file system to store VM data
"""
cmd = [constants.EXT_MKFS, "-q", "-j", "-E", "nodiscard", dev]
rc = misc.execCmd(cmd, sudo=True, deathSignal=signal.SIGKILL)[0]
if rc != 0:
raise se.MkfsError(dev)
def remiSCSIiface(initiator):
"""
Remove the iface with the given initiator name.
"""
cmd = ISCSIADM_IFACE + ["-o", "delete", "-I", initiator]
rc, out, err = misc.execCmd(cmd)
if rc != 0:
raise se.iSCSIifaceError()
def _initMailbox(self):
# Sync initial incoming mail state with storage view
(rc, out, err) = misc.execCmd(self._inCmd, sudo=False, raw=True)
if rc == 0:
self._incomingMail = out
self._init = True
else:
self.log.warning("HSM_MailboxMonitor - Could not initialize mailbox, will not accept requests until init succeeds")
def remiSCSIiface(initiator):
"""
Remove the iface with the given initiator name.
"""
cmd = ISCSIADM_IFACE + ["-o", "delete", "-I", initiator]
rc, out, err = misc.execCmd(cmd)
if rc != 0:
raise se.iSCSIifaceError()
def _createVMSfs(dev):
"""
Create a special file system to store VM data
"""
cmd = [constants.EXT_MKFS, "-q", "-j", "-K", dev]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.MkfsError(dev)
def _createVMSfs(dev):
"""
Create a special file system to store VM data
"""
cmd = [constants.EXT_MKFS, "-q", "-j", "-E", "nodiscard", dev]
rc = misc.execCmd(cmd, sudo=True, deathSignal=signal.SIGKILL)[0]
if rc != 0:
raise se.MkfsError(dev)
def _createVMSfs(dev):
"""
Create a special file system to store VM data
"""
cmd = [constants.EXT_MKFS, "-q", "-j", "-K", dev]
rc = misc.execCmd(cmd)[0]
if rc != 0:
raise se.MkfsError(dev)
def __init__(self, pool, maxHostID, monitorInterval=2):
self._messageTypes = {}
# Save arguments
self._stop = False
self._stopped = False
self._poolID = str(pool.spUUID)
self._spmStorageDir = pool.storage_repository
tpSize = config.getfloat('irs', 'thread_pool_size') / 2
waitTimeout = 3
maxTasks = config.getfloat('irs', 'max_tasks')
self.tp = ThreadPool(tpSize, waitTimeout, maxTasks)
# *** IMPORTANT NOTE: The SPM's inbox is the HSMs' outbox and vice
# versa *** #
self._inbox = os.path.join(self._spmStorageDir, self._poolID,
"mastersd", sd.DOMAIN_META_DATA, "inbox")
if not os.path.exists(self._inbox):
self.log.error("SPM_MailMonitor create failed - inbox %s does not "
"exist" % repr(self._inbox))
raise RuntimeError("SPM_MailMonitor create failed - inbox %s does "
"not exist" % repr(self._inbox))
self._outbox = os.path.join(self._spmStorageDir, self._poolID,
"mastersd", sd.DOMAIN_META_DATA, "outbox")
if not os.path.exists(self._outbox):
self.log.error("SPM_MailMonitor create failed - outbox %s does "
"not exist" % repr(self._outbox))
raise RuntimeError("SPM_MailMonitor create failed - outbox %s "
"does not exist" % repr(self._outbox))
self._numHosts = int(maxHostID)
self._outMailLen = MAILBOX_SIZE * self._numHosts
self._monitorInterval = monitorInterval
# TODO: add support for multiple paths (multiple mailboxes)
self._outgoingMail = self._outMailLen * "\0"
self._incomingMail = self._outgoingMail
self._inCmd = ['dd',
'if=' + str(self._inbox),
'iflag=direct,fullblock',
'count=1'
]
self._outCmd = ['dd',
'of=' + str(self._outbox),
'oflag=direct',
'iflag=fullblock',
'conv=notrunc',
'count=1'
]
self._outLock = thread.allocate_lock()
self._inLock = thread.allocate_lock()
# Clear outgoing mail
self.log.debug("SPM_MailMonitor - clearing outgoing mail, command is: "
"%s", self._outCmd)
cmd = self._outCmd + ['bs=' + str(self._outMailLen)]
(rc, out, err) = misc.execCmd(cmd, sudo=False, data=self._outgoingMail)
if rc:
self.log.warning("SPM_MailMonitor couldn't clear outgoing mail, "
"dd failed")
thread.start_new_thread(self.run, (self, ))
self.log.debug('SPM_MailMonitor created for pool %s' % self._poolID)
def __init__(self, pool, maxHostID, monitorInterval=2):
self._messageTypes = {}
# Save arguments
self._stop = False
self._stopped = False
self._poolID = str(pool.spUUID)
self._spmStorageDir = pool.storage_repository
tpSize = config.getfloat('irs', 'thread_pool_size') / 2
waitTimeout = 3
maxTasks = config.getfloat('irs', 'max_tasks')
self.tp = ThreadPool(tpSize, waitTimeout, maxTasks)
# *** IMPORTANT NOTE: The SPM's inbox is the HSMs' outbox and vice versa *** #
self._inbox = os.path.join(self._spmStorageDir, self._poolID,
"mastersd", sd.DOMAIN_META_DATA, "inbox")
if not os.path.exists(self._inbox):
self.log.error(
"SPM_MailMonitor create failed - inbox %s does not exist" %
repr(self._inbox))
raise RuntimeError(
"SPM_MailMonitor create failed - inbox %s does not exist" %
repr(self._inbox))
self._outbox = os.path.join(self._spmStorageDir, self._poolID,
"mastersd", sd.DOMAIN_META_DATA, "outbox")
if not os.path.exists(self._outbox):
self.log.error(
"SPM_MailMonitor create failed - outbox %s does not exist" %
repr(self._outbox))
raise RuntimeError(
"SPM_MailMonitor create failed - outbox %s does not exist" %
repr(self._outbox))
self._numHosts = int(maxHostID)
self._outMailLen = MAILBOX_SIZE * self._numHosts
self._monitorInterval = monitorInterval
# TODO: add support for multiple paths (multiple mailboxes)
self._outgoingMail = self._outMailLen * "\0"
self._incomingMail = self._outgoingMail
self._inCmd = [
'dd', 'if=' + str(self._inbox), 'iflag=direct,fullblock', 'count=1'
]
self._outCmd = [
'dd', 'of=' + str(self._outbox), 'oflag=direct', 'iflag=fullblock',
'conv=notrunc', 'count=1'
]
self._outLock = thread.allocate_lock()
self._inLock = thread.allocate_lock()
# Clear outgoing mail
self.log.debug(
"SPM_MailMonitor - clearing outgoing mail, command is: %s",
self._outCmd)
cmd = self._outCmd + ['bs=' + str(self._outMailLen)]
(rc, out, err) = misc.execCmd(cmd, sudo=False, data=self._outgoingMail)
if rc:
self.log.warning(
"SPM_MailMonitor couldn't clear outgoing mail, dd failed")
thread.start_new_thread(self.run, (self, ))
self.log.debug('SPM_MailMonitor created for pool %s' % self._poolID)
def addiSCSIiface(initiator):
"""
Create the iSCSI iface with the given initiator name.
For the sake of simplicity the iface is created with the same name
as an initiator. It makes the bookkeeping trivial.
"""
cmd = ISCSIADM_IFACE + ["-o", "new", "-I", initiator]
rc, out, err = misc.execCmd(cmd)
if rc != 0:
raise se.iSCSIifaceError()
cmd = ISCSIADM_IFACE + [
"-o", "update", "-I", initiator, "-n", "iface.initiatorname", "-v",
initiator
]
rc, out, err = misc.execCmd(cmd)
if rc != 0:
raise se.iSCSIifaceError()
def initLock(cls, path):
lockUtil = os.path.join(cls.lockUtilPath, "safelease")
initCommand = [lockUtil, "release", "-f", str(path), "0"]
rc, out, err = misc.execCmd(initCommand,
sudo=False,
cwd=cls.lockUtilPath)
if rc != 0:
cls.log.warn("could not initialise spm lease (%s): %s", rc, out)
raise se.ClusterLockInitError()
def createVolume(parent, parent_format, volume, size, format, prealloc):
"""
--- Create new volume.
'parent' - backing volume name
'parent_format' - backing volume format
'volume' - new volume name
'format' - volume format [ 'COW' or 'RAW' ]
'size' - in sectors, always multiple of the grain size (64KB)
'preallocate' - flag PREALLOCATED_VOL/SPARSE_VOL,
defines actual storage device type.
PREALLOCATED_VOL = preallocated storage using
non-sparse format (+ DD for file, use raw LV for SAN)
# SAN
Prealloc/RAW = Normal LV (if snapshot => create copy of LV)
Sparse/RAW = if snapshot create LVM snapshot
(in the future, use storage backend thin provisioning),
else create Normal LV <== Not supported
Prealloc/COW = build qcow2 image within a preallocated space -
used only for COPY
Sparse/COW = QCOW2 over LV
# File
Prealloc/RAW = Normal file + DD (implicit pre-zero)
Sparse/RAW = Normal file (touch)
Prealloc/COW = QCOW2 + DD <== Not supported
Sparse/COW = QCOW2
"""
# TODO: accept size only in bytes and convert before call to qemu-img
cmd = [constants.EXT_QEMUIMG, "create", "-f", fmt2str(format)]
cwd = None
if format == COW_FORMAT and parent:
# cmd += ["-b", parent, volume]
# cwd = os.path.split(os.path.split(volume)[0])[0]
# Temporary fix for qemu-img creation problem
cmd += ["-F", parent_format, "-b", os.path.join("..", parent), volume]
cwd = os.path.split(volume)[0]
else:
size = int(size)
if size < 1:
raise se.createVolumeSizeError()
# qemu-img expects size to be in kilobytes by default,
# can also accept size in M or G with appropriate suffix
# +1 is so that odd numbers will round upwards.
cmd += [volume, "%uK" % ((size + 1) / 2)]
(rc, out, err) = misc.execCmd(cmd, sudo=False, cwd=cwd,
deathSignal=signal.SIGKILL)
if rc:
raise se.VolumeCreationError(out)
return True
def _checkForMail(self):
#self.log.debug("HSM_MailMonitor - checking for mail")
#self.log.debug("Running command: " + str(self._inCmd))
(rc, in_mail, err) = misc.execCmd(self._inCmd, sudo=False, raw=True)
if rc:
raise RuntimeError("_handleResponses.Could not read mailbox - rc %s" % rc)
if (len(in_mail) != MAILBOX_SIZE):
raise RuntimeError("_handleResponses.Could not read mailbox - len %s != %s" % (len(in_mail), MAILBOX_SIZE))
#self.log.debug("Parsing inbox content: %s", in_mail)
return self._handleResponses(in_mail)
def _initMailbox(self):
# Sync initial incoming mail state with storage view
(rc, out, err) = misc.execCmd(self._inCmd, sudo=False, raw=True)
if rc == 0:
self._incomingMail = out
self._init = True
else:
self.log.warning(
"HSM_MailboxMonitor - Could not initialize mailbox, will not accept requests until init succeeds"
)
def createVolume(parent, parent_format, volume, size, format, prealloc):
"""
--- Create new volume.
'parent' - backing volume name
'parent_format' - backing volume format
'volume' - new volume name
'format' - volume format [ 'COW' or 'RAW' ]
'size' - in sectors, always multiple of the grain size (64KB)
'preallocate' - flag PREALLOCATED_VOL/SPARSE_VOL,
defines actual storage device type.
PREALLOCATED_VOL = preallocated storage using
non-sparse format (+ DD for file, use raw LV for SAN)
# SAN
Prealloc/RAW = Normal LV (if snapshot => create copy of LV)
Sparse/RAW = if snapshot create LVM snapshot
(in the future, use storage backend thin provisioning),
else create Normal LV <== Not supported
Prealloc/COW = build qcow2 image within a preallocated space -
used only for COPY
Sparse/COW = QCOW2 over LV
# File
Prealloc/RAW = Normal file + DD (implicit pre-zero)
Sparse/RAW = Normal file (touch)
Prealloc/COW = QCOW2 + DD <== Not supported
Sparse/COW = QCOW2
"""
# TODO: accept size only in bytes and convert before call to qemu-img
cmd = [constants.EXT_QEMUIMG, "create", "-f", fmt2str(format)]
cwd = None
if format == COW_FORMAT and parent:
# cmd += ["-b", parent, volume]
# cwd = os.path.split(os.path.split(volume)[0])[0]
# Temporary fix for qemu-img creation problem
cmd += ["-F", parent_format, "-b", os.path.join("..", parent), volume]
cwd = os.path.split(volume)[0]
else:
size = int(size)
if size < 1:
raise se.createVolumeSizeError()
# qemu-img expects size to be in kilobytes by default,
# can also accept size in M or G with appropriate suffix
# +1 is so that odd numbers will round upwards.
cmd += [volume, "%uK" % ((size + 1) / 2)]
(rc, out, err) = misc.execCmd(cmd, sudo=False, cwd=cwd)
if rc:
raise se.VolumeCreationError(out)
return True
def fuser(path, mountPoint=False):
cmd = [constants.EXT_FUSER]
if mountPoint:
cmd.append("-m")
cmd.append(path)
(rc, out, err) = misc.execCmd(cmd, sudo=True)
if rc != 0:
return []
return [int(pid) for pid in out.split()]
def rescan():
"""
Forces multipath daemon to rescan the list of available devices and
refresh the mapping table. New devices can be found under /dev/mapper
Should only be called from hsm._rescanDevices()
"""
# First rescan iSCSI and FCP connections
iscsi.rescan()
hba.rescan()
# Now let multipath daemon pick up new devices
misc.execCmd([constants.EXT_MULTIPATH], sudo=True)
# Scanning SCSI interconnects starts a storm of udev events. Wait until all
# events are processed, ensuring detection of new devices and creation or
# update of multipath devices.
timeout = config.getint('irs', 'scsi_settle_timeout')
udevadm.settle(timeout)
def fuser(path, mountPoint=False):
cmd = [constants.EXT_FUSER]
if mountPoint:
cmd.append("-m")
cmd.append(path)
(rc, out, err) = misc.execCmd(cmd, raw=True, sudo=False)
if rc != 0:
return []
return [int(pid) for pid in out.split()]
def getScsiSerial(physdev):
blkdev = os.path.join("/dev", physdev)
cmd = [
_scsi_id.cmd, "--page=0x80", "--whitelisted", "--export",
"--replace-whitespace", "--device=" + blkdev
]
(rc, out, err) = misc.execCmd(cmd)
if rc == 0:
for line in out:
if line.startswith("ID_SERIAL="):
return line.split("=")[1]
return ""
def _checkForMail(self):
# self.log.debug("HSM_MailMonitor - checking for mail")
# self.log.debug("Running command: " + str(self._inCmd))
(rc, in_mail, err) = misc.execCmd(self._inCmd, raw=True)
if rc:
raise RuntimeError("_handleResponses.Could not read mailbox - rc "
"%s" % rc)
if (len(in_mail) != MAILBOX_SIZE):
raise RuntimeError("_handleResponses.Could not read mailbox - len "
"%s != %s" % (len(in_mail), MAILBOX_SIZE))
# self.log.debug("Parsing inbox content: %s", in_mail)
return self._handleResponses(in_mail)
def _runcmd(self, cmd, timeout):
p = misc.execCmd(cmd, sudo=True, sync=False)
if not p.wait(timeout):
p.kill()
raise OSError(errno.ETIMEDOUT, "%s operation timed out" % os.path.basename(cmd[0]))
out, err = p.communicate()
rc = p.returncode
if rc == 0:
return
raise MountError(rc, ";".join((out, err)))
def getScsiSerial(physdev):
blkdev = os.path.join("/dev", physdev)
cmd = [constants.EXT_SCSI_ID, "--page=0x80",
"--whitelisted",
"--export",
"--replace-whitespace",
"--device=" + blkdev]
(rc, out, err) = misc.execCmd(cmd, sudo=False)
if rc == 0:
for line in out:
if line.startswith("ID_SERIAL="):
return line.split("=")[1]
return ""
def release(self):
with self._lock:
freeLockUtil = os.path.join(self.lockUtilPath, self.freeLockCmd)
releaseLockCommand = [freeLockUtil, self._sdUUID]
self.log.info("Releasing cluster lock for domain %s" %
self._sdUUID)
(rc, out, err) = misc.execCmd(releaseLockCommand,
cwd=self.lockUtilPath)
if rc != 0:
self.log.error("Could not release cluster lock "
"rc=%s out=%s, err=%s" % (str(rc), out, err))
self.log.debug("Cluster lock released successfully")
