def _create(cls, dom, imgUUID, volUUID, size, volFormat, preallocate,
volParent, srcImgUUID, srcVolUUID, volPath,
initialSize=None):
"""
Class specific implementation of volumeCreate. All the exceptions are
properly handled and logged in volume.create()
"""
if initialSize:
cls.log.error("initialSize is not supported for file-based "
"volumes")
raise se.InvalidParameterException("initial size",
initialSize)
sizeBytes = size * BLOCK_SIZE
truncSize = sizeBytes if volFormat == sc.RAW_FORMAT else 0
try:
oop.getProcessPool(dom.sdUUID).truncateFile(
volPath, truncSize, mode=sc.FILE_VOLUME_PERMISSIONS,
creatExcl=True)
except OSError as e:
if e.errno == errno.EEXIST:
raise se.VolumeAlreadyExists(volUUID)
raise
if preallocate == sc.PREALLOCATED_VOL:
try:
# ddWatchCopy expects size to be in bytes
misc.ddWatchCopy("/dev/zero", volPath,
vars.task.aborting, sizeBytes)
except exception.ActionStopped:
raise
except Exception:
cls.log.error("Unexpected error", exc_info=True)
raise se.VolumesZeroingError(volPath)
if not volParent:
cls.log.info("Request to create %s volume %s with size = %s "
"sectors", sc.type2name(volFormat), volPath,
size)
if volFormat == sc.COW_FORMAT:
qemuimg.create(volPath,
size=sizeBytes,
format=sc.fmt2str(volFormat),
qcow2Compat=dom.qcow2_compat())
else:
# Create hardlink to template and its meta file
cls.log.info("Request to create snapshot %s/%s of volume %s/%s",
imgUUID, volUUID, srcImgUUID, srcVolUUID)
volParent.clone(volPath, volFormat)
# Forcing the volume permissions in case one of the tools we use
# (dd, qemu-img, etc.) will mistakenly change the file permissiosn.
dom.oop.os.chmod(volPath, sc.FILE_VOLUME_PERMISSIONS)
return (volPath,)
def testStop(self):
"""
Test that stop really stops the copying process.
"""
try:
with tempfile.NamedTemporaryFile() as src:
os.unlink(src.name)
os.mkfifo(src.name)
with tempfile.NamedTemporaryFile() as dst:
misc.ddWatchCopy(src.name, dst.name, lambda: True, 100)
except exception.ActionStopped:
self.log.info("Looks like it stopped!")
else:
self.fail("Copying didn't stop!")
def testStop(self):
"""
Test that stop really stops the copying process.
"""
try:
with tempfile.NamedTemporaryFile() as src:
os.unlink(src.name)
os.mkfifo(src.name)
with tempfile.NamedTemporaryFile() as dst:
misc.ddWatchCopy(src.name, dst.name, lambda: True, 100)
except exception.ActionStopped:
self.log.info("Looks like it stopped!")
else:
self.fail("Copying didn't stop!")
def testCopy(self):
"""
Test that regular copying works.
"""
# Prepare source
data = "Everything starts somewhere, " + \
"though many physicists disagree." + \
"But people have always been dimly aware of the " + \
"problem with the start of things." + \
"They wonder how the snowplough driver gets to work, or " + \
"how the makers of dictionaries look up the spelling of words."
# (C) Terry Pratchet - Small Gods
# Makes sure we round up to a complete block size
data *= 512
with temporaryPath(perms=0o666, data=data) as srcPath:
with temporaryPath(perms=0o666) as dstPath:
# Copy
rc, out, err = misc.ddWatchCopy(srcPath, dstPath,
None, len(data))
# Get copied data
with open(dstPath) as f:
readData = f.read()
# Comapre
self.assertEqual(readData, data)
def testNonAlignedAppend(self):
data = "ABCD" * 256 # 1Kb
add_data = "E"
repetitions = misc.MEGA / len(data)
path = self._createDataFile(data, repetitions)
try:
with open(path, "a") as f: # Appending additional data
f.write(add_data)
self.assertEqual(os.stat(path).st_size, misc.MEGA + len(add_data))
# Using os.stat(path).st_size is part of the test, please do not
# remove or change.
rc, out, err = misc.ddWatchCopy(
"/dev/zero", path, None, misc.MEGA, os.stat(path).st_size)
self.assertEqual(rc, 0)
self.assertEquals(os.stat(path).st_size,
misc.MEGA * 2 + len(add_data))
with open(path, "r") as f:
for i in range(repetitions):
self.assertEqual(f.read(len(data)), data)
# Checking the additional data
self.assertEqual(f.read(len(add_data)), add_data)
finally:
os.unlink(path)
def testNonAlignedCopy(self):
"""
Test that copying a file with odd length works.
"""
data = '- "What\'re quantum mechanics?"' + \
'- "I don\'t know. People who repair quantums, I suppose."'
# (C) Terry Pratchet - Small Gods
# Make sure the length is appropriate
if (len(data) % 512) == 0:
data += "!"
with temporaryPath(perms=0o666, data=data) as srcPath:
with temporaryPath(perms=0o666) as dstPath:
# Copy
rc, out, err = misc.ddWatchCopy(srcPath, dstPath,
None, len(data))
# Get copied data
with open(dstPath) as dp:
readData = dp.read()
# Compare
self.assertEqual(readData, data)
def testCopy(self):
"""
Test that regular copying works.
"""
# Prepare source
data = "Everything starts somewhere, " + \
"though many physicists disagree." + \
"But people have always been dimly aware of the " + \
"problem with the start of things." + \
"They wonder how the snowplough driver gets to work, or " + \
"how the makers of dictionaries look up the spelling of words."
# (C) Terry Pratchet - Small Gods
# Makes sure we round up to a complete block size
data *= 512
with temporaryPath(perms=0o666, data=data) as srcPath:
with temporaryPath(perms=0o666) as dstPath:
# Copy
rc, out, err = misc.ddWatchCopy(srcPath, dstPath,
None, len(data))
# Get copied data
with open(dstPath) as f:
readData = f.read()
# Comapre
self.assertEquals(readData, data)
def testNonAlignedAppend(self):
data = "ABCD" * 256 # 1Kb
add_data = "E"
repetitions = misc.MEGA / len(data)
path = self._createDataFile(data, repetitions)
try:
with open(path, "a") as f: # Appending additional data
f.write(add_data)
self.assertEquals(os.stat(path).st_size, misc.MEGA + len(add_data))
# Using os.stat(path).st_size is part of the test, please do not
# remove or change.
rc, out, err = misc.ddWatchCopy(
"/dev/zero", path, None, misc.MEGA, os.stat(path).st_size)
self.assertEquals(rc, 0)
self.assertEquals(os.stat(path).st_size,
misc.MEGA * 2 + len(add_data))
with open(path, "r") as f:
for i in range(repetitions):
self.assertEquals(f.read(len(data)), data)
# Checking the additional data
self.assertEquals(f.read(len(add_data)), add_data)
finally:
os.unlink(path)
def testNonAlignedCopy(self, sudo=False):
"""
Test that copying a file with odd length works.
"""
data = '- "What\'re quantum mechanics?"' + \
'- "I don\'t know. People who repair quantums, I suppose."'
# (C) Terry Pratchet - Small Gods
# Make sure the length is appropriate
if (len(data) % 512) == 0:
data += "!"
with temporaryPath(perms=0o666, data=data) as srcPath:
with temporaryPath(perms=0o666) as dstPath:
# Copy
rc, out, err = misc.ddWatchCopy(srcPath, dstPath,
None, len(data))
# Get copied data
with open(dstPath) as dp:
readData = dp.read()
# Compare
self.assertEquals(readData, data)
def _extendSizeRaw(self, newSize):
volPath = self.getVolumePath()
curSizeBytes = self.oop.os.stat(volPath).st_size
newSizeBytes = newSize * BLOCK_SIZE
# No real sanity checks here, they should be included in the calling
# function/method. We just validate the sizes to be consistent since
# they're computed and used in the pre-allocated case.
if newSizeBytes == curSizeBytes:
return # Nothing to do
elif curSizeBytes <= 0:
raise se.StorageException("Volume size is impossible: %s" %
curSizeBytes)
elif newSizeBytes < curSizeBytes:
raise se.VolumeResizeValueError(newSize)
if self.getType() == sc.PREALLOCATED_VOL:
# for pre-allocated we need to zero to the file size
misc.ddWatchCopy("/dev/zero", volPath, vars.task.aborting,
newSizeBytes - curSizeBytes, curSizeBytes)
else:
# for sparse files we can just truncate to the correct size
self.oop.truncateFile(volPath, newSizeBytes)
def _extendSizeRaw(self, newSize):
volPath = self.getVolumePath()
curSizeBytes = self.oop.os.stat(volPath).st_size
newSizeBytes = newSize * BLOCK_SIZE
# No real sanity checks here, they should be included in the calling
# function/method. We just validate the sizes to be consistent since
# they're computed and used in the pre-allocated case.
if newSizeBytes == curSizeBytes:
return # Nothing to do
elif curSizeBytes <= 0:
raise se.StorageException(
"Volume size is impossible: %s" % curSizeBytes)
elif newSizeBytes < curSizeBytes:
raise se.VolumeResizeValueError(newSize)
if self.getType() == sc.PREALLOCATED_VOL:
# for pre-allocated we need to zero to the file size
misc.ddWatchCopy("/dev/zero", volPath, vars.task.aborting,
newSizeBytes - curSizeBytes, curSizeBytes)
else:
# for sparse files we can just truncate to the correct size
self.oop.truncateFile(volPath, newSizeBytes)
def testAlignedAppend(self):
data = "ABCD" * 256 # 1Kb
repetitions = misc.MEGA / len(data)
path = self._createDataFile(data, repetitions)
try:
# Using os.stat(path).st_size is part of the test, please do not
# remove or change.
rc, out, err = misc.ddWatchCopy(
"/dev/zero", path, None, misc.MEGA, os.stat(path).st_size)
self.assertEqual(rc, 0)
self.assertEqual(os.stat(path).st_size, misc.MEGA * 2)
with open(path, "r") as f:
for i in range(repetitions):
self.assertEqual(f.read(len(data)), data)
finally:
os.unlink(path)
def _create(cls,
dom,
imgUUID,
volUUID,
size,
volFormat,
preallocate,
volParent,
srcImgUUID,
srcVolUUID,
volPath,
initialSize=None):
"""
Class specific implementation of volumeCreate. All the exceptions are
properly handled and logged in volume.create()
"""
if initialSize:
cls.log.error("initialSize is not supported for file-based "
"volumes")
raise se.InvalidParameterException("initial size", initialSize)
sizeBytes = size * BLOCK_SIZE
truncSize = sizeBytes if volFormat == sc.RAW_FORMAT else 0
try:
oop.getProcessPool(dom.sdUUID).truncateFile(
volPath,
truncSize,
mode=sc.FILE_VOLUME_PERMISSIONS,
creatExcl=True)
except OSError as e:
if e.errno == errno.EEXIST:
raise se.VolumeAlreadyExists(volUUID)
raise
if preallocate == sc.PREALLOCATED_VOL:
try:
# ddWatchCopy expects size to be in bytes
misc.ddWatchCopy("/dev/zero", volPath, vars.task.aborting,
sizeBytes)
except exception.ActionStopped:
raise
except Exception:
cls.log.error("Unexpected error", exc_info=True)
raise se.VolumesZeroingError(volPath)
if not volParent:
cls.log.info(
"Request to create %s volume %s with size = %s "
"sectors", sc.type2name(volFormat), volPath, size)
if volFormat == sc.COW_FORMAT:
qemuimg.create(volPath, sizeBytes, sc.fmt2str(volFormat))
else:
# Create hardlink to template and its meta file
cls.log.info("Request to create snapshot %s/%s of volume %s/%s",
imgUUID, volUUID, srcImgUUID, srcVolUUID)
volParent.clone(volPath, volFormat)
# Forcing the volume permissions in case one of the tools we use
# (dd, qemu-img, etc.) will mistakenly change the file permissiosn.
dom.oop.os.chmod(volPath, sc.FILE_VOLUME_PERMISSIONS)
return (volPath, )
def delete(self, postZero, force):
""" Delete volume
'postZero' - zeroing file before deletion
'force' is required to remove shared and internal volumes
"""
self.log.info("Request to delete LV %s of image %s in VG %s ",
self.volUUID, self.imgUUID, self.sdUUID)
vol_path = self.getVolumePath()
size = self.getVolumeSize(bs=1)
offs = self.getMetaOffset()
# On block storage domains we store a volume's parent UUID in two
# places: 1) in the domain's metadata LV, and 2) in a LV tag attached
# to the volume LV itself. The LV tag is more efficient to access
# than the domain metadata but it may only be updated by the SPM.
#
# This means that after a live merge completes the domain metadata LV
# will be updated but the LV tag will not. We can detect this case
# here and fix the LV tag since this is an SPM verb.
#
# File domains do not have this complexity because the metadata is
# stored in only one place and that metadata is updated by the HSM
# host when the live merge finishes.
sync = False
for childID in self.getChildren():
child = BlockVolume(self.repoPath, self.sdUUID, self.imgUUID,
childID)
metaParent = child.getParentMeta()
tagParent = child.getParentTag()
if metaParent != tagParent:
self.log.debug(
"Updating stale PUUID LV tag from %s to %s for "
"volume %s", tagParent, metaParent, child.volUUID)
child.setParentTag(metaParent)
sync = True
if sync:
self.recheckIfLeaf()
if not force:
self.validateDelete()
# Mark volume as illegal before deleting
self.setLegality(sc.ILLEGAL_VOL)
discard = config.getboolean('irs', 'discard_enable')
if postZero or discard:
self.prepare(justme=True,
rw=True,
chainrw=force,
setrw=True,
force=True)
try:
if postZero:
try:
misc.ddWatchCopy("/dev/zero", vol_path,
vars.task.aborting, int(size))
except exception.ActionStopped:
raise
except Exception:
self.log.error("Unexpected error", exc_info=True)
raise se.VolumesZeroingError(vol_path)
if discard:
try:
blkdiscard.blkdiscard(vol_path)
except cmdutils.Error as e:
log.warning('Discarding %s failed: %s', vol_path, e)
finally:
self.teardown(self.sdUUID, self.volUUID, justme=True)
# try to cleanup as much as possible
eFound = se.CannotDeleteVolume(self.volUUID)
puuid = None
try:
# We need to blank parent record in our metadata
# for parent to become leaf successfully.
puuid = self.getParent()
self.setParent(sc.BLANK_UUID)
if puuid and puuid != sc.BLANK_UUID:
pvol = BlockVolume(self.repoPath, self.sdUUID, self.imgUUID,
puuid)
pvol.recheckIfLeaf()
except Exception as e:
eFound = e
self.log.warning("cannot finalize parent volume %s",
puuid,
exc_info=True)
try:
try:
lvm.removeLVs(self.sdUUID, self.volUUID)
except se.CannotRemoveLogicalVolume:
# At this point LV is already marked as illegal, we will
# try to cleanup whatever we can...
pass
self.removeMetadata([self.sdUUID, offs])
except Exception as e:
eFound = e
self.log.error("cannot remove volume %s/%s",
self.sdUUID,
self.volUUID,
exc_info=True)
try:
self.log.debug("Unlinking %s", vol_path)
os.unlink(vol_path)
return True
except Exception as e:
eFound = e
self.log.error("cannot delete volume's %s/%s link path: %s",
self.sdUUID,
self.volUUID,
vol_path,
exc_info=True)
raise eFound
def delete(self, postZero, force):
""" Delete volume
'postZero' - zeroing file before deletion
'force' is required to remove shared and internal volumes
"""
self.log.info("Request to delete LV %s of image %s in VG %s ",
self.volUUID, self.imgUUID, self.sdUUID)
vol_path = self.getVolumePath()
size = self.getVolumeSize(bs=1)
offs = self.getMetaOffset()
# On block storage domains we store a volume's parent UUID in two
# places: 1) in the domain's metadata LV, and 2) in a LV tag attached
# to the volume LV itself. The LV tag is more efficient to access
# than the domain metadata but it may only be updated by the SPM.
#
# This means that after a live merge completes the domain metadata LV
# will be updated but the LV tag will not. We can detect this case
# here and fix the LV tag since this is an SPM verb.
#
# File domains do not have this complexity because the metadata is
# stored in only one place and that metadata is updated by the HSM
# host when the live merge finishes.
sync = False
for childID in self.getChildren():
child = BlockVolume(self.repoPath, self.sdUUID, self.imgUUID,
childID)
metaParent = child.getParentMeta()
tagParent = child.getParentTag()
if metaParent != tagParent:
self.log.debug("Updating stale PUUID LV tag from %s to %s for "
"volume %s", tagParent, metaParent,
child.volUUID)
child.setParentTag(metaParent)
sync = True
if sync:
self.recheckIfLeaf()
if not force:
self.validateDelete()
# Mark volume as illegal before deleting
self.setLegality(sc.ILLEGAL_VOL)
discard = config.getboolean('irs', 'discard_enable')
if postZero or discard:
self.prepare(justme=True, rw=True, chainrw=force, setrw=True,
force=True)
try:
if postZero:
try:
misc.ddWatchCopy("/dev/zero", vol_path,
vars.task.aborting, int(size))
except exception.ActionStopped:
raise
except Exception:
self.log.error("Unexpected error", exc_info=True)
raise se.VolumesZeroingError(vol_path)
if discard:
try:
blkdiscard.blkdiscard(vol_path)
except cmdutils.Error as e:
log.warning('Discarding %s failed: %s', vol_path, e)
finally:
self.teardown(self.sdUUID, self.volUUID, justme=True)
# try to cleanup as much as possible
eFound = se.CannotDeleteVolume(self.volUUID)
puuid = None
try:
# We need to blank parent record in our metadata
# for parent to become leaf successfully.
puuid = self.getParent()
self.setParent(sc.BLANK_UUID)
if puuid and puuid != sc.BLANK_UUID:
pvol = BlockVolume(self.repoPath, self.sdUUID, self.imgUUID,
puuid)
pvol.recheckIfLeaf()
except Exception as e:
eFound = e
self.log.warning("cannot finalize parent volume %s", puuid,
exc_info=True)
try:
try:
lvm.removeLVs(self.sdUUID, self.volUUID)
except se.CannotRemoveLogicalVolume:
# At this point LV is already marked as illegal, we will
# try to cleanup whatever we can...
pass
self.removeMetadata([self.sdUUID, offs])
except Exception as e:
eFound = e
self.log.error("cannot remove volume %s/%s", self.sdUUID,
self.volUUID, exc_info=True)
try:
self.log.debug("Unlinking %s", vol_path)
os.unlink(vol_path)
return True
except Exception as e:
eFound = e
self.log.error("cannot delete volume's %s/%s link path: %s",
self.sdUUID, self.volUUID, vol_path, exc_info=True)
raise eFound
