def _create_random_sparse_file(file_obj, size):
"""
Create a sparse file with randomly distributed holes. The mapped areas are
filled with semi-random data. Returns a tuple containing 2 lists:
1. a list of mapped block ranges, same as 'Filemap.get_mapped_ranges()'
2. a list of unmapped block ranges (holes), same as
'Filemap.get_unmapped_ranges()'
"""
file_obj.truncate(0)
block_size = BmapHelpers.get_block_size(file_obj)
blocks_cnt = (size + block_size - 1) / block_size
def process_block(block):
"""
This is a helper function which processes a block. It randomly decides
whether the block should be filled with random data or should become a
hole. Returns 'True' if the block was mapped and 'False' otherwise.
"""
map_the_block = random.getrandbits(1)
if map_the_block:
# Randomly select how much we are going to write
seek = random.randint(0, block_size - 1)
write = random.randint(1, block_size - seek)
assert seek + write <= block_size
file_obj.seek(block * block_size + seek)
file_obj.write(chr(random.getrandbits(8)) * write)
else:
file_obj.truncate(block * block_size)
return map_the_block
mapped = []
unmapped = []
iterator = xrange(0, blocks_cnt)
for was_mapped, group in itertools.groupby(iterator, process_block):
# Start of a mapped region or a hole. Find the last element in the
# group.
first = group.next()
last = first
for last in group:
pass
if was_mapped:
mapped.append((first, last))
else:
unmapped.append((first, last))
file_obj.truncate(size)
file_obj.flush()
return (mapped, unmapped)
def _create_random_sparse_file(file_obj, size):
"""
Create a sparse file with randomly distributed holes. The mapped areas are
filled with semi-random data. Returns a tuple containing 2 lists:
1. a list of mapped block ranges, same as 'Filemap.get_mapped_ranges()'
2. a list of unmapped block ranges (holes), same as
'Filemap.get_unmapped_ranges()'
"""
file_obj.truncate(size)
block_size = BmapHelpers.get_block_size(file_obj)
blocks_cnt = (size + block_size - 1) / block_size
def process_block(block):
"""
This is a helper function which processes a block. It randomly decides
whether the block should be filled with random data or should become a
hole. Returns 'True' if the block was mapped and 'False' otherwise.
"""
map_the_block = random.getrandbits(1)
if map_the_block:
# Randomly select how much we are going to write
seek = random.randint(0, block_size - 1)
write = random.randint(1, block_size - seek)
assert seek + write <= block_size
file_obj.seek(block * block_size + seek)
file_obj.write(chr(random.getrandbits(8)) * write)
return map_the_block
mapped = []
unmapped = []
iterator = xrange(0, blocks_cnt)
for was_mapped, group in itertools.groupby(iterator, process_block):
# Start of a mapped region or a hole. Find the last element in the
# group.
first = group.next()
last = first
for last in group:
pass
if was_mapped:
mapped.append((first, last))
else:
unmapped.append((first, last))
file_obj.truncate(size)
file_obj.flush()
return (mapped, unmapped)
def __init__(self, image):
"""
Initialize a class instance. The 'image' argument is full path to the
file or file object to operate on.
"""
self._f_image_needs_close = False
if hasattr(image, "fileno"):
self._f_image = image
self._image_path = image.name
else:
self._image_path = image
self._open_image_file()
try:
self.image_size = os.fstat(self._f_image.fileno()).st_size
except IOError as err:
raise Error("cannot get information about file '%s': %s" %
(self._f_image.name, err))
try:
self.block_size = BmapHelpers.get_block_size(self._f_image)
except IOError as err:
raise Error("cannot get block size for '%s': %s" %
(self._image_path, err))
self.blocks_cnt = (self.image_size + self.block_size -
1) // self.block_size
try:
self._f_image.flush()
except IOError as err:
raise Error("cannot flush image file '%s': %s" %
(self._image_path, err))
try:
os.fsync(self._f_image.fileno()),
except OSError as err:
raise Error("cannot synchronize image file '%s': %s " %
(self._image_path, err.strerror))
if not BmapHelpers.is_compatible_file_system(self._image_path):
fstype = BmapHelpers.get_file_system_type(self._image_path)
raise Error(
"image file on incompatible file system '%s': '%s': see docs for fix"
% (self._image_path, fstype))
_log.debug("opened image \"%s\"" % self._image_path)
_log.debug("block size %d, blocks count %d, image size %d" %
(self.block_size, self.blocks_cnt, self.image_size))
def __init__(self, image, log=None):
"""
Initialize a class instance. The 'image' argument is full path to the
file or file object to operate on.
"""
self._log = log
if self._log is None:
self._log = logging.getLogger(__name__)
self._f_image_needs_close = False
if hasattr(image, "fileno"):
self._f_image = image
self._image_path = image.name
else:
self._image_path = image
self._open_image_file()
try:
self.image_size = os.fstat(self._f_image.fileno()).st_size
except IOError as err:
raise Error("cannot get information about file '%s': %s" %
(self._f_image.name, err))
try:
self.block_size = BmapHelpers.get_block_size(self._f_image)
except IOError as err:
raise Error("cannot get block size for '%s': %s" %
(self._image_path, err))
self.blocks_cnt = self.image_size + self.block_size - 1
self.blocks_cnt /= self.block_size
try:
self._f_image.flush()
except IOError as err:
raise Error("cannot flush image file '%s': %s" %
(self._image_path, err))
try:
os.fsync(self._f_image.fileno()),
except OSError as err:
raise Error("cannot synchronize image file '%s': %s " %
(self._image_path, err.strerror))
self._log.debug("opened image \"%s\"" % self._image_path)
self._log.debug("block size %d, blocks count %d, image size %d" %
(self.block_size, self.blocks_cnt, self.image_size))
def __init__(self, image):
"""
Initialize a class instance. The 'image' argument is full path to the
file or file object to operate on.
"""
self._f_image_needs_close = False
if hasattr(image, "fileno"):
self._f_image = image
self._image_path = image.name
else:
self._image_path = image
self._open_image_file()
try:
self.image_size = os.fstat(self._f_image.fileno()).st_size
except IOError as err:
raise Error("cannot get information about file '%s': %s"
% (self._f_image.name, err))
try:
self.block_size = BmapHelpers.get_block_size(self._f_image)
except IOError as err:
raise Error("cannot get block size for '%s': %s"
% (self._image_path, err))
self.blocks_cnt = self.image_size + self.block_size - 1
self.blocks_cnt /= self.block_size
try:
self._f_image.flush()
except IOError as err:
raise Error("cannot flush image file '%s': %s"
% (self._image_path, err))
try:
os.fsync(self._f_image.fileno()),
except OSError as err:
raise Error("cannot synchronize image file '%s': %s "
% (self._image_path, err.strerror))
_log.debug("opened image \"%s\"" % self._image_path)
_log.debug("block size %d, blocks count %d, image size %d"
% (self.block_size, self.blocks_cnt, self.image_size))
def generate_test_files(max_size=4*1024*1024, directory=None, delete=True):
"""
This is a generator which yields files which other tests use as the input
for the testing. The generator tries to yield "interesting" files which
cover various corner-cases. For example, a large hole file, a file with
no holes, files of unaligned length, etc.
The 'directory' argument specifies the directory path where the yielded
test files should be created. The 'delete' argument specifies whether the
yielded test files have to be automatically deleted.
The generator yields tuples consisting of the following elements:
1. the test file object
2. file size in bytes
3. a list of mapped block ranges, same as 'Filemap.get_mapped_ranges()'
4. a list of unmapped block ranges (holes), same as
'Filemap.get_unmapped_ranges()'
"""
#
# Generate sparse files with one single hole spanning the entire file
#
# A block-sized hole
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Khole_",
delete=delete, dir=directory,
suffix=".img")
block_size = BmapHelpers.get_block_size(file_obj)
file_obj.truncate(block_size)
yield (file_obj, block_size, [], [(0, 0)])
file_obj.close()
# A block size + 1 byte hole
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Khole_plus_1_",
delete=delete, dir=directory,
suffix=".img")
file_obj.truncate(block_size + 1)
yield (file_obj, block_size + 1, [], [(0, 1)])
file_obj.close()
# A block size - 1 byte hole
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Khole_minus_1_",
delete=delete, dir=directory,
suffix=".img")
file_obj.truncate(block_size - 1)
yield (file_obj, block_size - 1, [], [(0, 0)])
file_obj.close()
# A 1-byte hole
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="1byte_hole_",
delete=delete, dir=directory,
suffix=".img")
file_obj.truncate(1)
yield (file_obj, 1, [], [(0, 0)])
file_obj.close()
# And 10 holes of random size
for i in xrange(10):
size = random.randint(1, max_size)
file_obj = tempfile.NamedTemporaryFile("wb+", suffix=".img",
delete=delete, dir=directory,
prefix="rand_hole_%d_"%i)
file_obj.truncate(size)
blocks_cnt = (size + block_size - 1) / block_size
yield (file_obj, size, [], [(0, blocks_cnt - 1)])
file_obj.close()
#
# Generate a random sparse files
#
# The maximum size
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="sparse_",
delete=delete, dir=directory,
suffix=".img")
mapped, unmapped = _create_random_sparse_file(file_obj, max_size)
yield (file_obj, max_size, mapped, unmapped)
file_obj.close()
# The maximum size + 1 byte
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="sparse_plus_1_",
delete=delete, dir=directory,
suffix=".img")
mapped, unmapped = _create_random_sparse_file(file_obj, max_size + 1)
yield (file_obj, max_size + 1, mapped, unmapped)
file_obj.close()
# The maximum size - 1 byte
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="sparse_minus_1_",
delete=delete, dir=directory,
suffix=".img")
mapped, unmapped = _create_random_sparse_file(file_obj, max_size - 1)
yield (file_obj, max_size - 1, mapped, unmapped)
file_obj.close()
# And 10 files of random size
for i in xrange(10):
size = random.randint(1, max_size)
file_obj = tempfile.NamedTemporaryFile("wb+", suffix=".img",
delete=delete, dir=directory,
prefix="sparse_%d_"%i)
mapped, unmapped = _create_random_sparse_file(file_obj, size)
yield (file_obj, size, mapped, unmapped)
file_obj.close()
#
# Generate random fully-mapped files
#
# A block-sized file
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Kmapped_",
delete=delete, dir=directory,
suffix=".img")
_create_random_file(file_obj, block_size)
yield (file_obj, block_size, [(0, 0)], [])
file_obj.close()
# A block size + 1 byte file
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Kmapped_plus_1_",
delete=delete, dir=directory,
suffix=".img")
_create_random_file(file_obj, block_size + 1)
yield (file_obj, block_size + 1, [(0, 1)], [])
file_obj.close()
# A block size - 1 byte file
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Kmapped_minus_1_",
delete=delete, dir=directory,
suffix=".img")
_create_random_file(file_obj, block_size - 1)
yield (file_obj, block_size - 1, [(0, 0)], [])
file_obj.close()
# A 1-byte file
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="1byte_mapped_",
delete=delete, dir=directory,
suffix=".img")
_create_random_file(file_obj, 1)
yield (file_obj, 1, [(0, 0)], [])
file_obj.close()
# And 10 mapped files of random size
for i in xrange(10):
size = random.randint(1, max_size)
file_obj = tempfile.NamedTemporaryFile("wb+", suffix=".img",
delete=delete, dir=directory,
prefix="rand_mapped_%d_" % i)
_create_random_file(file_obj, size)
blocks_cnt = (size + block_size - 1) / block_size
yield (file_obj, size, [(0, blocks_cnt - 1)], [])
file_obj.close()
def generate_test_files(max_size=4 * 1024 * 1024, directory=None, delete=True):
"""
This is a generator which yields files which other tests use as the input
for the testing. The generator tries to yield "interesting" files which
cover various corner-cases. For example, a large hole file, a file with
no holes, files of unaligned length, etc.
The 'directory' argument specifies the directory path where the yielded
test files should be created. The 'delete' argument specifies whether the
yielded test files have to be automatically deleted.
The generator yields tuples consisting of the following elements:
1. the test file object
2. file size in bytes
3. a list of mapped block ranges, same as 'Filemap.get_mapped_ranges()'
4. a list of unmapped block ranges (holes), same as
'Filemap.get_unmapped_ranges()'
"""
#
# Generate sparse files with one single hole spanning the entire file
#
# A block-sized hole
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Khole_",
delete=delete, dir=directory,
suffix=".img")
block_size = BmapHelpers.get_block_size(file_obj)
file_obj.truncate(block_size)
yield (file_obj, block_size, [], [(0, 0)])
file_obj.close()
# A block size + 1 byte hole
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Khole_plus_1_",
delete=delete, dir=directory,
suffix=".img")
file_obj.truncate(block_size + 1)
yield (file_obj, block_size + 1, [], [(0, 1)])
file_obj.close()
# A block size - 1 byte hole
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Khole_minus_1_",
delete=delete, dir=directory,
suffix=".img")
file_obj.truncate(block_size - 1)
yield (file_obj, block_size - 1, [], [(0, 0)])
file_obj.close()
# A 1-byte hole
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="1byte_hole_",
delete=delete, dir=directory,
suffix=".img")
file_obj.truncate(1)
yield (file_obj, 1, [], [(0, 0)])
file_obj.close()
# And 10 holes of random size
for i in xrange(10):
size = random.randint(1, max_size)
file_obj = tempfile.NamedTemporaryFile("wb+", suffix=".img",
delete=delete, dir=directory,
prefix="rand_hole_%d_" % i)
file_obj.truncate(size)
blocks_cnt = (size + block_size - 1) / block_size
yield (file_obj, size, [], [(0, blocks_cnt - 1)])
file_obj.close()
#
# Generate a random sparse files
#
# The maximum size
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="sparse_",
delete=delete, dir=directory,
suffix=".img")
mapped, unmapped = _create_random_sparse_file(file_obj, max_size)
yield (file_obj, max_size, mapped, unmapped)
file_obj.close()
# The maximum size + 1 byte
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="sparse_plus_1_",
delete=delete, dir=directory,
suffix=".img")
mapped, unmapped = _create_random_sparse_file(file_obj, max_size + 1)
yield (file_obj, max_size + 1, mapped, unmapped)
file_obj.close()
# The maximum size - 1 byte
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="sparse_minus_1_",
delete=delete, dir=directory,
suffix=".img")
mapped, unmapped = _create_random_sparse_file(file_obj, max_size - 1)
yield (file_obj, max_size - 1, mapped, unmapped)
file_obj.close()
# And 10 files of random size
for i in xrange(10):
size = random.randint(1, max_size)
file_obj = tempfile.NamedTemporaryFile("wb+", suffix=".img",
delete=delete, dir=directory,
prefix="sparse_%d_" % i)
mapped, unmapped = _create_random_sparse_file(file_obj, size)
yield (file_obj, size, mapped, unmapped)
file_obj.close()
#
# Generate random fully-mapped files
#
# A block-sized file
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Kmapped_",
delete=delete, dir=directory,
suffix=".img")
_create_random_file(file_obj, block_size)
yield (file_obj, block_size, [(0, 0)], [])
file_obj.close()
# A block size + 1 byte file
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Kmapped_plus_1_",
delete=delete, dir=directory,
suffix=".img")
_create_random_file(file_obj, block_size + 1)
yield (file_obj, block_size + 1, [(0, 1)], [])
file_obj.close()
# A block size - 1 byte file
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="4Kmapped_minus_1_",
delete=delete, dir=directory,
suffix=".img")
_create_random_file(file_obj, block_size - 1)
yield (file_obj, block_size - 1, [(0, 0)], [])
file_obj.close()
# A 1-byte file
file_obj = tempfile.NamedTemporaryFile("wb+", prefix="1byte_mapped_",
delete=delete, dir=directory,
suffix=".img")
_create_random_file(file_obj, 1)
yield (file_obj, 1, [(0, 0)], [])
file_obj.close()
# And 10 mapped files of random size
for i in xrange(10):
size = random.randint(1, max_size)
file_obj = tempfile.NamedTemporaryFile("wb+", suffix=".img",
delete=delete, dir=directory,
prefix="rand_mapped_%d_" % i)
_create_random_file(file_obj, size)
blocks_cnt = (size + block_size - 1) / block_size
yield (file_obj, size, [(0, blocks_cnt - 1)], [])
file_obj.close()