def test_writable_cow_multiline_table(self):
# Let's do some silly math
size = directio.size(self._ramdisk)
megs = size / 1024 / 1024
megs = megs - megs % 4
# 12 megs for a volume, 4 for lvm itself
alloc = megs - 12 - 4
vg = self.conf.string('volume', 'volume_group', None)
# Reserve a 4m hole at the front, and 8m at the end
execute('lvcreate', vg, size='4m', name='tmpvol')
execute('lvcreate', vg, size='%sm' % alloc, name='wasted')
execute('lvremove', '%s/tmpvol' % vg, force=None)
foo = execute('pvs', self._ramdisk)
foo = execute('vgs', vg)
foo = execute('lvs', vg)
volume_id = str(uuid4())
self.volume.create(volume_id)
volume = self.volume.get(volume_id)
execute('lvremove', '%s/wasted' % vg, force=None)
dmname = '%s-%s' % (re.sub('-', '--', vg), re.sub(
'-', '--', volume_id))
foo = execute('dmsetup', 'table', dmname)
self.assert_('\n' in foo)
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
scrub = Scrub(LunrConfig())
(cow_name, cow_path) = scrub.get_writable_cow(snapshot, volume)
execute('dmsetup', 'remove', cow_name)
self.assertTrue(True)
def test_writable_cow_multiline_table(self):
# Let's do some silly math
size = directio.size(self._ramdisk)
megs = size / 1024 / 1024
megs = megs - megs % 4
# 12 megs for a volume, 4 for lvm itself
alloc = megs - 12 - 4
vg = self.conf.string('volume', 'volume_group', None)
# Reserve a 4m hole at the front, and 8m at the end
execute('lvcreate', vg, size='4m', name='tmpvol')
execute('lvcreate', vg, size='%sm' % alloc, name='wasted')
execute('lvremove', '%s/tmpvol' % vg, force=None)
foo = execute('pvs', self._ramdisk)
foo = execute('vgs', vg)
foo = execute('lvs', vg)
volume_id = str(uuid4())
self.volume.create(volume_id)
volume = self.volume.get(volume_id)
execute('lvremove', '%s/wasted' % vg, force=None)
dmname = '%s-%s' % (re.sub('-', '--', vg),
re.sub('-', '--', volume_id))
foo = execute('dmsetup', 'table', dmname)
self.assert_('\n' in foo)
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
scrub = Scrub(LunrConfig())
(cow_name, cow_path) = scrub.get_writable_cow(snapshot, volume)
execute('dmsetup', 'remove', cow_name)
self.assertTrue(True)
def test_overflow_snapshot(self):
# Better to use conf, but helper is already created.
self.volume.max_snapshot_bytes = 4 * 1024 * 1024
volume_id = str(uuid4())
self.volume.create(volume_id)
volume = self.volume.get(volume_id)
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
def callback():
self.volume.delete(snapshot['id'])
self.fail("didnt get the proper error callback")
def error_callback():
self.volume.delete(snapshot['id'])
error_callback.ran = True
error_callback.ran = False
# Overflow the snapshot! Only reserved 4m
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
block_size = 32768
for i in xrange(0, size / block_size):
file.write('ZERG' * (block_size / 4))
with open(snapshot['path']) as file:
self.assertRaises(IOError, file.read, block_size)
self.backup.create(snapshot,
backup_id,
callback=callback,
error_callback=error_callback,
lock=MockResourceLock())
self.assertTrue(error_callback.ran)
# Make sure scrubbing still happened correctly.
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
block_size = 32768
for i in xrange(0, size / block_size):
file.write('\0' * block_size)
# Read full disk for hidden zergs.
with directio.open(self._ramdisk) as file:
size = directio.size(self._ramdisk)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.fail("Found zergs on disk: %s" % self._ramdisk)
def test_restore_backup(self):
# Create a Volume
volume_id = str(uuid4())
self.volume.create(volume_id)
# Write ZERG to the volume
volume = self.volume.get(volume_id)
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
block_size = 32768
for i in xrange(0, size / block_size):
file.write('ZERG' * (block_size / 4))
# Create a snap-shot with a timestamp of 123456
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
def callback():
# Delete the snapshot after completion
self.volume.delete(snapshot['id'])
# Create the backup
self.backup.create(snapshot,
backup_id,
callback=callback,
lock=MockResourceLock())
# Deleting the origin also removes the snapshot
self.volume.remove(self.volume.get(volume_id)['path'])
# Create a Restore Volume
restore_volume_id = str(uuid4())
self.volume.create(restore_volume_id,
backup_source_volume_id=volume_id,
backup_id=backup_id,
lock=MockResourceLock())
volume = self.volume.get(restore_volume_id)
# Read the restored volume, it should contain ZERGS
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' not in block:
self.fail("zergs missing on disk: %s" % volume['path'])
def wipe_device(cls, device):
size = directio.size(device)
# Create a 32k block
block = '\0' * 32768
with directio.open(device) as file:
# Divide the size into 32k chunks
for i in xrange(0, size / 32768):
# Write a block of nulls
file.write(block)
def test_overflow_snapshot(self):
# Better to use conf, but helper is already created.
self.volume.max_snapshot_bytes = 4 * 1024 * 1024
volume_id = str(uuid4())
self.volume.create(volume_id)
volume = self.volume.get(volume_id)
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
def callback():
self.volume.delete(snapshot['id'])
self.fail("didnt get the proper error callback")
def error_callback():
self.volume.delete(snapshot['id'])
error_callback.ran = True
error_callback.ran = False
# Overflow the snapshot! Only reserved 4m
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
block_size = 32768
for i in xrange(0, size / block_size):
file.write('ZERG' * (block_size / 4))
with open(snapshot['path']) as file:
self.assertRaises(IOError, file.read, block_size)
self.backup.create(
snapshot, backup_id, callback=callback,
error_callback=error_callback, lock=MockResourceLock())
self.assertTrue(error_callback.ran)
# Make sure scrubbing still happened correctly.
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
block_size = 32768
for i in xrange(0, size / block_size):
file.write('\0' * block_size)
# Read full disk for hidden zergs.
with directio.open(self._ramdisk) as file:
size = directio.size(self._ramdisk)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.fail("Found zergs on disk: %s" % self._ramdisk)
def test_restore_backup(self):
# Create a Volume
volume_id = str(uuid4())
self.volume.create(volume_id)
# Write ZERG to the volume
volume = self.volume.get(volume_id)
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
block_size = 32768
for i in xrange(0, size / block_size):
file.write('ZERG' * (block_size / 4))
# Create a snap-shot with a timestamp of 123456
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
def callback():
# Delete the snapshot after completion
self.volume.delete(snapshot['id'])
# Create the backup
self.backup.create(snapshot, backup_id,
callback=callback,
lock=MockResourceLock())
# Deleting the origin also removes the snapshot
self.volume.remove(self.volume.get(volume_id)['path'])
# Create a Restore Volume
restore_volume_id = str(uuid4())
self.volume.create(
restore_volume_id, backup_source_volume_id=volume_id,
backup_id=backup_id, lock=MockResourceLock())
volume = self.volume.get(restore_volume_id)
# Read the restored volume, it should contain ZERGS
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' not in block:
self.fail("zergs missing on disk: %s" % volume['path'])
def test_create_clone(self):
block_size = 32768
# Create 2 Volumes
self.volume1 = str(uuid4())
self.volume2 = str(uuid4())
self.volume.create(self.volume1)
self.volume.create(self.volume2)
# Write some stuff to volume1
volume = self.volume.get(self.volume1)
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
# 32768 / 4 = 8192
file.write('ZERG' * (block_size / 4))
# Create an export for volume 2
info = self.export.create(self.volume2)
# Now clone Volume 1 to Volume 2
clone = self.volume.create_clone(self.volume1,
self.volume2,
info['name'],
'127.0.0.1',
3260,
lock=MockResourceLock())
compare_block = 'ZERG' * (block_size / 4)
# Ensure the stuff we wrote to Volume 1 is in Volume 2
volume = self.volume.get(self.volume2)
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
block = file.read(block_size)
self.assertTrue(block == compare_block)
# Remove the export
self.export.delete(self.volume2)
def test_create_clone(self):
block_size = 32768
# Create 2 Volumes
self.volume1 = str(uuid4())
self.volume2 = str(uuid4())
self.volume.create(self.volume1)
self.volume.create(self.volume2)
# Write some stuff to volume1
volume = self.volume.get(self.volume1)
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
# 32768 / 4 = 8192
file.write('ZERG' * (block_size / 4))
# Create an export for volume 2
info = self.export.create(self.volume2)
# Now clone Volume 1 to Volume 2
clone = self.volume.create_clone(self.volume1, self.volume2,
info['name'], '127.0.0.1',
3260, lock=MockResourceLock())
compare_block = 'ZERG' * (block_size / 4)
# Ensure the stuff we wrote to Volume 1 is in Volume 2
volume = self.volume.get(self.volume2)
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
block = file.read(block_size)
self.assertTrue(block == compare_block)
# Remove the export
self.export.delete(self.volume2)
def find_ram_disk(cls, inUse=False):
# Get a list of devices lvm is using
stdout = cls.check_sudo("pvs --noheadings --options=pv_name")
used = [disk.strip() for disk in stdout.splitlines()]
if inUse:
return used
# Find a Ram disk that is not in use by lvm
for index in xrange(0, 15):
disk = "/dev/ram%d" % index
if disk in used:
continue
# if the block device is less than 60MB, skip it
if (directio.size(disk) / 1048576) < 60:
continue
return disk
return None
def test_snapshot_scrub(self):
block_size = 32768
# Create a Volume
volume_id = str(uuid4())
self.volume.create(volume_id)
# Get the volume information
volume = self.volume.get(volume_id)
# Fill the volume with 'ZERG's
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
# 32768 / 4 = 8192
file.write('ZERG' * (block_size / 4))
# Create a snap-shot with a timestamp of 123456
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
# Now that the snapshot is made, simulate users making writes
# to the origin during a normal backup. This should generate
# exceptions in the cow
with directio.open(volume['path']) as file:
# Overwrite all the zergs.
for i in xrange(0, size / block_size):
file.write('A' * block_size)
# Tell scrub we don't want it to remove the cow after scrubbing
scrub = Scrub(LunrConfig())
# Build the cow-zero
(cow_name, cow_path) = scrub.get_writable_cow(snapshot, volume)
with directio.open(cow_path) as file:
size = directio.size(cow_path)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.assert_(True)
break
with directio.open(self._ramdisk) as file:
size = directio.size(self._ramdisk)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.assert_(True)
break
# Scrub the cow of all exceptions
scrub.scrub_cow(cow_path)
scrub.remove_cow(cow_name)
# Remove & scrub the volume. LVM removes snapshot itself.
self.volume.remove_lvm_volume(volume)
# Read full disk for hidden zergs.
with directio.open(self._ramdisk) as file:
size = directio.size(self._ramdisk)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.fail("Found zergs on disk: %s" % self._ramdisk)
def test_snapshot_scrub(self):
block_size = 32768
# Create a Volume
volume_id = str(uuid4())
self.volume.create(volume_id)
# Get the volume information
volume = self.volume.get(volume_id)
# Fill the volume with 'ZERG's
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
# 32768 / 4 = 8192
file.write('ZERG' * (block_size / 4))
# Create a snap-shot with a timestamp of 123456
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
# Now that the snapshot is made, simulate users making writes
# to the origin during a normal backup. This should generate
# exceptions in the cow
with directio.open(volume['path']) as file:
# Overwrite all the zergs.
for i in xrange(0, size / block_size):
file.write('A' * block_size)
# Tell scrub we don't want it to remove the cow after scrubbing
scrub = Scrub(LunrConfig())
# Build the cow-zero
(cow_name, cow_path) = scrub.get_writable_cow(snapshot, volume)
with directio.open(cow_path) as file:
size = directio.size(cow_path)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.assert_(True)
break
with directio.open(self._ramdisk) as file:
size = directio.size(self._ramdisk)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.assert_(True)
break
# Scrub the cow of all exceptions
scrub.scrub_cow(cow_path)
scrub.remove_cow(cow_name)
# Remove & scrub the volume. LVM removes snapshot itself.
self.volume.remove_lvm_volume(volume)
# Read full disk for hidden zergs.
with directio.open(self._ramdisk) as file:
size = directio.size(self._ramdisk)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.fail("Found zergs on disk: %s" % self._ramdisk)
