def scrub_cow(self, cow_path):
try:
log.info("Opening Cow '%s'" % cow_path)
# Open the cow block device
fd = directio.open(cow_path, mode='+o', buffered=32768)
except OSError, e:
raise ScrubError("Failed to open cow '%s'" % e)
def scrub_cow(self, cow_path):
try:
log.info("Opening Cow '%s'" % cow_path)
# Open the cow block device
fd = directio.open(cow_path, mode='+o', buffered=32768)
except OSError, e:
raise ScrubError("Failed to open cow '%s'" % e)
def test_overflow_snapshot(self):
# Better to use conf, but helper is already created.
self.volume.max_snapshot_bytes = 4 * 1024 * 1024
volume_id = str(uuid4())
self.volume.create(volume_id)
volume = self.volume.get(volume_id)
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
def callback():
self.volume.delete(snapshot['id'])
self.fail("didnt get the proper error callback")
def error_callback():
self.volume.delete(snapshot['id'])
error_callback.ran = True
error_callback.ran = False
# Overflow the snapshot! Only reserved 4m
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
block_size = 32768
for i in xrange(0, size / block_size):
file.write('ZERG' * (block_size / 4))
with open(snapshot['path']) as file:
self.assertRaises(IOError, file.read, block_size)
self.backup.create(snapshot,
backup_id,
callback=callback,
error_callback=error_callback,
lock=MockResourceLock())
self.assertTrue(error_callback.ran)
# Make sure scrubbing still happened correctly.
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
block_size = 32768
for i in xrange(0, size / block_size):
file.write('\0' * block_size)
# Read full disk for hidden zergs.
with directio.open(self._ramdisk) as file:
size = directio.size(self._ramdisk)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.fail("Found zergs on disk: %s" % self._ramdisk)
def test_restore_backup(self):
# Create a Volume
volume_id = str(uuid4())
self.volume.create(volume_id)
# Write ZERG to the volume
volume = self.volume.get(volume_id)
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
block_size = 32768
for i in xrange(0, size / block_size):
file.write('ZERG' * (block_size / 4))
# Create a snap-shot with a timestamp of 123456
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
def callback():
# Delete the snapshot after completion
self.volume.delete(snapshot['id'])
# Create the backup
self.backup.create(snapshot,
backup_id,
callback=callback,
lock=MockResourceLock())
# Deleting the origin also removes the snapshot
self.volume.remove(self.volume.get(volume_id)['path'])
# Create a Restore Volume
restore_volume_id = str(uuid4())
self.volume.create(restore_volume_id,
backup_source_volume_id=volume_id,
backup_id=backup_id,
lock=MockResourceLock())
volume = self.volume.get(restore_volume_id)
# Read the restored volume, it should contain ZERGS
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' not in block:
self.fail("zergs missing on disk: %s" % volume['path'])
def wipe_device(cls, device):
size = directio.size(device)
# Create a 32k block
block = '\0' * 32768
with directio.open(device) as file:
# Divide the size into 32k chunks
for i in xrange(0, size / 32768):
# Write a block of nulls
file.write(block)
def md5sum(cls, file):
hasher = hashlib.md5()
with directio.open(file) as f:
while True:
block = f.read(32768)
if len(block) == 0:
break
hasher.update(block)
return hasher.hexdigest()
def test_overflow_snapshot(self):
# Better to use conf, but helper is already created.
self.volume.max_snapshot_bytes = 4 * 1024 * 1024
volume_id = str(uuid4())
self.volume.create(volume_id)
volume = self.volume.get(volume_id)
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
def callback():
self.volume.delete(snapshot['id'])
self.fail("didnt get the proper error callback")
def error_callback():
self.volume.delete(snapshot['id'])
error_callback.ran = True
error_callback.ran = False
# Overflow the snapshot! Only reserved 4m
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
block_size = 32768
for i in xrange(0, size / block_size):
file.write('ZERG' * (block_size / 4))
with open(snapshot['path']) as file:
self.assertRaises(IOError, file.read, block_size)
self.backup.create(
snapshot, backup_id, callback=callback,
error_callback=error_callback, lock=MockResourceLock())
self.assertTrue(error_callback.ran)
# Make sure scrubbing still happened correctly.
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
block_size = 32768
for i in xrange(0, size / block_size):
file.write('\0' * block_size)
# Read full disk for hidden zergs.
with directio.open(self._ramdisk) as file:
size = directio.size(self._ramdisk)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.fail("Found zergs on disk: %s" % self._ramdisk)
def test_restore_backup(self):
# Create a Volume
volume_id = str(uuid4())
self.volume.create(volume_id)
# Write ZERG to the volume
volume = self.volume.get(volume_id)
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
block_size = 32768
for i in xrange(0, size / block_size):
file.write('ZERG' * (block_size / 4))
# Create a snap-shot with a timestamp of 123456
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
def callback():
# Delete the snapshot after completion
self.volume.delete(snapshot['id'])
# Create the backup
self.backup.create(snapshot, backup_id,
callback=callback,
lock=MockResourceLock())
# Deleting the origin also removes the snapshot
self.volume.remove(self.volume.get(volume_id)['path'])
# Create a Restore Volume
restore_volume_id = str(uuid4())
self.volume.create(
restore_volume_id, backup_source_volume_id=volume_id,
backup_id=backup_id, lock=MockResourceLock())
volume = self.volume.get(restore_volume_id)
# Read the restored volume, it should contain ZERGS
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' not in block:
self.fail("zergs missing on disk: %s" % volume['path'])
def test_create_clone(self):
block_size = 32768
# Create 2 Volumes
self.volume1 = str(uuid4())
self.volume2 = str(uuid4())
self.volume.create(self.volume1)
self.volume.create(self.volume2)
# Write some stuff to volume1
volume = self.volume.get(self.volume1)
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
# 32768 / 4 = 8192
file.write('ZERG' * (block_size / 4))
# Create an export for volume 2
info = self.export.create(self.volume2)
# Now clone Volume 1 to Volume 2
clone = self.volume.create_clone(self.volume1,
self.volume2,
info['name'],
'127.0.0.1',
3260,
lock=MockResourceLock())
compare_block = 'ZERG' * (block_size / 4)
# Ensure the stuff we wrote to Volume 1 is in Volume 2
volume = self.volume.get(self.volume2)
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
block = file.read(block_size)
self.assertTrue(block == compare_block)
# Remove the export
self.export.delete(self.volume2)
def test_create_clone(self):
block_size = 32768
# Create 2 Volumes
self.volume1 = str(uuid4())
self.volume2 = str(uuid4())
self.volume.create(self.volume1)
self.volume.create(self.volume2)
# Write some stuff to volume1
volume = self.volume.get(self.volume1)
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
# 32768 / 4 = 8192
file.write('ZERG' * (block_size / 4))
# Create an export for volume 2
info = self.export.create(self.volume2)
# Now clone Volume 1 to Volume 2
clone = self.volume.create_clone(self.volume1, self.volume2,
info['name'], '127.0.0.1',
3260, lock=MockResourceLock())
compare_block = 'ZERG' * (block_size / 4)
# Ensure the stuff we wrote to Volume 1 is in Volume 2
volume = self.volume.get(self.volume2)
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
block = file.read(block_size)
self.assertTrue(block == compare_block)
# Remove the export
self.export.delete(self.volume2)
def read(self, device=None, offset=0, bs=None, count=1):
"""
Using DIRECT_O read from the block device specified to stdout
(Without any optional arguments will read the first 4k from the device)
"""
volume = self.get_volume(device)
block_size = bs or BLOCK_SIZE
offset = int(offset) * block_size
count = int(count)
print("Offset: ", offset)
total = 0
with directio.open(volume['path'], buffered=block_size) as file:
file.seek(offset)
for i in range(0, count):
total += os.write(sys.stdout.fileno(), file.read(block_size))
os.write(sys.stdout.fileno(), "\nRead: %d Bytes\n" % total)
def read(self, device=None, offset=0, bs=None, count=1):
"""
Using DIRECT_O read from the block device specified to stdout
(Without any optional arguments will read the first 4k from the device)
"""
volume = self.get_volume(device)
block_size = bs or BLOCK_SIZE
offset = int(offset) * block_size
count = int(count)
print("Offset: ", offset)
total = 0
with directio.open(volume['path'], buffered=block_size) as file:
file.seek(offset)
for i in range(0, count):
total += os.write(sys.stdout.fileno(), file.read(block_size))
os.write(sys.stdout.fileno(), "\nRead: %d Bytes\n" % total)
def write(self, device=None, char=0, bs=None, count=None):
"""
Using DIRECT_O write a character in 4k chunks to a specified block
device (Without any optional arguments will write NULL's to the
entire device)
"""
volume = self.get_volume(device)
block_size = bs or BLOCK_SIZE
# Calculate the number of blocks that are in the volume
count = count or (volume['size'] / block_size)
data = "".join([chr(int(char)) for i in range(0, block_size)])
print("Writing: '%c'" % data[0])
total = 0
with directio.open(volume['path'], buffered=block_size) as file:
for i in range(0, count):
self.dot()
total += file.write(data)
print("\nWrote: ", total)
return 0
def write(self, device=None, char=0, bs=None, count=None):
"""
Using DIRECT_O write a character in 4k chunks to a specified block
device (Without any optional arguments will write NULL's to the
entire device)
"""
volume = self.get_volume(device)
block_size = bs or BLOCK_SIZE
# Calculate the number of blocks that are in the volume
count = count or (volume['size'] / block_size)
data = "".join([chr(int(char)) for i in range(0, block_size)])
print("Writing: '%c'" % data[0])
total = 0
with directio.open(volume['path'], buffered=block_size) as file:
for i in range(0, count):
self.dot()
total += file.write(data)
print("\nWrote: ", total)
return 0
def test_snapshot_scrub(self):
block_size = 32768
# Create a Volume
volume_id = str(uuid4())
self.volume.create(volume_id)
# Get the volume information
volume = self.volume.get(volume_id)
# Fill the volume with 'ZERG's
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
# 32768 / 4 = 8192
file.write('ZERG' * (block_size / 4))
# Create a snap-shot with a timestamp of 123456
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
# Now that the snapshot is made, simulate users making writes
# to the origin during a normal backup. This should generate
# exceptions in the cow
with directio.open(volume['path']) as file:
# Overwrite all the zergs.
for i in xrange(0, size / block_size):
file.write('A' * block_size)
# Tell scrub we don't want it to remove the cow after scrubbing
scrub = Scrub(LunrConfig())
# Build the cow-zero
(cow_name, cow_path) = scrub.get_writable_cow(snapshot, volume)
with directio.open(cow_path) as file:
size = directio.size(cow_path)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.assert_(True)
break
with directio.open(self._ramdisk) as file:
size = directio.size(self._ramdisk)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.assert_(True)
break
# Scrub the cow of all exceptions
scrub.scrub_cow(cow_path)
scrub.remove_cow(cow_name)
# Remove & scrub the volume. LVM removes snapshot itself.
self.volume.remove_lvm_volume(volume)
# Read full disk for hidden zergs.
with directio.open(self._ramdisk) as file:
size = directio.size(self._ramdisk)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.fail("Found zergs on disk: %s" % self._ramdisk)
def test_snapshot_scrub(self):
block_size = 32768
# Create a Volume
volume_id = str(uuid4())
self.volume.create(volume_id)
# Get the volume information
volume = self.volume.get(volume_id)
# Fill the volume with 'ZERG's
with directio.open(volume['path']) as file:
size = directio.size(volume['path'])
for i in xrange(0, size / block_size):
# 32768 / 4 = 8192
file.write('ZERG' * (block_size / 4))
# Create a snap-shot with a timestamp of 123456
backup_id = str(uuid4())
snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
# Now that the snapshot is made, simulate users making writes
# to the origin during a normal backup. This should generate
# exceptions in the cow
with directio.open(volume['path']) as file:
# Overwrite all the zergs.
for i in xrange(0, size / block_size):
file.write('A' * block_size)
# Tell scrub we don't want it to remove the cow after scrubbing
scrub = Scrub(LunrConfig())
# Build the cow-zero
(cow_name, cow_path) = scrub.get_writable_cow(snapshot, volume)
with directio.open(cow_path) as file:
size = directio.size(cow_path)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.assert_(True)
break
with directio.open(self._ramdisk) as file:
size = directio.size(self._ramdisk)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.assert_(True)
break
# Scrub the cow of all exceptions
scrub.scrub_cow(cow_path)
scrub.remove_cow(cow_name)
# Remove & scrub the volume. LVM removes snapshot itself.
self.volume.remove_lvm_volume(volume)
# Read full disk for hidden zergs.
with directio.open(self._ramdisk) as file:
size = directio.size(self._ramdisk)
for i in xrange(0, size / block_size):
block = file.read(block_size)
if 'ZERG' in block:
self.fail("Found zergs on disk: %s" % self._ramdisk)
