def create_volume_from_snapshot(self, volume, snapshot):
"""
Creates a volume from a snapshot.
:param volume: new volume object
:param snapshot: existing snapshot object
"""
new_volume_name = self.VOLUME_PREFIX + str(volume.id)
api = self._setup_ovs_client()
# pick a random storagerouter to deploy the new clone on
storagerouter_ip = random.choice([
sr
for sr in StoragerouterHelper.get_storagerouters(api=api).values()
if self.vpool_guid in sr['vpools_guids']
])
VDiskSetup.create_clone(new_vdisk_name=new_volume_name,
storagerouter_ip=storagerouter_ip,
api=api,
vpool_guid=self.vpool_guid,
snapshot_name=snapshot['name'])
# return volume location
storage_ip = VDiskHelper.get_vdisk_location_by_name(
vdisk_name=new_volume_name, vpool_guid=self.vpool_guid,
api=api)['storage_ip']
location = self._get_volume_location(volume_name=new_volume_name,
storage_ip=storage_ip)
volume['provider_location'] = location
LOG.debug('libovsvolumedriver.ovs_clone_from_snapshot {0} '.format(
volume.id))
return {'provider_location': volume['provider_location']}
def create_cloned_volume(self, volume, src_vref):
"""
Creates a clone of the specified volume.
:param volume: new volume object
:param src_vref: source volume object
"""
source_volume_name = self.VOLUME_PREFIX + str(src_vref.id)
new_volume_name = self.VOLUME_PREFIX + str(volume.id)
LOG.debug('libovsvolumedriver.ovs_clone new={0} source={1}'.format(
volume.id, src_vref.id))
api = self._setup_ovs_client()
# pick a random storagerouter to deploy the new clone on
storagerouter_ip = random.choice([
sr
for sr in StoragerouterHelper.get_storagerouters(api=api).values()
if self.vpool_guid in sr['vpools_guids']
])
VDiskSetup.create_clone(vdisk_name=source_volume_name,
new_vdisk_name=new_volume_name,
storagerouter_ip=storagerouter_ip,
api=api,
vpool_guid=self.vpool_guid)
# return volume location
storage_ip = VDiskHelper.get_vdisk_location_by_name(
vdisk_name=new_volume_name, vpool_guid=self.vpool_guid,
api=api)['storage_ip']
location = self._get_volume_location(volume_name=new_volume_name,
storage_ip=storage_ip)
volume['provider_location'] = location
return {'provider_location': volume['provider_location']}
def restore_backup(self, context, backup, volume, backup_service):
"""
Restore an existing backup to a new or existing volume.
"""
volume_name = self.VOLUME_PREFIX + str(volume.id)
backup_snapshot_name = self.SNAPSHOT_BACKUP_PREFIX + str(backup['id'])
LOG.debug('libovsvolumedriver.ovs_backup_restore {0} {1}'.format(
volume_name, backup_snapshot_name))
api = self._setup_ovs_client()
snapshot, vdisk_guid, vdisk_name = VDiskHelper.get_snapshot_by_name(
snapshot_name=backup_snapshot_name,
vpool_guid=self.vpool_guid,
api=api)
if vdisk_name == volume_name:
# rollback
VDiskSetup.rollback_to_snapshot(vpool_guid=self.vpool_guid,
snapshot_name=backup_snapshot_name,
api=api)
else:
# to new volume
storagerouter_ip = random.choice([
sr for sr in StoragerouterHelper.get_storagerouters(
api=api).values() if self.vpool_guid in sr['vpools_guids']
])
VDiskSetup.create_clone(vpool_guid=self.vpool_guid,
new_vdisk_name=volume_name,
storagerouter_ip=storagerouter_ip,
api=api,
snapshot_name=backup_snapshot_name)
# return volume location
storage_ip = VDiskHelper.get_vdisk_location_by_name(
vdisk_name=volume_name, vpool_guid=self.vpool_guid,
api=api)['storage_ip']
location = self._get_volume_location(volume_name=volume_name,
storage_ip=storage_ip)
volume['provider_location'] = location
return {'provider_location': volume['provider_location']}
def create_vdisks(cls,
storagedriver,
amount_vdisks=AMOUNT_VDISKS_TO_SCRUB,
size=SIZE_VDISK,
cloned=False):
vpool = storagedriver.vpool
cls.LOGGER.info(
"Start deploying vdisks for scrubbing with clone status: {0}".
format(cloned))
vdisks = {
'parents': [], # Non cloned
'clones': []
} # Cloned
if cloned is True:
parent_vdisk_name = '{0}_clone_parent_{1}'.format(
cls.PREFIX,
str(0).zfill(3))
parent_vdisk_guid = VDiskSetup.create_vdisk(
vdisk_name=parent_vdisk_name,
vpool_name=vpool.name,
size=size,
storagerouter_ip=storagedriver.storagerouter.ip)
parent_vdisk = VDiskHelper.get_vdisk_by_guid(parent_vdisk_guid)
vdisks['parents'].append(parent_vdisk)
for vdisk_nr in xrange(amount_vdisks):
clone_vdisk_name = '{0}_clone{1}'.format(
parent_vdisk.name,
str(len(vdisks['clones']) + 1).zfill(3))
cloned_vdisk = VDiskHelper.get_vdisk_by_guid(
VDiskSetup.create_clone(
vdisk_name=parent_vdisk_name,
vpool_name=vpool.name,
new_vdisk_name=clone_vdisk_name,
storagerouter_ip=storagedriver.storagerouter.ip)
['vdisk_guid'])
vdisks['clones'].append(cloned_vdisk)
else:
for vdisk_nr in xrange(amount_vdisks):
vdisk_name = '{0}_{1}'.format(cls.PREFIX,
str(vdisk_nr).zfill(3))
vdisk_guid = VDiskSetup.create_vdisk(
vdisk_name=vdisk_name,
vpool_name=vpool.name,
size=size,
storagerouter_ip=storagedriver.storagerouter.ip)
vdisk = VDiskHelper.get_vdisk_by_guid(vdisk_guid)
vdisks['parents'].append(vdisk)
return vdisks
def validate_vdisk_clone(cls):
"""
Validate if vdisk deployment works via various ways
INFO: 1 vPool should be available on 2 storagerouters
:return:
"""
cls.LOGGER.info("Starting to validate clone vdisks")
vpools = VPoolHelper.get_vpools()
assert len(vpools) >= 1, "Not enough vPools to test"
try:
vpool = next(
(vpool for vpool in vpools if len(vpool.storagedrivers) >= 2))
except StopIteration:
assert False, "Not enough Storagedrivers to test"
# Setup base information
storagedriver_source = vpool.storagedrivers[0]
storagedriver_destination = vpool.storagedrivers[1]
vdisks = []
try:
# Create required vdisk for test
original_vdisk_name = '{0}_{1}'.format(cls.PREFIX, str(1).zfill(3))
cls.LOGGER.info(
"Creating the vdisk: {0} to clone".format(original_vdisk_name))
original_vdisk = VDiskHelper.get_vdisk_by_guid(
VDiskSetup.create_vdisk(
vdisk_name=original_vdisk_name,
vpool_name=vpool.name,
size=cls.VDISK_SIZE,
storagerouter_ip=storagedriver_source.storagerouter.ip))
vdisks.append(original_vdisk)
time.sleep(cls.CLONE_SLEEP_AFTER_CREATE)
###############
# Clone vdisk #
###############
cloned_vdisk_name = original_vdisk_name + '-clone-nosnapshot'
cloned_vdisk = VDiskHelper.get_vdisk_by_guid(
VDiskSetup.create_clone(
vdisk_name=original_vdisk_name,
vpool_name=vpool.name,
new_vdisk_name=cloned_vdisk_name,
storagerouter_ip=storagedriver_destination.storagerouter.ip
)['vdisk_guid'])
vdisks.append(cloned_vdisk)
time.sleep(cls.CLONE_SLEEP_BEFORE_CHECK)
######################################
# clone vdisk from existing snapshot #
######################################
cloned_vdisk_name = original_vdisk_name + '-clone-snapshot'
snapshot_id = VDiskSetup.create_snapshot(
vdisk_name=original_vdisk_name,
vpool_name=vpool.name,
snapshot_name=cls.PREFIX + 'snapshot')
cloned_vdisk = VDiskHelper.get_vdisk_by_guid(
VDiskSetup.create_clone(
vdisk_name=original_vdisk_name,
vpool_name=vpool.name,
new_vdisk_name=cloned_vdisk_name,
storagerouter_ip=storagedriver_destination.storagerouter.
ip,
snapshot_id=snapshot_id)['vdisk_guid'])
vdisks.append(cloned_vdisk)
finally:
VDiskRemover.remove_vdisks_with_structure(vdisks)
cls.LOGGER.info("Finished validating clone vdisks")
def prepare_vm_disks(self,
source_storagedriver,
cloud_image_path,
cloud_init_loc,
vm_name,
data_disk_size,
edge_user_info=None,
logger=LOGGER):
"""
Will create all necessary vdisks to create the bulk of vms
:param source_storagedriver: storagedriver to create the disks on
:param cloud_image_path: path to the cloud image
:param cloud_init_loc: path to the cloud init script
:param vm_name: name prefix for the vms
:param data_disk_size: size of the data disk
:param edge_user_info: user information for the edge. Optional
:param logger: logging instance
:return:
"""
logger.info('Starting with preparing vm disk(s)')
vm_amount = self.amount_of_vms
if isinstance(edge_user_info, dict):
required_edge_params = {
'username': (str, None, False),
'password': (str, None, False)
}
ExtensionsToolbox.verify_required_params(required_edge_params,
edge_user_info)
if edge_user_info is None:
edge_user_info = {}
protocol = source_storagedriver.cluster_node_config[
'network_server_uri'].split(':')[0]
vpool = source_storagedriver.vpool
client = SSHClient(source_storagedriver.storagerouter, username='******')
edge_configuration = {
'ip': source_storagedriver.storage_ip,
'port': source_storagedriver.ports['edge'],
'protocol': protocol
}
edge_configuration.update(edge_user_info)
original_boot_disk_name = None # Cloning purposes
original_data_disk_name = None # Cloning purposes
connection_messages = []
vm_info = {}
volume_amount = 0
for vm_number in xrange(0, vm_amount):
filled_number = str(vm_number).zfill(3)
vm_name = '{0}-{1}'.format(vm_name, filled_number)
create_msg = '{0}_{1}'.format(str(uuid.uuid4()), vm_name)
boot_vdisk_name = '{0}_vdisk_boot_{1}'.format(
vm_name, filled_number)
data_vdisk_name = '{0}_vdisk_data_{1}'.format(
vm_name, filled_number)
cd_vdisk_name = '{0}_vdisk_cd_{1}'.format(vm_name, filled_number)
boot_vdisk_path = '/mnt/{0}/{1}.raw'.format(
vpool.name, boot_vdisk_name)
data_vdisk_path = '/mnt/{0}/{1}.raw'.format(
vpool.name, data_vdisk_name)
cd_vdisk_path = '/mnt/{0}/{1}.raw'.format(vpool.name,
cd_vdisk_name)
if vm_number == 0:
try:
# Create VDISKs
self.convert_image(client, cloud_image_path,
boot_vdisk_name, edge_configuration)
except RuntimeError as ex:
logger.error('Could not covert the image. Got {0}'.format(
str(ex)))
raise
boot_vdisk = VDiskHelper.get_vdisk_by_name(
'{0}.raw'.format(boot_vdisk_name), vpool.name)
original_boot_disk_name = boot_vdisk_name
logger.info('Boot VDisk successfully created.')
try:
data_vdisk = VDiskHelper.get_vdisk_by_guid(
VDiskSetup.create_vdisk(
data_vdisk_name, vpool.name, data_disk_size,
source_storagedriver.storage_ip))
logger.info('VDisk data_vdisk successfully created!')
except TimeOutError:
logger.error(
'The creation of the data vdisk has timed out.')
raise
except RuntimeError as ex:
logger.error(
'Could not create the data vdisk. Got {0}'.format(
str(ex)))
raise
original_data_disk_name = data_vdisk_name
else:
# Rely on cloning to speed up the process
boot_vdisk_info = VDiskSetup.create_clone(
vdisk_name=original_boot_disk_name,
vpool_name=vpool.name,
new_vdisk_name=boot_vdisk_name,
storagerouter_ip=source_storagedriver.storage_ip)
boot_vdisk = VDiskHelper.get_vdisk_by_guid(
boot_vdisk_info['vdisk_guid'])
data_vdisk_info = VDiskSetup.create_clone(
vdisk_name=original_data_disk_name,
vpool_name=vpool.name,
new_vdisk_name=data_vdisk_name,
storagerouter_ip=source_storagedriver.storage_ip)
data_vdisk = VDiskHelper.get_vdisk_by_guid(
data_vdisk_info['vdisk_guid'])
#######################
# GENERATE CLOUD INIT #
#######################
iso_loc = self._generate_cloud_init(
client=client,
convert_script_loc=cloud_init_loc,
create_msg=create_msg)
self.convert_image(client, iso_loc, cd_vdisk_name,
edge_configuration)
cd_creation_time = time.time()
cd_vdisk = None
while cd_vdisk is None:
if time.time() - cd_creation_time > 60:
raise RuntimeError(
'Could not fetch the cd vdisk after {}s'.format(
time.time() - cd_creation_time))
try:
cd_vdisk = VDiskHelper.get_vdisk_by_name(
cd_vdisk_name, vpool.name)
except VDiskNotFoundError:
logger.warning(
'Could not fetch the cd vdisk after {0}s.'.format(
time.time() - cd_creation_time))
time.sleep(0.5)
# Take snapshot to revert back to after every migrate scenario
data_snapshot_guid = VDiskSetup.create_snapshot(
'{0}_data'.format(vm_name),
data_vdisk.devicename,
vpool.name,
consistent=False)
vm_info[vm_name] = {
'data_snapshot_guid':
data_snapshot_guid,
'vdisks': [boot_vdisk, data_vdisk, cd_vdisk],
'cd_path':
cd_vdisk_path,
'disks': [{
'mountpoint': boot_vdisk_path
}, {
'mountpoint': data_vdisk_path
}],
'networks': [{
'network': 'default',
'mac': 'RANDOM',
'model': 'e1000'
}],
'created':
False,
'ip':
'',
'create_msg':
create_msg
}
connection_messages.append(create_msg)
volume_amount += len(vm_info[vm_name]['vdisks'])
logger.info('Prepped everything for VM {0}.'.format(vm_name))
self.vm_info = vm_info
self.connection_messages = connection_messages
self.volume_amount = volume_amount
def _deploy_vdisks(cls,
vpool,
storagedriver,
size=SIZE_VDISK,
amount_vdisks=AMOUNT_VDISKS,
cloned=False):
"""
Deploy X amount of vdisks, write some data to it & snapshot
:param vpool: a valid vpool object
:type vpool: ovs.model.hybrids.vpool
:param storagedriver: a valid storagedriver object
:type storagedriver: ovs.mode.hybrids.storagedriver
:param size: size of a single vdisk in bytes
:type size: int
:return: tuple[0]: stored vdisks, snapshot, location; tuple[1]: base_vdisks_guids that are used for clones
[{
'vdisk_guid': u 'b789b23e-1077-4d96-9ec2-a7cc3785686c',
'snapshots': {
0: {
'snapshot_name': 'integration-tests-rollback0-snapshot0',
'snapshot_guid': u 'fbd1c961-7d33-4bd3-8c92-c8a3c52eb74f',
'stored_data': 52428800
},
1: {
'snapshot_name': 'integration-tests-rollback0-snapshot1',
'snapshot_guid': u '15eb7119-d984-4c84-985c-1fb1cc44a95e',
'stored_data': 104857600
}
}
}, {
'vdisk_guid': u '9c2cd023-d15b-4994-8a62-07edc36d748c',
'snapshots': {
0: {
'snapshot_name': 'integration-tests-rollback1-snapshot0',
'snapshot_guid': u 'c7500fec-cc5a-4593-89dc-fca78dcb2783',
'stored_data': 52428800
},
1: {
'snapshot_name': 'integration-tests-rollback1-snapshot1',
'snapshot_guid': u 'e46bd42b-516d-4636-9d15-d9c1a8f489e4',
'stored_data': 104857600
}
}
}], ['8858717a-e6d2-11e6-831d-00249b133798', '8c644a82-e6d2-11e6-8efe-00249b133798']
:rtype: tuple
"""
cls.LOGGER.info(
"Starting deploying {0} vdisks with clone status: {1}".format(
amount_vdisks, cloned))
client = SSHClient(storagedriver.storagerouter.ip, username='******')
vdisks = []
base_vdisks = []
for vdisk_nr in xrange(amount_vdisks):
# create a vdisk & collect results
vdisk_name = cls.VDISK_NAME + str(vdisk_nr)
vdisk_guid = VDiskSetup.create_vdisk(
vdisk_name=vdisk_name + '.raw',
vpool_name=vpool.name,
size=size,
storagerouter_ip=storagedriver.storagerouter.ip)
# clone
if cloned:
clone_vdisk_name = vdisk_name + '_clone'
cls.LOGGER.info(
"Creating clone from vdisk `{0}` with new name `{1}`".
format(vdisk_name, clone_vdisk_name))
base_vdisks.append(str(vdisk_guid))
cls.LOGGER.info(
"Stored old base vdisk guid in list: {0}".format(
vdisk_guid))
vdisk_guid = VDiskSetup.create_clone(
vdisk_name=vdisk_name + '.raw',
vpool_name=vpool.name,
new_vdisk_name=clone_vdisk_name,
storagerouter_ip=storagedriver.storagerouter.ip
)['vdisk_guid']
vdisk_name = clone_vdisk_name
vdisk = VDiskHelper.get_vdisk_by_guid(vdisk_guid)
results = {'vdisk_guid': vdisk_guid, 'snapshots': {}}
cls.LOGGER.info(
"Finished deploying vdisk `{0}`".format(vdisk_name))
cls.LOGGER.info(
"Starting writing & snapshotting vdisk `{0}`".format(
vdisk_name))
for i in xrange(cls.WRITE_AMOUNT_OF_TIMES):
# write some data
try:
RollbackChecks.LOGGER.info(
"Starting FIO on vdisk `{0}`".format(vdisk_name))
client.run([
"fio", "--name=test",
"--filename=/mnt/{0}/{1}.raw".format(
vpool.name, vdisk_name), "--ioengine=libaio",
"--iodepth=4", "--rw=write", "--bs=4k", "--direct=1",
"--size={0}b".format(size)
])
RollbackChecks.LOGGER.info(
"Finished FIO on vdisk `{0}`".format(vdisk_name))
except subprocess.CalledProcessError as ex:
raise VDiskNotFoundError(
"VDisk `/mnt/{0}/{1}.raw` does not seem to be present "
"or has problems on storagerouter `{2}`: {3}".format(
vpool.name, vdisk_name,
storagedriver.storagerouter.ip, str(ex)))
# create snapshot
cls.LOGGER.info(
"Starting snapshot creation on vdisk `{0}`".format(
vdisk_name))
snapshot_guid = VDiskSetup.create_snapshot(
snapshot_name=vdisk_name + '-snapshot{0}'.format(i),
vdisk_name=vdisk_name + '.raw',
vpool_name=vpool.name,
consistent=False,
sticky=False)
# save the current stored_data for comparison
stored_data = vdisk.storagedriver_client.info_volume(
str(vdisk.volume_id)).stored
cls.LOGGER.info(
"Logged `{0}` stored data for VDisk `{1}` in mapper".
format(stored_data, vdisk_name))
# add details to snapshot mapper
results['snapshots'][i] = {
'snapshot_guid': snapshot_guid,
'snapshot_name': vdisk_name + '-snapshot{0}'.format(i),
'stored_data': stored_data
}
cls.LOGGER.info(
"Snapshot creation finished on vdisk `{0}`".format(
vdisk_name))
vdisks.append(results)
cls.LOGGER.info(
"Finished writing & snapshotting vdisk `{0}`. Results: {1}".
format(vdisk_name, results))
return vdisks, base_vdisks
def validate_vdisk_clone(cls):
"""
Validate if vdisk deployment works via various ways
INFO: 1 vPool should be available on 2 storagerouters
:return:
"""
cls.LOGGER.info("Starting to validate template vdisks")
vpools = VPoolHelper.get_vpools()
assert len(vpools) >= 1, "Not enough vPools to test"
try:
vpool = next(
(vpool for vpool in vpools if len(vpool.storagedrivers) >= 2))
except StopIteration:
assert False, "Not enough Storagedrivers to test"
# setup base information
storagedriver_source = vpool.storagedrivers[0]
vdisks = []
try:
# create required vdisk for test
parent_vdisk_name = '{0}_{1}'.format(cls.PREFIX, str(1).zfill(3))
parent_vdisk = VDiskHelper.get_vdisk_by_guid(
VDiskSetup.create_vdisk(
vdisk_name=parent_vdisk_name,
vpool_name=vpool.name,
size=cls.VDISK_SIZE,
storagerouter_ip=storagedriver_source.storagerouter.ip))
vdisks.append(parent_vdisk)
time.sleep(cls.TEMPLATE_SLEEP_AFTER_CREATE)
# Create vdisk template #
VDiskSetup.set_vdisk_as_template(vdisk_name=parent_vdisk_name,
vpool_name=vpool.name)
time.sleep(cls.TEMPLATE_SLEEP_AFTER_CREATE)
clone_vdisk_name = '{0}_from-template'.format(parent_vdisk_name)
clone_vdisk = VDiskHelper.get_vdisk_by_guid(
VDiskSetup.create_from_template(
vdisk_name=parent_vdisk_name,
vpool_name=vpool.name,
new_vdisk_name=clone_vdisk_name,
storagerouter_ip=storagedriver_source.storagerouter.ip)
['vdisk_guid'])
vdisks.append(clone_vdisk)
time.sleep(cls.TEMPLATE_SLEEP_BEFORE_DELETE)
try:
# try to delete template with clones (should fail) #
VDiskRemover.remove_vtemplate_by_name(
vdisk_name=parent_vdisk_name, vpool_name=vpool.name)
error_msg = "Removing vtemplate `{0}` should have failed!"
cls.LOGGER.error(error_msg)
raise RuntimeError(error_msg)
except HttpException:
cls.LOGGER.info(
"Removing vtemplate `{0}` has failed as expected (because of leftover clones)!"
.format(parent_vdisk_name))
finally:
while len(vdisks) > 0:
vdisk = vdisks.pop()
VDiskRemover.remove_vdisk(vdisk.guid)
try:
# template vdisk from clone (should fail) #
parent_vdisk = VDiskHelper.get_vdisk_by_guid(
VDiskSetup.create_vdisk(
vdisk_name=parent_vdisk_name,
vpool_name=vpool.name,
size=cls.VDISK_SIZE,
storagerouter_ip=storagedriver_source.storagerouter.ip))
vdisks.append(parent_vdisk)
# create a clone from the vdisk
clone_vdisk_name = '{0}_clone'.format(parent_vdisk_name)
cloned_vdisk = VDiskHelper.get_vdisk_by_guid(
VDiskSetup.create_clone(
vdisk_name=parent_vdisk_name,
vpool_name=vpool.name,
new_vdisk_name=clone_vdisk_name,
storagerouter_ip=storagedriver_source.storagerouter.ip)
['vdisk_guid'])
vdisks.append(cloned_vdisk)
# try to create a vTemplate from a clone
try:
VDiskSetup.set_vdisk_as_template(vdisk_name=clone_vdisk_name,
vpool_name=vpool.name)
error_msg = "Setting vdisk `{0}` as template should have failed!".format(
clone_vdisk_name)
cls.LOGGER.error(error_msg)
raise RuntimeError(error_msg)
except RuntimeError:
cls.LOGGER.info(
"Setting vdisk `{0}` as template failed as expected (because vdisk is clone)!"
.format(clone_vdisk_name))
finally:
parent_vdisks = []
while len(vdisks) > 0: # Remove clones first
vdisk = vdisks.pop()
if vdisk.parent_vdisk_guid is None:
parent_vdisks.append(vdisk)
continue
VDiskRemover.remove_vdisk(vdisk.guid)
for parent_vdisk in parent_vdisks:
VDiskRemover.remove_vdisk(parent_vdisk.guid)
cls.LOGGER.info("Finished to validate template vdisks")