def sync_with_hypervisor(vmachineguid, storagedriver_id=None):
"""
Updates a given vmachine with data retrieved from a given pmachine
:param vmachineguid: Guid of the virtual machine
:param storagedriver_id: Storage Driver hosting the vmachine
"""
try:
vmachine = VMachine(vmachineguid)
except Exception as ex:
VMachineController._logger.info('Cannot get VMachine object: {0}'.format(str(ex)))
raise
vm_object = None
if vmachine.pmachine.mgmtcenter and storagedriver_id is not None and vmachine.devicename is not None:
try:
mgmt_center = Factory.get_mgmtcenter(vmachine.pmachine)
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
VMachineController._logger.info('Syncing vMachine (name {0}) with Management center {1}'.format(vmachine.name, vmachine.pmachine.mgmtcenter.name))
vm_object = mgmt_center.get_vm_agnostic_object(devicename=vmachine.devicename,
ip=storagedriver.storage_ip,
mountpoint=storagedriver.mountpoint)
except Exception as ex:
VMachineController._logger.info('Error while fetching vMachine info from management center: {0}'.format(str(ex)))
if vm_object is None and storagedriver_id is None and vmachine.hypervisor_id is not None and vmachine.pmachine is not None:
try:
# Only the vmachine was received, so base the sync on hypervisor id and pmachine
hypervisor = Factory.get(vmachine.pmachine)
VMachineController._logger.info('Syncing vMachine (name {0})'.format(vmachine.name))
vm_object = hypervisor.get_vm_agnostic_object(vmid=vmachine.hypervisor_id)
except Exception as ex:
VMachineController._logger.info('Error while fetching vMachine info from hypervisor: {0}'.format(str(ex)))
if vm_object is None and storagedriver_id is not None and vmachine.devicename is not None:
try:
# Storage Driver id was given, using the devicename instead (to allow hypervisor id updates
# which can be caused by re-adding a vm to the inventory)
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
hypervisor = Factory.get(pmachine)
if not hypervisor.file_exists(storagedriver, hypervisor.clean_vmachine_filename(vmachine.devicename)):
return
vmachine.pmachine = pmachine
vmachine.save()
VMachineController._logger.info('Syncing vMachine (device {0}, ip {1}, mountpoint {2})'.format(vmachine.devicename,
storagedriver.storage_ip,
storagedriver.mountpoint))
vm_object = hypervisor.get_vm_object_by_devicename(devicename=vmachine.devicename,
ip=storagedriver.storage_ip,
mountpoint=storagedriver.mountpoint)
except Exception as ex:
VMachineController._logger.info('Error while fetching vMachine info from hypervisor using devicename: {0}'.format(str(ex)))
if vm_object is None:
message = 'Not enough information to sync vmachine'
VMachineController._logger.info('Error: {0}'.format(message))
raise RuntimeError(message)
VMachineController.update_vmachine_config(vmachine, vm_object)
def sync_with_hypervisor(vmachineguid, storagedriver_id=None):
"""
Updates a given vmachine with data retrieved from a given pmachine
:param vmachineguid: Guid of the virtual machine
:param storagedriver_id: Storage Driver hosting the vmachine
"""
try:
vmachine = VMachine(vmachineguid)
except Exception as ex:
VMachineController._logger.info('Cannot get VMachine object: {0}'.format(str(ex)))
raise
vm_object = None
if vmachine.pmachine.mgmtcenter and storagedriver_id is not None and vmachine.devicename is not None:
try:
mgmt_center = Factory.get_mgmtcenter(vmachine.pmachine)
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
VMachineController._logger.info('Syncing vMachine (name {0}) with Management center {1}'.format(vmachine.name, vmachine.pmachine.mgmtcenter.name))
vm_object = mgmt_center.get_vm_agnostic_object(devicename=vmachine.devicename,
ip=storagedriver.storage_ip,
mountpoint=storagedriver.mountpoint)
except Exception as ex:
VMachineController._logger.info('Error while fetching vMachine info from management center: {0}'.format(str(ex)))
if vm_object is None and storagedriver_id is None and vmachine.hypervisor_id is not None and vmachine.pmachine is not None:
try:
# Only the vmachine was received, so base the sync on hypervisor id and pmachine
hypervisor = Factory.get(vmachine.pmachine)
VMachineController._logger.info('Syncing vMachine (name {0})'.format(vmachine.name))
vm_object = hypervisor.get_vm_agnostic_object(vmid=vmachine.hypervisor_id)
except Exception as ex:
VMachineController._logger.info('Error while fetching vMachine info from hypervisor: {0}'.format(str(ex)))
if vm_object is None and storagedriver_id is not None and vmachine.devicename is not None:
try:
# Storage Driver id was given, using the devicename instead (to allow hypervisor id updates
# which can be caused by re-adding a vm to the inventory)
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
hypervisor = Factory.get(pmachine)
if not hypervisor.file_exists(storagedriver, hypervisor.clean_vmachine_filename(vmachine.devicename)):
return
vmachine.pmachine = pmachine
vmachine.save()
VMachineController._logger.info('Syncing vMachine (device {0}, ip {1}, mountpoint {2})'.format(vmachine.devicename,
storagedriver.storage_ip,
storagedriver.mountpoint))
vm_object = hypervisor.get_vm_object_by_devicename(devicename=vmachine.devicename,
ip=storagedriver.storage_ip,
mountpoint=storagedriver.mountpoint)
except Exception as ex:
VMachineController._logger.info('Error while fetching vMachine info from hypervisor using devicename: {0}'.format(str(ex)))
if vm_object is None:
message = 'Not enough information to sync vmachine'
VMachineController._logger.info('Error: {0}'.format(message))
raise RuntimeError(message)
VMachineController.update_vmachine_config(vmachine, vm_object)
def resize_from_voldrv(volumename, volumesize, volumepath,
storagedriver_id):
"""
Resize a disk
Triggered by volumedriver messages on the queue
@param volumepath: path on hypervisor to the volume
@param volumename: volume id of the disk
@param volumesize: size of the volume
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
hypervisor = Factory.get(pmachine)
volumepath = hypervisor.clean_backing_disk_filename(volumepath)
mutex = VolatileMutex('{}_{}'.format(volumename, volumepath))
try:
mutex.acquire(wait=30)
disk = VDiskList.get_vdisk_by_volume_id(volumename)
if disk is None:
disk = VDiskList.get_by_devicename_and_vpool(
volumepath, storagedriver.vpool)
if disk is None:
disk = VDisk()
finally:
mutex.release()
disk.devicename = volumepath
disk.volume_id = volumename
disk.size = volumesize
disk.vpool = storagedriver.vpool
disk.save()
def resize_from_voldrv(volumename, volumesize, volumepath, storagedriver_id):
"""
Resize a disk
Triggered by volumedriver messages on the queue
@param volumepath: path on hypervisor to the volume
@param volumename: volume id of the disk
@param volumesize: size of the volume
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
hypervisor = Factory.get(pmachine)
volumepath = hypervisor.clean_backing_disk_filename(volumepath)
mutex = VolatileMutex('{}_{}'.format(volumename, volumepath))
try:
mutex.acquire(wait=30)
disk = VDiskList.get_vdisk_by_volume_id(volumename)
if disk is None:
disk = VDiskList.get_by_devicename_and_vpool(volumepath, storagedriver.vpool)
if disk is None:
disk = VDisk()
finally:
mutex.release()
disk.devicename = volumepath
disk.volume_id = volumename
disk.size = volumesize
disk.vpool = storagedriver.vpool
disk.save()
VDiskController.sync_with_mgmtcenter(disk, pmachine, storagedriver)
MDSServiceController.ensure_safety(disk)
def update_status(storagedriver_id):
"""
Sets Storage Driver offline in case hypervisor management Center
reports the hypervisor pmachine related to this Storage Driver
as unavailable.
:param storagedriver_id: ID of the storagedriver to update its status
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
storagerouter = storagedriver.storagerouter
if pmachine.mgmtcenter:
# Update status
pmachine.invalidate_dynamics(['host_status'])
else:
# No management Center, cannot update status via api
logger.info('Updating status of pmachine {0} using SSHClient'.format(pmachine.name))
host_status = 'RUNNING'
try:
client = SSHClient(storagerouter, username='******')
configuration_dir = EtcdConfiguration.get('/ovs/framework/paths|cfgdir')
logger.info('SSHClient connected successfully to {0} at {1}'.format(pmachine.name, client.ip))
with Remote(client.ip, [LocalStorageRouterClient]) as remote:
lsrc = remote.LocalStorageRouterClient('{0}/storagedriver/storagedriver/{1}.json'.format(configuration_dir,
storagedriver.vpool.name))
lsrc.server_revision()
logger.info('LocalStorageRouterClient connected successfully to {0} at {1}'.format(pmachine.name, client.ip))
except Exception as ex:
logger.error('Connectivity check failed, assuming host {0} is halted. {1}'.format(pmachine.name, ex))
host_status = 'HALTED'
if host_status != 'RUNNING':
# Host is stopped
storagedriver_client = StorageDriverClient.load(storagedriver.vpool)
storagedriver_client.mark_node_offline(str(storagedriver.storagedriver_id))
def delete_from_voldrv(name, storagedriver_id):
"""
This method will delete a vmachine based on the name of the vmx given
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
if pmachine.hvtype not in ['VMWARE', 'KVM']:
return
hypervisor = Factory.get(pmachine)
name = hypervisor.clean_vmachine_filename(name)
if pmachine.hvtype == 'VMWARE':
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
vpool = storagedriver.vpool
else:
vpool = None
vm = VMachineList.get_by_devicename_and_vpool(name, vpool)
if vm is not None:
MessageController.fire(MessageController.Type.EVENT, {
'type': 'vmachine_deleted',
'metadata': {
'name': vm.name
}
})
vm.delete(abandon=['vdisks'])
def get_stats_proxies(cls):
"""
Retrieve statistics for all ALBA proxies
"""
if cls._config is None:
cls.validate_and_retrieve_config()
stats = []
errors = False
environment = cls._config['environment']
vpool_namespace_cache = {}
for storagedriver in StorageDriverList.get_storagedrivers():
for alba_proxy_service in storagedriver.alba_proxies:
ip = storagedriver.storage_ip
port = alba_proxy_service.service.ports[0]
try:
vpool = storagedriver.vpool
if vpool.guid not in vpool_namespace_cache:
vpool_namespace_cache[
vpool.
guid] = vpool.storagedriver_client.list_volumes(
req_timeout_secs=5)
active_namespaces = vpool_namespace_cache[vpool.guid]
for namespace_stats in AlbaCLI.run(
command='proxy-statistics',
named_params={
'host': ip,
'port': port
})['ns_stats']:
namespace = namespace_stats[0]
if namespace not in active_namespaces:
continue
stats.append({
'tags': {
'server':
storagedriver.storagerouter.name,
'namespace':
namespace,
'vpool_name':
vpool.name,
'environment':
environment,
'backend_name':
vpool.metadata['backend']['backend_info']
['name'],
'service_name':
alba_proxy_service.service.name
},
'fields':
cls._convert_to_float_values(namespace_stats[1]),
'measurement':
'proxyperformance_namespace'
})
except Exception:
errors = True
cls._logger.exception(
"Failed to retrieve proxy statistics for proxy service running at {0}:{1}"
.format(ip, port))
return errors, stats
def create_volume(self,
target_path,
metadata_backend_config,
volume_size,
node_id,
req_timeout_secs=None):
"""
Create a mocked volume
"""
_ = req_timeout_secs
from ovs.dal.lists.storagedriverlist import StorageDriverList
volume_id = str(uuid.uuid4())
storagedriver = StorageDriverList.get_by_storagedriver_id(node_id)
if storagedriver is None:
raise ValueError(
'Failed to retrieve storagedriver with ID {0}'.format(node_id))
StorageRouterClient.vrouter_id[self.vpool_guid][volume_id] = node_id
StorageRouterClient._metadata_backend_config[
self.vpool_guid][volume_id] = metadata_backend_config
StorageRouterClient.volumes[self.vpool_guid][volume_id] = {
'volume_id': volume_id,
'volume_size': volume_size,
'target_path': target_path
}
return volume_id
def get_storagedrivers():
"""
Fetches all storagedrivers
:return: list of all storagedrivers
:rtype: (ovs.dal.hybrids.storagedriver.STORAGEDRIVER)
"""
return StorageDriverList.get_storagedrivers()
def clone(diskguid, snapshotid, devicename, pmachineguid, machinename, machineguid=None):
"""
Clone a disk
"""
pmachine = PMachine(pmachineguid)
hypervisor = Factory.get(pmachine)
description = '{} {}'.format(machinename, devicename)
properties_to_clone = ['description', 'size', 'type', 'retentionpolicyguid',
'snapshotpolicyguid', 'autobackup']
vdisk = VDisk(diskguid)
location = hypervisor.get_backing_disk_path(machinename, devicename)
new_vdisk = VDisk()
new_vdisk.copy(vdisk, include=properties_to_clone)
new_vdisk.parent_vdisk = vdisk
new_vdisk.name = '{0}-clone'.format(vdisk.name)
new_vdisk.description = description
new_vdisk.devicename = hypervisor.clean_backing_disk_filename(location)
new_vdisk.parentsnapshot = snapshotid
new_vdisk.vmachine = VMachine(machineguid) if machineguid else vdisk.vmachine
new_vdisk.vpool = vdisk.vpool
new_vdisk.save()
try:
storagedriver = StorageDriverList.get_by_storagedriver_id(vdisk.storagedriver_id)
if storagedriver is None:
raise RuntimeError('Could not find StorageDriver with id {0}'.format(vdisk.storagedriver_id))
mds_service = MDSServiceController.get_preferred_mds(storagedriver.storagerouter, vdisk.vpool)
if mds_service is None:
raise RuntimeError('Could not find a MDS service')
logger.info('Clone snapshot {} of disk {} to location {}'.format(snapshotid, vdisk.name, location))
volume_id = vdisk.storagedriver_client.create_clone(
target_path=location,
metadata_backend_config=MDSMetaDataBackendConfig([MDSNodeConfig(address=str(mds_service.service.storagerouter.ip),
port=mds_service.service.ports[0])]),
parent_volume_id=str(vdisk.volume_id),
parent_snapshot_id=str(snapshotid),
node_id=str(vdisk.storagedriver_id)
)
except Exception as ex:
logger.error('Caught exception during clone, trying to delete the volume. {0}'.format(ex))
new_vdisk.delete()
VDiskController.delete_volume(location)
raise
new_vdisk.volume_id = volume_id
new_vdisk.save()
try:
MDSServiceController.ensure_safety(new_vdisk)
except Exception as ex:
logger.error('Caught exception during "ensure_safety" {0}'.format(ex))
return {'diskguid': new_vdisk.guid,
'name': new_vdisk.name,
'backingdevice': location}
def _log(task, kwargs, storagedriver_id):
log = Log()
log.source = 'VOLUMEDRIVER_EVENT'
log.module = task.__class__.__module__
log.method = task.__class__.__name__
log.method_kwargs = kwargs
log.time = time.time()
log.storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
log.save()
def list(self, vpool_guid=None):
"""
Overview of all StorageDrivers
:param vpool_guid: Guid of the vPool
:type vpool_guid: str
"""
if vpool_guid is not None:
return VPool(vpool_guid).storagedrivers
return StorageDriverList.get_storagedrivers()
def list(self, vpool_guid=None):
"""
Overview of all StorageDrivers
:param vpool_guid: Guid of the vPool
:type vpool_guid: str
"""
if vpool_guid is not None:
return VPool(vpool_guid).storagedrivers
return StorageDriverList.get_storagedrivers()
def get_storagedriver_by_id(storagedriver_id):
"""
Fetches the storagedriver with its storagedriver_id
:param storagedriver_id: id of the storagedriver
:type storagedriver_id: str
:return: The storagedriver DAL object
:rtype: ovs.dal.hybrids.storagedriver.STORAGEDRIVER
"""
return StorageDriverList.get_by_storagedriver_id(storagedriver_id)
def _log(task, kwargs, storagedriver_id):
log = Log()
log.source = 'VOLUMEDRIVER_EVENT'
log.module = task.__class__.__module__
log.method = task.__class__.__name__
log.method_kwargs = kwargs
log.time = time.time()
log.storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
log.save()
def up_and_running(storagedriver_id):
"""
Volumedriver informs us that the service is completely started. Post-start events can be executed
:param storagedriver_id: ID of the storagedriver
"""
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
if storagedriver is None:
raise RuntimeError('A Storage Driver with id {0} could not be found.'.format(storagedriver_id))
storagedriver.startup_counter += 1
storagedriver.save()
def up_and_running(storagedriver_id):
"""
Volumedriver informs us that the service is completely started. Post-start events can be executed
:param storagedriver_id: ID of the storagedriver
"""
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
if storagedriver is None:
raise RuntimeError('A Storage Driver with id {0} could not be found.'.format(storagedriver_id))
storagedriver.startup_counter += 1
storagedriver.save()
def migrate(self, volume_id, node_id, force_restart, req_timeout_secs=None):
"""
Dummy migrate method
"""
_ = force_restart, req_timeout_secs
from ovs.dal.lists.storagedriverlist import StorageDriverList
storagedriver = StorageDriverList.get_by_storagedriver_id(node_id)
if storagedriver is None:
raise ValueError('Failed to retrieve storagedriver with ID {0}'.format(node_id))
StorageRouterClient.vrouter_id[self.vpool_guid][volume_id] = node_id
def getEdgeconnection(vpoolname=VPOOLNAME):
protocol = getEdgeProtocol()
storagedrivers = list(StorageDriverList.get_storagedrivers())
random.shuffle(storagedrivers)
for storagedriver in storagedrivers:
if storagedriver.status == 'FAILURE':
continue
if (vpoolname is not None and storagedriver.vpool.name == vpoolname) or \
(vpoolname is None and storagedriver.vpool.name != VPOOLNAME):
return storagedriver.storage_ip, storagedriver.ports['edge'], protocol
return None, None, protocol
def delete(diskguid):
"""
Delete a vdisk through API
@param diskguid: GUID of the vdisk to delete
"""
vdisk = VDisk(diskguid)
storagedriver = StorageDriverList.get_by_storagedriver_id(vdisk.storagedriver_id)
location = os.path.join(storagedriver.mountpoint, vdisk.devicename)
logger.info('Deleting disk {0} on location {1}'.format(vdisk.name, location))
VDiskController.delete_volume(location=location)
logger.info('Deleted disk {0}'.format(location))
def _execute_scrub_work(scrub_location, vdisk_guids):
def _verify_mds_config(current_vdisk):
current_vdisk.invalidate_dynamics(['info'])
vdisk_configs = current_vdisk.info['metadata_backend_config']
if len(vdisk_configs) == 0:
raise RuntimeError('Could not load MDS configuration')
return vdisk_configs
ScheduledTaskController._logger.info('Execute Scrub - Started')
ScheduledTaskController._logger.info('Execute Scrub - Scrub location - {0}'.format(scrub_location))
total = len(vdisk_guids)
skipped = 0
storagedrivers = {}
failures = []
for vdisk_guid in vdisk_guids:
vdisk = VDisk(vdisk_guid)
try:
# Load the vDisk's StorageDriver
ScheduledTaskController._logger.info('Execute Scrub - Virtual disk {0} - {1} - Started'.format(vdisk.guid, vdisk.name))
vdisk.invalidate_dynamics(['storagedriver_id'])
if vdisk.storagedriver_id not in storagedrivers:
storagedrivers[vdisk.storagedriver_id] = StorageDriverList.get_by_storagedriver_id(vdisk.storagedriver_id)
storagedriver = storagedrivers[vdisk.storagedriver_id]
# Load the vDisk's MDS configuration
configs = _verify_mds_config(current_vdisk=vdisk)
# Check MDS master is local. Trigger MDS handover if necessary
if configs[0].get('ip') != storagedriver.storagerouter.ip:
ScheduledTaskController._logger.debug('Execute Scrub - Virtual disk {0} - {1} - MDS master is not local, trigger handover'.format(vdisk.guid, vdisk.name))
MDSServiceController.ensure_safety(vdisk)
configs = _verify_mds_config(current_vdisk=vdisk)
if configs[0].get('ip') != storagedriver.storagerouter.ip:
skipped += 1
ScheduledTaskController._logger.info('Execute Scrub - Virtual disk {0} - {1} - Skipping because master MDS still not local'.format(vdisk.guid, vdisk.name))
continue
with vdisk.storagedriver_client.make_locked_client(str(vdisk.volume_id)) as locked_client:
ScheduledTaskController._logger.info('Execute Scrub - Virtual disk {0} - {1} - Retrieve and apply scrub work'.format(vdisk.guid, vdisk.name))
work_units = locked_client.get_scrubbing_workunits()
for work_unit in work_units:
scrubbing_result = locked_client.scrub(work_unit, scrub_location, log_sinks=[SCRUBBER_LOGFILE_LOCATION])
locked_client.apply_scrubbing_result(scrubbing_result)
if work_units:
ScheduledTaskController._logger.info('Execute Scrub - Virtual disk {0} - {1} - Scrub successfully applied'.format(vdisk.guid, vdisk.name))
else:
ScheduledTaskController._logger.info('Execute Scrub - Virtual disk {0} - {1} - No scrubbing required'.format(vdisk.guid, vdisk.name))
except Exception as ex:
failures.append('Failed scrubbing work unit for volume {0} with guid {1}: {2}'.format(vdisk.name, vdisk.guid, ex))
failed = len(failures)
ScheduledTaskController._logger.info('Execute Scrub - Finished - Success: {0} - Failed: {1} - Skipped: {2}'.format((total - failed - skipped), failed, skipped))
if failed > 0:
raise Exception('\n - '.join(failures))
return vdisk_guids
def migrate(self, volume_id, node_id, force_restart, req_timeout_secs=None):
"""
Dummy migrate method
"""
_ = force_restart, req_timeout_secs
from ovs.dal.lists.storagedriverlist import StorageDriverList
storagedriver = StorageDriverList.get_by_storagedriver_id(node_id)
if storagedriver is None:
raise ValueError("Failed to retrieve storagedriver with ID {0}".format(node_id))
StorageRouterClient.vrouter_id[self.vpool_guid][volume_id] = node_id
def mountpoint_available_from_voldrv(mountpoint, storagedriver_id):
"""
Hook for (re)exporting the NFS mountpoint
"""
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
if storagedriver is None:
raise RuntimeError('A Storage Driver with id {0} could not be found.'.format(storagedriver_id))
if storagedriver.storagerouter.pmachine.hvtype == 'VMWARE':
nfs = Nfsexports()
nfs.unexport(mountpoint)
nfs.export(mountpoint)
nfs.trigger_rpc_mountd()
def get_storagedrivers_by_storagerouterguid(storagerouter_guid):
"""
Get the storagedriver connected to a storagerouter by its guid
:param storagerouter_guid: guid of a storagerouter
:type storagerouter_guid: str
:return: collection of available storagedrivers on the storagerouter
:rtype: list
"""
return StorageDriverList.get_storagedrivers_by_storagerouter(
storagerouter_guid)
def _log(task, kwargs, storagedriver_id):
"""
Log an event
"""
metadata = {'storagedriver': StorageDriverList.get_by_storagedriver_id(storagedriver_id).guid}
_logger = LogHandler.get('log', name='volumedriver_event')
_logger.info('[{0}.{1}] - {2} - {3}'.format(
task.__class__.__module__,
task.__class__.__name__,
json.dumps(kwargs),
json.dumps(metadata)
))
def _execute_scrub_work(scrub_location, vdisk_guids):
def verify_mds_config(current_vdisk):
current_vdisk.invalidate_dynamics(["info"])
vdisk_configs = current_vdisk.info["metadata_backend_config"]
if len(vdisk_configs) == 0:
raise RuntimeError("Could not load MDS configuration")
return vdisk_configs
logger.info("Scrub location: {0}".format(scrub_location))
total = len(vdisk_guids)
skipped = 0
storagedrivers = {}
failures = []
for vdisk_guid in vdisk_guids:
vdisk = VDisk(vdisk_guid)
try:
# Load the vDisk's StorageDriver
logger.info("Scrubbing virtual disk {0} with guid {1}".format(vdisk.name, vdisk.guid))
vdisk.invalidate_dynamics(["storagedriver_id"])
if vdisk.storagedriver_id not in storagedrivers:
storagedrivers[vdisk.storagedriver_id] = StorageDriverList.get_by_storagedriver_id(
vdisk.storagedriver_id
)
storagedriver = storagedrivers[vdisk.storagedriver_id]
# Load the vDisk's MDS configuration
configs = verify_mds_config(current_vdisk=vdisk)
# Check MDS master is local. Trigger MDS handover if necessary
if configs[0].get("ip") != storagedriver.storagerouter.ip:
logger.debug("MDS for volume {0} is not local. Trigger handover".format(vdisk.volume_id))
MDSServiceController.ensure_safety(vdisk)
configs = verify_mds_config(current_vdisk=vdisk)
if configs[0].get("ip") != storagedriver.storagerouter.ip:
skipped += 1
logger.info(
"Skipping scrubbing work unit for volume {0}: MDS master is not local".format(
vdisk.volume_id
)
)
continue
with vdisk.storagedriver_client.make_locked_client(str(vdisk.volume_id)) as locked_client:
work_units = locked_client.get_scrubbing_workunits()
for work_unit in work_units:
scrubbing_result = locked_client.scrub(work_unit, scrub_location)
locked_client.apply_scrubbing_result(scrubbing_result)
if work_units:
logger.info("Scrubbing successfully applied")
except Exception, ex:
failures.append(
"Failed scrubbing work unit for volume {0} with guid {1}: {2}".format(vdisk.name, vdisk.guid, ex)
)
def create(self):
"""
Prepares a new Storagedriver for a given vPool and Storagerouter
:return: None
:rtype: NoneType
"""
if self.sr_installer is None:
raise RuntimeError('No StorageRouterInstaller instance found')
machine_id = System.get_my_machine_id(client=self.sr_installer.root_client)
port_range = Configuration.get('/ovs/framework/hosts/{0}/ports|storagedriver'.format(machine_id))
storagerouter = self.sr_installer.storagerouter
with volatile_mutex('add_vpool_get_free_ports_{0}'.format(machine_id), wait=30):
model_ports_in_use = []
for sd in StorageDriverList.get_storagedrivers():
if sd.storagerouter_guid == storagerouter.guid:
model_ports_in_use += sd.ports.values()
for proxy in sd.alba_proxies:
model_ports_in_use.append(proxy.service.ports[0])
ports = System.get_free_ports(selected_range=port_range, exclude=model_ports_in_use, amount=4 + self.sr_installer.requested_proxies, client=self.sr_installer.root_client)
vpool = self.vp_installer.vpool
vrouter_id = '{0}{1}'.format(vpool.name, machine_id)
storagedriver = StorageDriver()
storagedriver.name = vrouter_id.replace('_', ' ')
storagedriver.ports = {'management': ports[0],
'xmlrpc': ports[1],
'dtl': ports[2],
'edge': ports[3]}
storagedriver.vpool = vpool
storagedriver.cluster_ip = Configuration.get('/ovs/framework/hosts/{0}/ip'.format(machine_id))
storagedriver.storage_ip = self.storage_ip
storagedriver.mountpoint = '/mnt/{0}'.format(vpool.name)
storagedriver.description = storagedriver.name
storagedriver.storagerouter = storagerouter
storagedriver.storagedriver_id = vrouter_id
storagedriver.save()
# ALBA Proxies
proxy_service_type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ALBA_PROXY)
for proxy_id in xrange(self.sr_installer.requested_proxies):
service = Service()
service.storagerouter = storagerouter
service.ports = [ports[4 + proxy_id]]
service.name = 'albaproxy_{0}_{1}'.format(vpool.name, proxy_id)
service.type = proxy_service_type
service.save()
alba_proxy = AlbaProxy()
alba_proxy.service = service
alba_proxy.storagedriver = storagedriver
alba_proxy.save()
self.storagedriver = storagedriver
def _log(task, kwargs, storagedriver_id):
"""
Log an event
"""
metadata = {
'storagedriver':
StorageDriverList.get_by_storagedriver_id(storagedriver_id).guid
}
_logger = LogHandler.get('log', name='volumedriver_event')
_logger.info('[{0}.{1}] - {2} - {3}'.format(task.__class__.__module__,
task.__class__.__name__,
json.dumps(kwargs),
json.dumps(metadata)))
def sync_with_hypervisor(vmachineguid, storagedriver_id=None):
"""
Updates a given vmachine with data retreived from a given pmachine
"""
try:
vmachine = VMachine(vmachineguid)
if storagedriver_id is None and vmachine.hypervisor_id is not None and vmachine.pmachine is not None:
# Only the vmachine was received, so base the sync on hypervisorid and pmachine
hypervisor = Factory.get(vmachine.pmachine)
logger.info('Syncing vMachine (name {})'.format(vmachine.name))
vm_object = hypervisor.get_vm_agnostic_object(
vmid=vmachine.hypervisor_id)
elif storagedriver_id is not None and vmachine.devicename is not None:
# Storage Driver id was given, using the devicename instead (to allow hypervisorid updates
# which can be caused by re-adding a vm to the inventory)
pmachine = PMachineList.get_by_storagedriver_id(
storagedriver_id)
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
hypervisor = Factory.get(pmachine)
if not hypervisor.file_exists(
vmachine.vpool,
hypervisor.clean_vmachine_filename(
vmachine.devicename)):
return
vmachine.pmachine = pmachine
vmachine.save()
logger.info(
'Syncing vMachine (device {}, ip {}, mtpt {})'.format(
vmachine.devicename, storagedriver.storage_ip,
storagedriver.mountpoint))
vm_object = hypervisor.get_vm_object_by_devicename(
devicename=vmachine.devicename,
ip=storagedriver.storage_ip,
mountpoint=storagedriver.mountpoint)
else:
message = 'Not enough information to sync vmachine'
logger.info('Error: {0}'.format(message))
raise RuntimeError(message)
except Exception as ex:
logger.info('Error while fetching vMachine info: {0}'.format(
str(ex)))
raise
if vm_object is None:
message = 'Could not retreive hypervisor vmachine object'
logger.info('Error: {0}'.format(message))
raise RuntimeError(message)
else:
VMachineController.update_vmachine_config(vmachine, vm_object)
def get_by_storagedriver_id(storagedriver_id):
"""
Get pMachine that hosts a given storagedriver_id
"""
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
if storagedriver is None:
raise RuntimeError('StorageDriver {0} could not be found'.format(storagedriver_id))
storagerouter = storagedriver.storagerouter
if storagerouter is None:
raise RuntimeError('StorageDriver {0} not linked to a StorageRouter'.format(storagedriver.name))
pmachine = storagerouter.pmachine
if pmachine is None:
raise RuntimeError('StorageRouter {0} not linked to a pMachine'.format(storagerouter.name))
return pmachine
def get_by_storagedriver_id(storagedriver_id):
"""
Get pMachine that hosts a given storagedriver_id
"""
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
if storagedriver is None:
raise RuntimeError('StorageDriver {0} could not be found'.format(storagedriver_id))
storagerouter = storagedriver.storagerouter
if storagerouter is None:
raise RuntimeError('StorageDriver {0} not linked to a StorageRouter'.format(storagedriver.name))
pmachine = storagerouter.pmachine
if pmachine is None:
raise RuntimeError('StorageRouter {0} not linked to a pMachine'.format(storagerouter.name))
return pmachine
def get_storagedrivers_in_same_domain(domain_guid):
"""
Get storagerouter guids in a domain
:param domain_guid: guid of a domain
:type domain_guid: str
:return: list of storagerouter guids
:rtype: list
"""
return [
storagedriver
for storagedriver in StorageDriverList.get_storagedrivers()
if domain_guid in storagedriver.storagerouter.regular_domains
]
def delete_snapshots(timestamp=None):
# type: (float) -> GroupResult
"""
Delete snapshots based on the retention policy
Offloads concurrency to celery
Returns a GroupResult. Waiting for the result can be done using result.get()
:param timestamp: Timestamp to determine whether snapshots should be kept or not, if none provided, current time will be used
:type timestamp: float
:return: The GroupResult
:rtype: GroupResult
"""
# The result cannot be fetched in this task
group_id = uuid()
return group(GenericController.delete_snapshots_storagedriver.s(storagedriver.guid, timestamp, group_id)
for storagedriver in StorageDriverList.get_storagedrivers()).apply_async(task_id=group_id)
def mountpoint_available_from_voldrv(mountpoint, storagedriver_id):
"""
Hook for (re)exporting the NFS mountpoint
"""
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
if storagedriver is None:
raise RuntimeError(
'A Storage Driver with id {0} could not be found.'.format(
storagedriver_id))
if storagedriver.storagerouter.pmachine.hvtype == 'VMWARE':
nfs = Nfsexports()
nfs.unexport(mountpoint)
nfs.export(mountpoint)
nfs.trigger_rpc_mountd()
def create_volume(self, target_path, metadata_backend_config, volume_size, node_id):
"""
Create a mocked volume
"""
from ovs.dal.lists.storagedriverlist import StorageDriverList
volume_id = str(uuid.uuid4())
storagedriver = StorageDriverList.get_by_storagedriver_id(node_id)
if storagedriver is None:
raise ValueError('Failed to retrieve storagedriver with ID {0}'.format(node_id))
StorageRouterClient.vrouter_id[self.vpool_guid][volume_id] = node_id
StorageRouterClient._metadata_backend_config[self.vpool_guid][volume_id] = metadata_backend_config
StorageRouterClient.volumes[self.vpool_guid][volume_id] = {'volume_id': volume_id,
'volume_size': volume_size,
'target_path': target_path}
return volume_id
def up_and_running(mountpoint, storagedriver_id):
"""
Volumedriver informs us that the service is completely started. Post-start events can be executed
"""
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
if storagedriver is None:
raise RuntimeError('A Storage Driver with id {0} could not be found.'.format(storagedriver_id))
storagedriver.startup_counter += 1
storagedriver.save()
if storagedriver.storagerouter.pmachine.hvtype == 'VMWARE':
client = SSHClient(storagedriver.storagerouter)
if client.config_read('ovs.storagedriver.vmware_mode') == 'classic':
nfs = Nfsexports()
nfs.unexport(mountpoint)
nfs.export(mountpoint)
nfs.trigger_rpc_mountd()
def dtl_state_transition(volume_name, old_state, new_state, storagedriver_id):
"""
Triggered by volumedriver when DTL state changes
:param volume_name: ID of the volume
:param old_state: Previous DTL status
:param new_state: New DTL status
:param storagedriver_id: ID of the storagedriver hosting the volume
:return: None
"""
if new_state == VolumeDriverEvents_pb2.Degraded and old_state != VolumeDriverEvents_pb2.Standalone:
vdisk = VDiskList.get_vdisk_by_volume_id(volume_name)
if vdisk:
logger.info('Degraded DTL detected for volume {0} with guid {1}'.format(vdisk.name, vdisk.guid))
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
VDiskController.dtl_checkup(vdisk_guid=vdisk.guid,
storagerouters_to_exclude=[storagedriver.storagerouter.guid],
chain_timeout=600)
def rename_from_voldrv(old_name, new_name, storagedriver_id):
"""
This machine will handle the rename of a vmx file
:param old_name: Old name of vmx
:param new_name: New name for the vmx
:param storagedriver_id: Storage Driver hosting the vmachine
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
if pmachine.hvtype not in ['VMWARE', 'KVM']:
return
hypervisor = Factory.get(pmachine)
if pmachine.hvtype == 'VMWARE':
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
vpool = storagedriver.vpool
else:
vpool = None
old_name = hypervisor.clean_vmachine_filename(old_name)
new_name = hypervisor.clean_vmachine_filename(new_name)
scenario = hypervisor.get_rename_scenario(old_name, new_name)
if scenario == 'RENAME':
# Most likely a change from path. Updating path
vm = VMachineList.get_by_devicename_and_vpool(old_name, vpool)
if vm is not None:
vm.devicename = new_name
vm.save()
elif scenario == 'UPDATE':
vm = VMachineList.get_by_devicename_and_vpool(new_name, vpool)
if vm is None:
# The vMachine doesn't seem to exist, so it's likely the create didn't came trough
# Let's create it anyway
VMachineController.update_from_voldrv(
new_name, storagedriver_id=storagedriver_id)
vm = VMachineList.get_by_devicename_and_vpool(new_name, vpool)
if vm is None:
raise RuntimeError(
'Could not create vMachine on rename. Aborting.')
try:
VMachineController.sync_with_hypervisor(
vm.guid, storagedriver_id=storagedriver_id)
vm.status = 'SYNC'
except:
vm.status = 'SYNC_NOK'
vm.save()
def update_status(storagedriver_id):
"""
Sets Storage Driver offline in case hypervisor management Center
reports the hypervisor pmachine related to this Storage Driver
as unavailable.
:param storagedriver_id: ID of the storagedriver to update its status
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
storagerouter = storagedriver.storagerouter
if pmachine.mgmtcenter:
# Update status
pmachine.invalidate_dynamics(['host_status'])
else:
# No management Center, cannot update status via api
logger.info(
'Updating status of pmachine {0} using SSHClient'.format(
pmachine.name))
host_status = 'RUNNING'
try:
client = SSHClient(storagerouter, username='******')
configuration_dir = EtcdConfiguration.get(
'/ovs/framework/paths|cfgdir')
logger.info(
'SSHClient connected successfully to {0} at {1}'.format(
pmachine.name, client.ip))
with Remote(client.ip, [LocalStorageRouterClient]) as remote:
lsrc = remote.LocalStorageRouterClient(
'{0}/storagedriver/storagedriver/{1}.json'.format(
configuration_dir, storagedriver.vpool.name))
lsrc.server_revision()
logger.info(
'LocalStorageRouterClient connected successfully to {0} at {1}'
.format(pmachine.name, client.ip))
except Exception as ex:
logger.error(
'Connectivity check failed, assuming host {0} is halted. {1}'
.format(pmachine.name, ex))
host_status = 'HALTED'
if host_status != 'RUNNING':
# Host is stopped
storagedriver_client = StorageDriverClient.load(
storagedriver.vpool)
storagedriver_client.mark_node_offline(
str(storagedriver.storagedriver_id))
def clone(diskguid, snapshotid, devicename, pmachineguid, machinename, machineguid=None):
"""
Clone a disk
"""
pmachine = PMachine(pmachineguid)
hypervisor = Factory.get(pmachine)
description = "{} {}".format(machinename, devicename)
properties_to_clone = ["description", "size", "type", "retentionpolicyguid", "snapshotpolicyguid", "autobackup"]
vdisk = VDisk(diskguid)
location = hypervisor.get_backing_disk_path(machinename, devicename)
new_vdisk = VDisk()
new_vdisk.copy(vdisk, include=properties_to_clone)
new_vdisk.parent_vdisk = vdisk
new_vdisk.name = "{0}-clone".format(vdisk.name)
new_vdisk.description = description
new_vdisk.devicename = hypervisor.clean_backing_disk_filename(location)
new_vdisk.parentsnapshot = snapshotid
new_vdisk.vmachine = VMachine(machineguid) if machineguid else vdisk.vmachine
new_vdisk.vpool = vdisk.vpool
new_vdisk.save()
storagedriver = StorageDriverList.get_by_storagedriver_id(vdisk.storagedriver_id)
if storagedriver is None:
raise RuntimeError("Could not find StorageDriver with id {0}".format(vdisk.storagedriver_id))
mds_service = MDSServiceController.get_preferred_mds(storagedriver.storagerouter, vdisk.vpool)
if mds_service is None:
raise RuntimeError("Could not find a MDS service")
logger.info("Clone snapshot {} of disk {} to location {}".format(snapshotid, vdisk.name, location))
volume_id = vdisk.storagedriver_client.create_clone(
target_path=location,
metadata_backend_config=MDSMetaDataBackendConfig(
[MDSNodeConfig(address=str(mds_service.service.storagerouter.ip), port=mds_service.service.ports[0])]
),
parent_volume_id=str(vdisk.volume_id),
parent_snapshot_id=str(snapshotid),
node_id=str(vdisk.storagedriver_id),
)
new_vdisk.volume_id = volume_id
new_vdisk.save()
MDSServiceController.ensure_safety(new_vdisk)
return {"diskguid": new_vdisk.guid, "name": new_vdisk.name, "backingdevice": location}
def create_volume(self, target_path, metadata_backend_config, volume_size, node_id, req_timeout_secs=None):
"""
Create a mocked volume
"""
_ = req_timeout_secs
from ovs.dal.lists.storagedriverlist import StorageDriverList
volume_id = str(uuid.uuid4())
storagedriver = StorageDriverList.get_by_storagedriver_id(node_id)
if storagedriver is None:
raise ValueError("Failed to retrieve storagedriver with ID {0}".format(node_id))
StorageRouterClient.vrouter_id[self.vpool_guid][volume_id] = node_id
StorageRouterClient._metadata_backend_config[self.vpool_guid][volume_id] = metadata_backend_config
StorageRouterClient.volumes[self.vpool_guid][volume_id] = {
"volume_id": volume_id,
"volume_size": volume_size,
"target_path": target_path,
}
return volume_id
def delete_from_voldrv(volumename, storagedriver_id):
"""
Delete a disk
Triggered by volumedriver messages on the queue
@param volumename: volume id of the disk
"""
_ = storagedriver_id # For logging purposes
disk = VDiskList.get_vdisk_by_volume_id(volumename)
if disk is not None:
mutex = VolatileMutex('{}_{}'.format(volumename, disk.devicename))
try:
mutex.acquire(wait=20)
pmachine = None
try:
pmachine = PMachineList.get_by_storagedriver_id(
disk.storagedriver_id)
except RuntimeError as ex:
if 'could not be found' not in str(ex):
raise
# else: pmachine can't be loaded, because the volumedriver doesn't know about it anymore
if pmachine is not None:
limit = 5
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
hypervisor = Factory.get(pmachine)
exists = hypervisor.file_exists(storagedriver,
disk.devicename)
while limit > 0 and exists is True:
time.sleep(1)
exists = hypervisor.file_exists(
storagedriver, disk.devicename)
limit -= 1
if exists is True:
logger.info(
'Disk {0} still exists, ignoring delete'.format(
disk.devicename))
return
logger.info('Delete disk {}'.format(disk.name))
for mds_service in disk.mds_services:
mds_service.delete()
disk.delete()
finally:
mutex.release()
def up_and_running(mountpoint, storagedriver_id):
"""
Volumedriver informs us that the service is completely started. Post-start events can be executed
:param mountpoint: Mountpoint to check
:param storagedriver_id: ID of the storagedriver
"""
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
if storagedriver is None:
raise RuntimeError('A Storage Driver with id {0} could not be found.'.format(storagedriver_id))
storagedriver.startup_counter += 1
storagedriver.save()
if storagedriver.storagerouter.pmachine.hvtype == 'VMWARE':
client = SSHClient(storagedriver.storagerouter)
machine_id = System.get_my_machine_id(client)
if EtcdConfiguration.get('/ovs/framework/hosts/{0}/storagedriver|vmware_mode'.format(machine_id)) == 'classic':
nfs = Nfsexports()
nfs.unexport(mountpoint)
nfs.export(mountpoint)
nfs.trigger_rpc_mountd()
def up_and_running(mountpoint, storagedriver_id):
"""
Volumedriver informs us that the service is completely started. Post-start events can be executed
:param mountpoint: Mountpoint to check
:param storagedriver_id: ID of the storagedriver
"""
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
if storagedriver is None:
raise RuntimeError('A Storage Driver with id {0} could not be found.'.format(storagedriver_id))
storagedriver.startup_counter += 1
storagedriver.save()
if storagedriver.storagerouter.pmachine.hvtype == 'VMWARE':
client = SSHClient(storagedriver.storagerouter)
machine_id = System.get_my_machine_id(client)
if EtcdConfiguration.get('/ovs/framework/hosts/{0}/storagedriver|vmware_mode'.format(machine_id)) == 'classic':
nfs = Nfsexports()
nfs.unexport(mountpoint)
nfs.export(mountpoint)
nfs.trigger_rpc_mountd()
def rename_from_voldrv(old_name, new_name, storagedriver_id):
"""
This machine will handle the rename of a vmx file
:param old_name: Old name of vmx
:param new_name: New name for the vmx
:param storagedriver_id: Storage Driver hosting the vmachine
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
if pmachine.hvtype not in ['VMWARE', 'KVM']:
return
hypervisor = Factory.get(pmachine)
if pmachine.hvtype == 'VMWARE':
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
vpool = storagedriver.vpool
else:
vpool = None
old_name = hypervisor.clean_vmachine_filename(old_name)
new_name = hypervisor.clean_vmachine_filename(new_name)
scenario = hypervisor.get_rename_scenario(old_name, new_name)
if scenario == 'RENAME':
# Most likely a change from path. Updating path
vm = VMachineList.get_by_devicename_and_vpool(old_name, vpool)
if vm is not None:
vm.devicename = new_name
vm.save()
elif scenario == 'UPDATE':
vm = VMachineList.get_by_devicename_and_vpool(new_name, vpool)
if vm is None:
# The vMachine doesn't seem to exist, so it's likely the create didn't came trough
# Let's create it anyway
VMachineController.update_from_voldrv(new_name, storagedriver_id=storagedriver_id)
vm = VMachineList.get_by_devicename_and_vpool(new_name, vpool)
if vm is None:
raise RuntimeError('Could not create vMachine on rename. Aborting.')
try:
VMachineController.sync_with_hypervisor(vm.guid, storagedriver_id=storagedriver_id)
vm.status = 'SYNC'
except:
vm.status = 'SYNC_NOK'
vm.save()
def sync_with_hypervisor(vmachineguid, storagedriver_id=None):
"""
Updates a given vmachine with data retreived from a given pmachine
"""
try:
vmachine = VMachine(vmachineguid)
if storagedriver_id is None and vmachine.hypervisor_id is not None and vmachine.pmachine is not None:
# Only the vmachine was received, so base the sync on hypervisorid and pmachine
hypervisor = Factory.get(vmachine.pmachine)
logger.info('Syncing vMachine (name {})'.format(vmachine.name))
vm_object = hypervisor.get_vm_agnostic_object(vmid=vmachine.hypervisor_id)
elif storagedriver_id is not None and vmachine.devicename is not None:
# Storage Driver id was given, using the devicename instead (to allow hypervisorid updates
# which can be caused by re-adding a vm to the inventory)
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
hypervisor = Factory.get(pmachine)
if not hypervisor.file_exists(vmachine.vpool, hypervisor.clean_vmachine_filename(vmachine.devicename)):
return
vmachine.pmachine = pmachine
vmachine.save()
logger.info('Syncing vMachine (device {}, ip {}, mtpt {})'.format(vmachine.devicename,
storagedriver.storage_ip,
storagedriver.mountpoint))
vm_object = hypervisor.get_vm_object_by_devicename(devicename=vmachine.devicename,
ip=storagedriver.storage_ip,
mountpoint=storagedriver.mountpoint)
else:
message = 'Not enough information to sync vmachine'
logger.info('Error: {0}'.format(message))
raise RuntimeError(message)
except Exception as ex:
logger.info('Error while fetching vMachine info: {0}'.format(str(ex)))
raise
if vm_object is None:
message = 'Could not retreive hypervisor vmachine object'
logger.info('Error: {0}'.format(message))
raise RuntimeError(message)
else:
VMachineController.update_vmachine_config(vmachine, vm_object)
def new_function(*args, **kwargs):
"""
Wrapped function
"""
# Log the call
if event_type == 'VOLUMEDRIVER_TASK':
metadata = {'storagedriver': StorageDriverList.get_by_storagedriver_id(kwargs['storagedriver_id']).guid}
else:
metadata = {}
_logger = LogHandler.get('log', name=event_type.lower())
_logger.info('[{0}.{1}] - {2} - {3} - {4}'.format(
f.__module__,
f.__name__,
json.dumps(list(args)),
json.dumps(kwargs),
json.dumps(metadata)
))
# Call the function
return f(*args, **kwargs)
def new_function(*args, **kwargs):
"""
Wrapped function
"""
# Log the call
if event_type == 'VOLUMEDRIVER_TASK':
metadata = {
'storagedriver':
StorageDriverList.get_by_storagedriver_id(
kwargs['storagedriver_id']).guid
}
else:
metadata = {}
_logger = LogHandler.get('log', name=event_type.lower())
_logger.info('[{0}.{1}] - {2} - {3} - {4}'.format(
f.__module__, f.__name__, json.dumps(list(args)),
json.dumps(kwargs), json.dumps(metadata)))
# Call the function
return f(*args, **kwargs)
def delete_from_voldrv(name, storagedriver_id):
"""
This method will delete a vmachine based on the name of the vmx given
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
if pmachine.hvtype not in ['VMWARE', 'KVM']:
return
hypervisor = Factory.get(pmachine)
name = hypervisor.clean_vmachine_filename(name)
if pmachine.hvtype == 'VMWARE':
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
vpool = storagedriver.vpool
else:
vpool = None
vm = VMachineList.get_by_devicename_and_vpool(name, vpool)
if vm is not None:
MessageController.fire(MessageController.Type.EVENT, {'type': 'vmachine_deleted',
'metadata': {'name': vm.name}})
vm.delete(abandon=['vdisks'])
def update_status(storagedriver_id):
"""
Sets Storage Driver offline in case hypervisor management Center
reports the hypervisor pmachine related to this Storage Driver
as unavailable.
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
if pmachine.mgmtcenter:
# Update status
pmachine.invalidate_dynamics(['host_status'])
host_status = pmachine.host_status
if host_status != 'RUNNING':
# Host is stopped
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
storagedriver_client = StorageDriverClient().load(storagedriver.vpool)
storagedriver_client.mark_node_offline(str(storagedriver.storagedriver_id))
else:
# No management Center, cannot update status via api
#TODO: should we try manually (ping, ssh)?
pass
def migrate_from_voldrv(volume_id, new_owner_id):
"""
Triggered when volume has changed owner (Clean migration or stolen due to other reason)
Triggered by volumedriver messages
:param volume_id: Volume ID of the disk
:type volume_id: unicode
:param new_owner_id: ID of the storage driver the volume migrated to
:type new_owner_id: unicode
:returns: None
"""
sd = StorageDriverList.get_by_storagedriver_id(storagedriver_id=new_owner_id)
vdisk = VDiskList.get_vdisk_by_volume_id(volume_id=volume_id)
if vdisk is not None:
logger.info('Migration - Guid {0} - ID {1} - Detected migration for virtual disk {2}'.format(vdisk.guid, vdisk.volume_id, vdisk.name))
if sd is not None:
logger.info('Migration - Guid {0} - ID {1} - Storage Router {2} is the new owner of virtual disk {3}'.format(vdisk.guid, vdisk.volume_id, sd.storagerouter.name, vdisk.name))
MDSServiceController.mds_checkup()
VDiskController.dtl_checkup(vdisk_guid=vdisk.guid)
def clean_bad_disk(vdiskguid):
"""
Cleanup bad vdisk:
- in case create_from_template failed
- remove mds_services so the vdisk can be properly cleaned up
:param vdiskguid: guid of vdisk
:return: None
"""
vdisk = VDisk(vdiskguid)
logger.info('Cleanup vdisk {0}'.format(vdisk.name))
for mdss in vdisk.mds_services:
mdss.delete()
storagedriver = StorageDriverList.get_by_storagedriver_id(vdisk.storagedriver_id)
if storagedriver is not None and vdisk.devicename is not None:
logger.debug('Removing volume from filesystem')
volumepath = vdisk.devicename
mountpoint = storagedriver.mountpoint
devicepath = '{0}/{1}'.format(mountpoint, volumepath)
VDiskController.delete_volume(devicepath)
logger.debug('Deleting vdisk {0} from model'.format(vdisk.name))
vdisk.delete()
def new_function(*args, **kwargs):
"""
Wrapped function
:param args: Arguments without default values
:param kwargs: Arguments with default values
"""
# Log the call
if event_type == 'VOLUMEDRIVER_TASK' and 'storagedriver_id' in kwargs:
metadata = {
'storagedriver':
StorageDriverList.get_by_storagedriver_id(
kwargs['storagedriver_id']).guid
}
else:
metadata = {}
_logger = Logger(event_type.lower())
_logger.info('[{0}.{1}] - {2} - {3} - {4}'.format(
f.__module__, f.__name__, json.dumps(list(args)),
json.dumps(kwargs), json.dumps(metadata)))
# Call the function
return f(*args, **kwargs)
def new_function(*args, **kwargs):
"""
Wrapped function
"""
# Log the call
log_entry = Log()
log_entry.source = event_type
log_entry.module = f.__module__
log_entry.method = f.__name__
log_entry.method_args = list(args)
log_entry.method_kwargs = kwargs
log_entry.time = time.time()
if event_type == 'VOLUMEDRIVER_TASK':
try:
log_entry.storagedriver = StorageDriverList.get_by_storagedriver_id(kwargs['storagedriver_id'])
log_entry.save()
except ObjectNotFoundException:
pass
else:
log_entry.save()
# Call the function
return f(*args, **kwargs)
def update_status(storagedriver_id):
"""
Sets Storage Driver offline in case hypervisor management Center
reports the hypervisor pmachine related to this Storage Driver
as unavailable.
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
if pmachine.mgmtcenter:
# Update status
pmachine.invalidate_dynamics(['host_status'])
host_status = pmachine.host_status
if host_status != 'RUNNING':
# Host is stopped
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
storagedriver_client = StorageDriverClient.load(
storagedriver.vpool)
storagedriver_client.mark_node_offline(
str(storagedriver.storagedriver_id))
else:
# No management Center, cannot update status via api
# @TODO: should we try manually (ping, ssh)?
pass
def new_function(*args, **kwargs):
"""
Wrapped function
"""
# Log the call
log_entry = Log()
log_entry.source = event_type
log_entry.module = f.__module__
log_entry.method = f.__name__
log_entry.method_args = list(args)
log_entry.method_kwargs = kwargs
log_entry.time = time.time()
if event_type == 'VOLUMEDRIVER_TASK':
try:
log_entry.storagedriver = StorageDriverList.get_by_storagedriver_id(kwargs['storagedriver_id'])
log_entry.save()
except ObjectNotFoundException:
pass
else:
log_entry.save()
# Call the function
return f(*args, **kwargs)
def delete_from_voldrv(volumename, storagedriver_id):
"""
Delete a disk
Triggered by volumedriver messages on the queue
@param volumename: volume id of the disk
"""
_ = storagedriver_id # For logging purposes
disk = VDiskList.get_vdisk_by_volume_id(volumename)
if disk is not None:
mutex = VolatileMutex('{}_{}'.format(volumename, disk.devicename))
try:
mutex.acquire(wait=20)
pmachine = None
try:
pmachine = PMachineList.get_by_storagedriver_id(disk.storagedriver_id)
except RuntimeError as ex:
if 'could not be found' not in str(ex):
raise
# else: pmachine can't be loaded, because the volumedriver doesn't know about it anymore
if pmachine is not None:
limit = 5
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
hypervisor = Factory.get(pmachine)
exists = hypervisor.file_exists(storagedriver, disk.devicename)
while limit > 0 and exists is True:
time.sleep(1)
exists = hypervisor.file_exists(storagedriver, disk.devicename)
limit -= 1
if exists is True:
logger.info('Disk {0} still exists, ignoring delete'.format(disk.devicename))
return
logger.info('Delete disk {}'.format(disk.name))
for mds_service in disk.mds_services:
mds_service.delete()
disk.delete()
finally:
mutex.release()
def migrate():
"""
Executes async migrations. It doesn't matter too much when they are executed, as long as they get eventually
executed. This code will typically contain:
* "dangerous" migration code (it needs certain running services)
* Migration code depending on a cluster-wide state
* ...
* Successfully finishing a piece of migration code, should create an entry in /ovs/framework/migration in case it should not be executed again
* Eg: /ovs/framework/migration|stats_monkey_integration: True
"""
MigrationController._logger.info('Preparing out of band migrations...')
from ovs.dal.lists.servicetypelist import ServiceTypeList
from ovs.dal.lists.storagedriverlist import StorageDriverList
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.dal.lists.vpoollist import VPoolList
from ovs.extensions.db.arakooninstaller import ArakoonInstaller
from ovs.extensions.generic.configuration import Configuration
from ovs.extensions.generic.sshclient import SSHClient
from ovs_extensions.generic.toolbox import ExtensionsToolbox
from ovs.extensions.migration.migration.ovsmigrator import ExtensionMigrator
from ovs.extensions.packages.packagefactory import PackageFactory
from ovs_extensions.services.interfaces.systemd import Systemd
from ovs.extensions.services.servicefactory import ServiceFactory
from ovs.extensions.storageserver.storagedriver import StorageDriverConfiguration
from ovs.lib.helpers.storagedriver.installer import StorageDriverInstaller
MigrationController._logger.info('Start out of band migrations...')
service_manager = ServiceFactory.get_manager()
sr_client_map = {}
for storagerouter in StorageRouterList.get_storagerouters():
sr_client_map[storagerouter.guid] = SSHClient(endpoint=storagerouter.ip, # Is triggered during post-update code too during which the ovs-watcher-framework service is still down and thus not refreshing the heartbeat --> use IP i/o StorageRouter
username='******')
#########################################################
# Addition of 'ExecReload' for AlbaProxy SystemD services
if ServiceFactory.get_service_type() == 'systemd':
changed_clients = set()
for storagedriver in StorageDriverList.get_storagedrivers():
root_client = sr_client_map[storagedriver.storagerouter_guid]
for alba_proxy in storagedriver.alba_proxies:
service = alba_proxy.service
service_name = 'ovs-{0}'.format(service.name)
if not service_manager.has_service(name=service_name, client=root_client):
continue
if 'ExecReload=' in root_client.file_read(filename='/lib/systemd/system/{0}.service'.format(service_name)):
continue
try:
service_manager.regenerate_service(name=StorageDriverInstaller.SERVICE_TEMPLATE_PROXY, client=root_client, target_name=service_name)
changed_clients.add(root_client)
except:
MigrationController._logger.exception('Error rebuilding service {0}'.format(service_name))
for root_client in changed_clients:
root_client.run(['systemctl', 'daemon-reload'])
##################################################################
# Adjustment of open file descriptors for Arakoon services to 8192
changed_clients = set()
for storagerouter in StorageRouterList.get_storagerouters():
root_client = sr_client_map[storagerouter.guid]
for service_name in service_manager.list_services(client=root_client):
if not service_name.startswith('ovs-arakoon-'):
continue
if ServiceFactory.get_service_type() == 'systemd':
path = '/lib/systemd/system/{0}.service'.format(service_name)
check = 'LimitNOFILE=8192'
else:
path = '/etc/init/{0}.conf'.format(service_name)
check = 'limit nofile 8192 8192'
if not root_client.file_exists(path):
continue
if check in root_client.file_read(path):
continue
try:
service_manager.regenerate_service(name='ovs-arakoon', client=root_client, target_name=service_name)
changed_clients.add(root_client)
ExtensionsToolbox.edit_version_file(client=root_client,
package_name='arakoon',
old_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, service_name))
except:
MigrationController._logger.exception('Error rebuilding service {0}'.format(service_name))
for root_client in changed_clients:
root_client.run(['systemctl', 'daemon-reload'])
#############################
# Migrate to multiple proxies
for storagedriver in StorageDriverList.get_storagedrivers():
vpool = storagedriver.vpool
root_client = sr_client_map[storagedriver.storagerouter_guid]
for alba_proxy in storagedriver.alba_proxies:
# Rename alba_proxy service in model
service = alba_proxy.service
old_service_name = 'albaproxy_{0}'.format(vpool.name)
new_service_name = 'albaproxy_{0}_0'.format(vpool.name)
if old_service_name != service.name:
continue
service.name = new_service_name
service.save()
if not service_manager.has_service(name=old_service_name, client=root_client):
continue
old_configuration_key = '/ovs/framework/hosts/{0}/services/{1}'.format(storagedriver.storagerouter.machine_id, old_service_name)
if not Configuration.exists(key=old_configuration_key):
continue
# Add '-reboot' to alba_proxy services (because of newly created services and removal of old service)
ExtensionsToolbox.edit_version_file(client=root_client,
package_name='alba',
old_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, old_service_name),
new_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, new_service_name))
# Register new service and remove old service
service_manager.add_service(name=StorageDriverInstaller.SERVICE_TEMPLATE_PROXY,
client=root_client,
params=Configuration.get(old_configuration_key),
target_name='ovs-{0}'.format(new_service_name))
# Update scrub proxy config
proxy_config_key = '/ovs/vpools/{0}/proxies/{1}/config/main'.format(vpool.guid, alba_proxy.guid)
proxy_config = None if Configuration.exists(key=proxy_config_key) is False else Configuration.get(proxy_config_key)
if proxy_config is not None:
fragment_cache = proxy_config.get(StorageDriverConfiguration.CACHE_FRAGMENT, ['none', {}])
if fragment_cache[0] == 'alba' and fragment_cache[1].get('cache_on_write') is True: # Accelerated ALBA configured
fragment_cache_scrub_info = copy.deepcopy(fragment_cache)
fragment_cache_scrub_info[1]['cache_on_read'] = False
proxy_scrub_config_key = '/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(vpool.guid)
proxy_scrub_config = None if Configuration.exists(key=proxy_scrub_config_key) is False else Configuration.get(proxy_scrub_config_key)
if proxy_scrub_config is not None and proxy_scrub_config[StorageDriverConfiguration.CACHE_FRAGMENT] == ['none']:
proxy_scrub_config[StorageDriverConfiguration.CACHE_FRAGMENT] = fragment_cache_scrub_info
Configuration.set(key=proxy_scrub_config_key, value=proxy_scrub_config)
# Update 'backend_connection_manager' section
changes = False
storagedriver_config = StorageDriverConfiguration(vpool.guid, storagedriver.storagedriver_id)
if 'backend_connection_manager' not in storagedriver_config.configuration:
continue
current_config = storagedriver_config.configuration['backend_connection_manager']
if current_config.get('backend_type') != 'MULTI':
changes = True
backend_connection_manager = {'backend_type': 'MULTI'}
for index, proxy in enumerate(sorted(storagedriver.alba_proxies, key=lambda pr: pr.service.ports[0])):
backend_connection_manager[str(index)] = copy.deepcopy(current_config)
# noinspection PyUnresolvedReferences
backend_connection_manager[str(index)]['alba_connection_use_rora'] = True
# noinspection PyUnresolvedReferences
backend_connection_manager[str(index)]['alba_connection_rora_manifest_cache_capacity'] = 5000
# noinspection PyUnresolvedReferences
for key, value in backend_connection_manager[str(index)].items():
if key.startswith('backend_interface'):
backend_connection_manager[key] = value
# noinspection PyUnresolvedReferences
del backend_connection_manager[str(index)][key]
for key, value in {'backend_interface_retries_on_error': 5,
'backend_interface_retry_interval_secs': 1,
'backend_interface_retry_backoff_multiplier': 2.0}.iteritems():
if key not in backend_connection_manager:
backend_connection_manager[key] = value
else:
backend_connection_manager = current_config
for value in backend_connection_manager.values():
if isinstance(value, dict):
for key, val in value.items():
if key.startswith('backend_interface'):
backend_connection_manager[key] = val
changes = True
del value[key]
for key, value in {'backend_interface_retries_on_error': 5,
'backend_interface_retry_interval_secs': 1,
'backend_interface_retry_backoff_multiplier': 2.0}.iteritems():
if key not in backend_connection_manager:
changes = True
backend_connection_manager[key] = value
if changes is True:
storagedriver_config.clear_backend_connection_manager()
storagedriver_config.configure_backend_connection_manager(**backend_connection_manager)
storagedriver_config.save(root_client)
# Add '-reboot' to volumedriver services (because of updated 'backend_connection_manager' section)
ExtensionsToolbox.edit_version_file(client=root_client,
package_name='volumedriver',
old_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, 'volumedriver_{0}'.format(vpool.name)))
if service_manager.__class__ == Systemd:
root_client.run(['systemctl', 'daemon-reload'])
########################################
# Update metadata_store_bits information
vpools = VPoolList.get_vpools()
for vpool in vpools:
bits = None
for storagedriver in vpool.storagedrivers:
key = '/ovs/framework/hosts/{0}/services/volumedriver_{1}'.format(storagedriver.storagerouter.machine_id, vpool.name)
if Configuration.exists(key=key) and 'METADATASTORE_BITS' not in Configuration.get(key=key):
if bits is None:
entries = service_manager.extract_from_service_file(name='ovs-volumedriver_{0}'.format(vpool.name),
client=sr_client_map[storagedriver.storagerouter_guid],
entries=['METADATASTORE_BITS='])
if len(entries) == 1:
bits = entries[0].split('=')[-1]
bits = int(bits) if bits.isdigit() else 5
if bits is not None:
try:
content = Configuration.get(key=key)
content['METADATASTORE_BITS'] = bits
Configuration.set(key=key, value=content)
except:
MigrationController._logger.exception('Error updating volumedriver info for vPool {0} on StorageRouter {1}'.format(vpool.name, storagedriver.storagerouter.name))
if bits is not None:
vpool.metadata_store_bits = bits
vpool.save()
#####################################
# Update the vPool metadata structure
def _update_metadata_structure(metadata):
metadata = copy.deepcopy(metadata)
cache_structure = {'read': False,
'write': False,
'is_backend': False,
'quota': None,
'backend_info': {'name': None, # Will be filled in when is_backend is true
'backend_guid': None,
'alba_backend_guid': None,
'policies': None,
'preset': None,
'arakoon_config': None,
'connection_info': {'client_id': None,
'client_secret': None,
'host': None,
'port': None,
'local': None}}
}
structure_map = {StorageDriverConfiguration.CACHE_BLOCK: {'read': 'block_cache_on_read',
'write': 'block_cache_on_write',
'quota': 'quota_bc',
'backend_prefix': 'backend_bc_{0}'},
StorageDriverConfiguration.CACHE_FRAGMENT: {'read': 'fragment_cache_on_read',
'write': 'fragment_cache_on_write',
'quota': 'quota_fc',
'backend_prefix': 'backend_aa_{0}'}}
if 'arakoon_config' in metadata['backend']: # Arakoon config should be placed under the backend info
metadata['backend']['backend_info']['arakoon_config'] = metadata['backend'].pop('arakoon_config')
if 'connection_info' in metadata['backend']: # Connection info sohuld be placed under the backend info
metadata['backend']['backend_info']['connection_info'] = metadata['backend'].pop('connection_info')
if 'caching_info' not in metadata: # Caching info is the new key
would_be_caching_info = {}
metadata['caching_info'] = would_be_caching_info
# Extract all caching data for every storagerouter
current_caching_info = metadata['backend'].pop('caching_info') # Pop to mutate metadata
for storagerouter_guid in current_caching_info.iterkeys():
current_cache_data = current_caching_info[storagerouter_guid]
storagerouter_caching_info = {}
would_be_caching_info[storagerouter_guid] = storagerouter_caching_info
for cache_type, cache_type_mapping in structure_map.iteritems():
new_cache_structure = copy.deepcopy(cache_structure)
storagerouter_caching_info[cache_type] = new_cache_structure
for new_structure_key, old_structure_key in cache_type_mapping.iteritems():
if new_structure_key == 'backend_prefix':
# Get possible backend related info
metadata_key = old_structure_key.format(storagerouter_guid)
if metadata_key not in metadata:
continue
backend_data = metadata.pop(metadata_key) # Pop to mutate metadata
new_cache_structure['is_backend'] = True
# Copy over the old data
new_cache_structure['backend_info']['arakoon_config'] = backend_data['arakoon_config']
new_cache_structure['backend_info'].update(backend_data['backend_info'])
new_cache_structure['backend_info']['connection_info'].update(backend_data['connection_info'])
else:
new_cache_structure[new_structure_key] = current_cache_data.get(old_structure_key)
return metadata
vpools = VPoolList.get_vpools()
for vpool in vpools:
try:
new_metadata = _update_metadata_structure(vpool.metadata)
vpool.metadata = new_metadata
vpool.save()
except KeyError:
MigrationController._logger.exception('Exceptions occurred when updating the metadata for vPool {0}'.format(vpool.name))
##############################################
# Always use indent=4 during Configuration set
def _resave_all_config_entries(config_path='/ovs'):
"""
Recursive functions which checks every config management key if its a directory or not.
If not a directory, we retrieve the config and just save it again using the new indentation logic
"""
for item in Configuration.list(config_path):
new_path = config_path + '/' + item
print new_path
if Configuration.dir_exists(new_path) is True:
_resave_all_config_entries(config_path=new_path)
else:
try:
_config = Configuration.get(new_path)
Configuration.set(new_path, _config)
except:
_config = Configuration.get(new_path, raw=True)
Configuration.set(new_path, _config, raw=True)
if ExtensionMigrator.THIS_VERSION <= 13: # There is no way of checking whether this new indentation logic has been applied, so we only perform this for version 13 and lower
MigrationController._logger.info('Re-saving every configuration setting with new indentation rules')
_resave_all_config_entries()
############################
# Update some default values
def _update_manifest_cache_size(_proxy_config_key):
updated = False
manifest_cache_size = 500 * 1024 * 1024
if Configuration.exists(key=_proxy_config_key):
_proxy_config = Configuration.get(key=_proxy_config_key)
for cache_type in [StorageDriverConfiguration.CACHE_BLOCK, StorageDriverConfiguration.CACHE_FRAGMENT]:
if cache_type in _proxy_config and _proxy_config[cache_type][0] == 'alba':
if _proxy_config[cache_type][1]['manifest_cache_size'] != manifest_cache_size:
updated = True
_proxy_config[cache_type][1]['manifest_cache_size'] = manifest_cache_size
if _proxy_config['manifest_cache_size'] != manifest_cache_size:
updated = True
_proxy_config['manifest_cache_size'] = manifest_cache_size
if updated is True:
Configuration.set(key=_proxy_config_key, value=_proxy_config)
return updated
for storagedriver in StorageDriverList.get_storagedrivers():
try:
vpool = storagedriver.vpool
root_client = sr_client_map[storagedriver.storagerouter_guid]
_update_manifest_cache_size('/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(vpool.guid)) # Generic scrub proxy is deployed every time scrubbing kicks in, so no need to restart these services
for alba_proxy in storagedriver.alba_proxies:
if _update_manifest_cache_size('/ovs/vpools/{0}/proxies/{1}/config/main'.format(vpool.guid, alba_proxy.guid)) is True:
# Add '-reboot' to alba_proxy services (because of newly created services and removal of old service)
ExtensionsToolbox.edit_version_file(client=root_client,
package_name='alba',
old_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, alba_proxy.service.name))
# Update 'backend_connection_manager' section
changes = False
storagedriver_config = StorageDriverConfiguration(vpool.guid, storagedriver.storagedriver_id)
if 'backend_connection_manager' not in storagedriver_config.configuration:
continue
current_config = storagedriver_config.configuration['backend_connection_manager']
for key, value in current_config.iteritems():
if key.isdigit() is True:
if value.get('alba_connection_asd_connection_pool_capacity') != 10:
changes = True
value['alba_connection_asd_connection_pool_capacity'] = 10
if value.get('alba_connection_timeout') != 30:
changes = True
value['alba_connection_timeout'] = 30
if value.get('alba_connection_rora_manifest_cache_capacity') != 25000:
changes = True
value['alba_connection_rora_manifest_cache_capacity'] = 25000
if changes is True:
storagedriver_config.clear_backend_connection_manager()
storagedriver_config.configure_backend_connection_manager(**current_config)
storagedriver_config.save(root_client)
# Add '-reboot' to volumedriver services (because of updated 'backend_connection_manager' section)
ExtensionsToolbox.edit_version_file(client=root_client,
package_name='volumedriver',
old_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, 'volumedriver_{0}'.format(vpool.name)))
except Exception:
MigrationController._logger.exception('Updating default configuration values failed for StorageDriver {0}'.format(storagedriver.storagedriver_id))
####################################################
# Adding proxy fail fast as env variable for proxies
changed_clients = set()
for storagerouter in StorageRouterList.get_storagerouters():
root_client = sr_client_map[storagerouter.guid]
for service_name in service_manager.list_services(client=root_client):
if not service_name.startswith('ovs-albaproxy_'):
continue
if ServiceFactory.get_service_type() == 'systemd':
path = '/lib/systemd/system/{0}.service'.format(service_name)
check = 'Environment=ALBA_FAIL_FAST=true'
else:
path = '/etc/init/{0}.conf'.format(service_name)
check = 'env ALBA_FAIL_FAST=true'
if not root_client.file_exists(path):
continue
if check in root_client.file_read(path):
continue
try:
service_manager.regenerate_service(name=StorageDriverInstaller.SERVICE_TEMPLATE_PROXY, client=root_client, target_name=service_name)
changed_clients.add(root_client)
ExtensionsToolbox.edit_version_file(client=root_client,
package_name='alba',
old_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, service_name))
except:
MigrationController._logger.exception('Error rebuilding service {0}'.format(service_name))
for root_client in changed_clients:
root_client.run(['systemctl', 'daemon-reload'])
######################################
# Integration of stats monkey (2.10.2)
if Configuration.get(key='/ovs/framework/migration|stats_monkey_integration', default=False) is False:
try:
# Get content of old key into new key
old_stats_monkey_key = '/statsmonkey/statsmonkey'
if Configuration.exists(key=old_stats_monkey_key) is True:
Configuration.set(key='/ovs/framework/monitoring/stats_monkey', value=Configuration.get(key=old_stats_monkey_key))
Configuration.delete(key=old_stats_monkey_key)
# Make sure to disable the stats monkey by default or take over the current schedule if it was configured manually before
celery_key = '/ovs/framework/scheduling/celery'
current_value = None
scheduling_config = Configuration.get(key=celery_key, default={})
if 'statsmonkey.run_all_stats' in scheduling_config: # Old celery task name of the stats monkey
current_value = scheduling_config.pop('statsmonkey.run_all_stats')
scheduling_config['ovs.stats_monkey.run_all'] = current_value
scheduling_config['alba.stats_monkey.run_all'] = current_value
Configuration.set(key=celery_key, value=scheduling_config)
support_key = '/ovs/framework/support'
support_config = Configuration.get(key=support_key)
support_config['support_agent'] = support_config.pop('enabled', True)
support_config['remote_access'] = support_config.pop('enablesupport', False)
Configuration.set(key=support_key, value=support_config)
# Make sure once this finished, it never runs again by setting this key to True
Configuration.set(key='/ovs/framework/migration|stats_monkey_integration', value=True)
except Exception:
MigrationController._logger.exception('Integration of stats monkey failed')
######################################################
# Write away cluster ID to a file for back-up purposes
try:
cluster_id = Configuration.get(key='/ovs/framework/cluster_id', default=None)
with open(Configuration.CONFIG_STORE_LOCATION, 'r') as config_file:
config = json.load(config_file)
if cluster_id is not None and config.get('cluster_id', None) is None:
config['cluster_id'] = cluster_id
with open(Configuration.CONFIG_STORE_LOCATION, 'w') as config_file:
json.dump(config, config_file, indent=4)
except Exception:
MigrationController._logger.exception('Writing cluster id to a file failed.')
#########################################################
# Additional string formatting in Arakoon services (2.11)
try:
if Configuration.get(key='/ovs/framework/migration|arakoon_service_update', default=False) is False:
arakoon_service_names = [ArakoonInstaller.get_service_name_for_cluster(cluster_name=cluster_name) for cluster_name in Configuration.list(key='ovs/arakoon')]
for storagerouter in StorageRouterList.get_masters():
for service_name in arakoon_service_names:
config_key = ServiceFactory.SERVICE_CONFIG_KEY.format(storagerouter.machine_id, service_name)
if Configuration.exists(key=config_key):
config = Configuration.get(key=config_key)
config['RUN_FILE_DIR'] = ServiceFactory.RUN_FILE_DIR
config['ARAKOON_PKG_NAME'] = PackageFactory.PKG_ARAKOON
config['ARAKOON_VERSION_CMD'] = PackageFactory.VERSION_CMD_ARAKOON
Configuration.set(key=config_key, value=config)
# Make sure once this finished, it never runs again by setting this key to True
Configuration.set(key='/ovs/framework/migration|arakoon_service_update', value=True)
except Exception:
MigrationController._logger.exception('Updating the string formatting for the Arakoon services failed')
############################################################
# Additional string formatting in ALBA proxy services (2.11)
changed_clients = set()
try:
if Configuration.get(key='/ovs/framework/migration|alba_proxy_service_update', default=False) is False:
alba_pkg_name, alba_version_cmd = PackageFactory.get_package_and_version_cmd_for(component=PackageFactory.COMP_ALBA)
for service in ServiceTypeList.get_by_name('AlbaProxy').services:
root_client = sr_client_map[service.storagerouter_guid]
config_key = ServiceFactory.SERVICE_CONFIG_KEY.format(service.storagerouter.machine_id, service.name)
if Configuration.exists(key=config_key):
config = Configuration.get(key=config_key)
config['RUN_FILE_DIR'] = ServiceFactory.RUN_FILE_DIR
config['ALBA_PKG_NAME'] = alba_pkg_name
config['ALBA_VERSION_CMD'] = alba_version_cmd
Configuration.set(key=config_key, value=config)
service_manager.regenerate_service(name=StorageDriverInstaller.SERVICE_TEMPLATE_PROXY,
client=root_client,
target_name='ovs-{0}'.format(service.name))
changed_clients.add(root_client)
# Make sure once this finished, it never runs again by setting this key to True
Configuration.set(key='/ovs/framework/migration|alba_proxy_service_update', value=True)
except Exception:
MigrationController._logger.exception('Updating the string formatting for the Arakoon services failed')
############################################################
# Additional string formatting in DTL/VOLDRV services (2.11)
try:
if Configuration.get(key='/ovs/framework/migration|voldrv_service_update', default=False) is False:
sd_pkg_name, sd_version_cmd = PackageFactory.get_package_and_version_cmd_for(component=PackageFactory.COMP_SD)
for vpool in VPoolList.get_vpools():
for storagedriver in vpool.storagedrivers:
root_client = sr_client_map[storagedriver.storagerouter_guid]
for entry in ['dtl', 'volumedriver']:
service_name = '{0}_{1}'.format(entry, vpool.name)
service_template = StorageDriverInstaller.SERVICE_TEMPLATE_DTL if entry == 'dtl' else StorageDriverInstaller.SERVICE_TEMPLATE_SD
config_key = ServiceFactory.SERVICE_CONFIG_KEY.format(storagedriver.storagerouter.machine_id, service_name)
if Configuration.exists(key=config_key):
config = Configuration.get(key=config_key)
config['RUN_FILE_DIR'] = ServiceFactory.RUN_FILE_DIR
config['VOLDRV_PKG_NAME'] = sd_pkg_name
config['VOLDRV_VERSION_CMD'] = sd_version_cmd
Configuration.set(key=config_key, value=config)
service_manager.regenerate_service(name=service_template,
client=root_client,
target_name='ovs-{0}'.format(service_name))
changed_clients.add(root_client)
# Make sure once this finished, it never runs again by setting this key to True
Configuration.set(key='/ovs/framework/migration|voldrv_service_update', value=True)
except Exception:
MigrationController._logger.exception('Updating the string formatting for the Arakoon services failed')
#######################################################
# Storing actual package name in version files (2.11.0) (https://github.com/openvstorage/framework/issues/1876)
if Configuration.get(key='/ovs/framework/migration|actual_package_name_in_version_file', default=False) is False:
try:
voldrv_pkg_name, _ = PackageFactory.get_package_and_version_cmd_for(component=PackageFactory.COMP_SD)
for storagerouter in StorageRouterList.get_storagerouters():
root_client = sr_client_map.get(storagerouter.guid)
if root_client is None:
continue
for file_name in root_client.file_list(directory=ServiceFactory.RUN_FILE_DIR):
if not file_name.endswith('.version'):
continue
file_path = '{0}/{1}'.format(ServiceFactory.RUN_FILE_DIR, file_name)
contents = root_client.file_read(filename=file_path)
regenerate = False
if voldrv_pkg_name == PackageFactory.PKG_VOLDRV_SERVER:
if 'volumedriver-server' in contents:
regenerate = True
contents = contents.replace('volumedriver-server', PackageFactory.PKG_VOLDRV_SERVER)
root_client.file_write(filename=file_path, contents=contents)
elif voldrv_pkg_name == PackageFactory.PKG_VOLDRV_SERVER_EE:
if 'volumedriver-server' in contents or PackageFactory.PKG_VOLDRV_SERVER in contents:
regenerate = True
contents = contents.replace('volumedriver-server', PackageFactory.PKG_VOLDRV_SERVER_EE)
contents = contents.replace(PackageFactory.PKG_VOLDRV_SERVER, PackageFactory.PKG_VOLDRV_SERVER_EE)
root_client.file_write(filename=file_path, contents=contents)
if regenerate is True:
service_manager.regenerate_service(name=StorageDriverInstaller.SERVICE_TEMPLATE_DTL if file_name.startswith('dtl') else StorageDriverInstaller.SERVICE_TEMPLATE_SD,
client=root_client,
target_name='ovs-{0}'.format(file_name.split('.')[0])) # Leave out .version
changed_clients.add(root_client)
Configuration.set(key='/ovs/framework/migration|actual_package_name_in_version_file', value=True)
except Exception:
MigrationController._logger.exception('Updating actual package name for version files failed')
for root_client in changed_clients:
try:
root_client.run(['systemctl', 'daemon-reload'])
except Exception:
MigrationController._logger.exception('Executing command "systemctl daemon-reload" failed')
#########################################################
# Addition of 'Environment=OCAMLRUNPARAM='b,a=1,s=4096k,O=50' for AlbaProxy SystemD services
if ServiceFactory.get_service_type() == 'systemd':
changed_clients = set()
for storagedriver in StorageDriverList.get_storagedrivers():
root_client = sr_client_map[storagedriver.storagerouter_guid]
for alba_proxy in storagedriver.alba_proxies:
service = alba_proxy.service
service_name = 'ovs-{0}'.format(service.name)
if not service_manager.has_service(name=service_name, client=root_client):
continue
if "Environment=OCAMLRUNPARAM='b,a=1,s=4096k,O=50" in root_client.file_read(filename='/lib/systemd/system/{0}.service'.format(service_name)):
continue
try:
service_manager.regenerate_service(name='ovs-albaproxy', client=root_client, target_name=service_name)
changed_clients.add(root_client)
except:
MigrationController._logger.exception('Error rebuilding service {0}'.format(service_name))
for root_client in changed_clients:
root_client.run(['systemctl', 'daemon-reload'])
#########################################################
# Addition of 'Environment=OCAMLRUNPARAM='b,a=1,s=4096k,O=50' for Arakoon SystemD services
if ServiceFactory.get_service_type() == 'systemd':
changed_clients = set()
for storagerouter in StorageRouterList.get_storagerouters():
root_client = sr_client_map[storagerouter.guid]
for service_name in service_manager.list_services(client=root_client):
if not service_name.startswith('ovs-arakoon-'):
continue
if not service_manager.has_service(name=service_name, client=root_client):
continue
if "Environment=OCAMLRUNPARAM='b,a=1,s=4096k,O=50" in root_client.file_read(filename='/lib/systemd/system/{0}.service'.format(service_name)):
continue
try:
service_manager.regenerate_service(name='ovs-arakoon', client=root_client, target_name=service_name)
changed_clients.add(root_client)
except:
MigrationController._logger.exception('Error rebuilding service {0}'.format(service_name))
for root_client in changed_clients:
root_client.run(['systemctl', 'daemon-reload'])
MigrationController._logger.info('Finished out of band migrations')
def create_from_template(diskguid, machinename, devicename, pmachineguid, machineguid=None, storagedriver_guid=None):
"""
Create a disk from a template
@param devicename: device file name for the disk (eg: mydisk-flat.vmdk)
@param machineguid: guid of the machine to assign disk to
@return diskguid: guid of new disk
"""
pmachine = PMachine(pmachineguid)
hypervisor = Factory.get(pmachine)
disk_path = hypervisor.get_disk_path(machinename, devicename)
description = '{} {}'.format(machinename, devicename)
properties_to_clone = [
'description', 'size', 'type', 'retentionpolicyid',
'snapshotpolicyid', 'vmachine', 'vpool']
vdisk = VDisk(diskguid)
if vdisk.vmachine and not vdisk.vmachine.is_vtemplate:
# Disk might not be attached to a vmachine, but still be a template
raise RuntimeError('The given vdisk does not belong to a template')
if storagedriver_guid is not None:
storagedriver_id = StorageDriver(storagedriver_guid).storagedriver_id
else:
storagedriver_id = vdisk.storagedriver_id
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
if storagedriver is None:
raise RuntimeError('Could not find StorageDriver with id {0}'.format(storagedriver_id))
new_vdisk = VDisk()
new_vdisk.copy(vdisk, include=properties_to_clone)
new_vdisk.vpool = vdisk.vpool
new_vdisk.devicename = hypervisor.clean_backing_disk_filename(disk_path)
new_vdisk.parent_vdisk = vdisk
new_vdisk.name = '{}-clone'.format(vdisk.name)
new_vdisk.description = description
new_vdisk.vmachine = VMachine(machineguid) if machineguid else vdisk.vmachine
new_vdisk.save()
mds_service = MDSServiceController.get_preferred_mds(storagedriver.storagerouter, vdisk.vpool)
if mds_service is None:
raise RuntimeError('Could not find a MDS service')
logger.info('Create disk from template {} to new disk {} to location {}'.format(
vdisk.name, new_vdisk.name, disk_path
))
try:
volume_id = vdisk.storagedriver_client.create_clone_from_template(
target_path=disk_path,
metadata_backend_config=MDSMetaDataBackendConfig([MDSNodeConfig(address=str(mds_service.service.storagerouter.ip),
port=mds_service.service.ports[0])]),
parent_volume_id=str(vdisk.volume_id),
node_id=str(storagedriver_id)
)
new_vdisk.volume_id = volume_id
new_vdisk.save()
MDSServiceController.ensure_safety(new_vdisk)
except Exception as ex:
logger.error('Clone disk on volumedriver level failed with exception: {0}'.format(str(ex)))
new_vdisk.delete()
raise
return {'diskguid': new_vdisk.guid, 'name': new_vdisk.name,
'backingdevice': disk_path}