def update_vdisk_name(volume_id, old_name, new_name):
"""
Update a vDisk name using Management Center: set new name
"""
vdisk = None
for mgmt_center in MgmtCenterList.get_mgmtcenters():
mgmt = Factory.get_mgmtcenter(mgmt_center = mgmt_center)
try:
disk_info = mgmt.get_vdisk_device_info(volume_id)
device_path = disk_info['device_path']
vpool_name = disk_info['vpool_name']
vp = VPoolList.get_vpool_by_name(vpool_name)
file_name = os.path.basename(device_path)
vdisk = VDiskList.get_by_devicename_and_vpool(file_name, vp)
if vdisk:
break
except Exception as ex:
logger.info('Trying to get mgmt center failed for disk {0} with volume_id {1}. {2}'.format(old_name, volume_id, ex))
if not vdisk:
logger.error('No vdisk found for name {0}'.format(old_name))
return
vpool = vdisk.vpool
mutex = VolatileMutex('{}_{}'.format(old_name, vpool.guid if vpool is not None else 'none'))
try:
mutex.acquire(wait=5)
vdisk.name = new_name
vdisk.save()
finally:
mutex.release()
def sync_with_reality(storagerouter_guid=None, max_attempts=3):
"""
Try to run sync_with_reality, retry in case of failure
always run sync, as tasks calling this expect this to be sync
:param storagerouter_guid:
:return:
"""
cache = VolatileFactory.get_client()
mutex = VolatileMutex('ovs_disk_sync_with_reality_{0}'.format(storagerouter_guid))
key = 'ovs_dedupe_sync_with_reality_{0}'.format(storagerouter_guid)
attempt = 1
while attempt < max_attempts:
task_id = cache.get(key)
if task_id:
revoke(task_id)
try:
mutex.acquire(wait=120)
return DiskController._sync_with_reality(storagerouter_guid)
except Exception as ex:
logger.warning('Sync with reality failed. {0}'.format(ex))
attempt += 1
time.sleep(attempt*30)
finally:
mutex.release()
raise RuntimeError('Sync with reality failed after 3 attempts')
def new_function(self, request, *args, **kwargs):
"""
Wrapped function
"""
now = time.time()
key = 'ovs_api_limit_{0}.{1}_{2}'.format(
f.__module__, f.__name__,
request.META['HTTP_X_REAL_IP']
)
client = VolatileFactory.get_client()
mutex = VolatileMutex(key)
try:
mutex.acquire()
rate_info = client.get(key, {'calls': [],
'timeout': None})
active_timeout = rate_info['timeout']
if active_timeout is not None:
if active_timeout > now:
return HttpResponse, {'error_code': 'rate_limit_timeout',
'error': 'Rate limit timeout ({0}s remaining)'.format(round(active_timeout - now, 2))}, 429
else:
rate_info['timeout'] = None
rate_info['calls'] = [call for call in rate_info['calls'] if call > (now - per)] + [now]
calls = len(rate_info['calls'])
if calls > amount:
rate_info['timeout'] = now + timeout
client.set(key, rate_info)
return HttpResponse, {'error_code': 'rate_limit_reached',
'error': 'Rate limit reached ({0} in last {1}s)'.format(calls, per)}, 429
client.set(key, rate_info)
finally:
mutex.release()
return f(self, request, *args, **kwargs)
def delete_from_voldrv(volumename, storagedriver_id):
"""
Delete a disk
Triggered by volumedriver messages on the queue
@param volumename: volume id of the disk
"""
_ = storagedriver_id # For logging purposes
disk = VDiskList.get_vdisk_by_volume_id(volumename)
if disk is not None:
mutex = VolatileMutex('{}_{}'.format(volumename, disk.devicename))
try:
mutex.acquire(wait=20)
pmachine = None
try:
pmachine = PMachineList.get_by_storagedriver_id(disk.storagedriver_id)
except RuntimeError as ex:
if 'could not be found' not in str(ex):
raise
# else: pmachine can't be loaded, because the volumedriver doesn't know about it anymore
if pmachine is not None:
limit = 5
hypervisor = Factory.get(pmachine)
exists = hypervisor.file_exists(disk.vpool, disk.devicename)
while limit > 0 and exists is True:
time.sleep(1)
exists = hypervisor.file_exists(disk.vpool, disk.devicename)
limit -= 1
if exists is True:
logger.info('Disk {0} still exists, ignoring delete'.format(disk.devicename))
return
logger.info('Delete disk {}'.format(disk.name))
disk.delete()
finally:
mutex.release()
def new_function(self, request, *args, **kwargs):
"""
Wrapped function
"""
now = time.time()
key = 'ovs_api_limit_{0}.{1}_{2}'.format(
f.__module__, f.__name__,
request.META['HTTP_X_REAL_IP']
)
client = VolatileFactory.get_client()
mutex = VolatileMutex(key)
try:
mutex.acquire()
rate_info = client.get(key, {'calls': [],
'timeout': None})
active_timeout = rate_info['timeout']
if active_timeout is not None:
if active_timeout > now:
raise Throttled(wait=active_timeout - now)
else:
rate_info['timeout'] = None
rate_info['calls'] = [call for call in rate_info['calls'] if call > (now - per)] + [now]
calls = len(rate_info['calls'])
if calls > amount:
rate_info['timeout'] = now + timeout
client.set(key, rate_info)
raise Throttled(wait=timeout)
client.set(key, rate_info)
finally:
mutex.release()
return f(self, request, *args, **kwargs)
def resize_from_voldrv(volumename, volumesize, volumepath, storagedriver_id):
"""
Resize a disk
Triggered by volumedriver messages on the queue
@param volumepath: path on hypervisor to the volume
@param volumename: volume id of the disk
@param volumesize: size of the volume
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
hypervisor = Factory.get(pmachine)
volumepath = hypervisor.clean_backing_disk_filename(volumepath)
mutex = VolatileMutex('{}_{}'.format(volumename, volumepath))
try:
mutex.acquire(wait=30)
disk = VDiskList.get_vdisk_by_volume_id(volumename)
if disk is None:
disk = VDiskList.get_by_devicename_and_vpool(volumepath, storagedriver.vpool)
if disk is None:
disk = VDisk()
finally:
mutex.release()
disk.devicename = volumepath
disk.volume_id = volumename
disk.size = volumesize
disk.vpool = storagedriver.vpool
disk.save()
VDiskController.sync_with_mgmtcenter(disk, pmachine, storagedriver)
MDSServiceController.ensure_safety(disk)
def __init__(self, *args, **kwargs):
"""
Initializes the distributed scheduler
"""
self._persistent = PersistentFactory.get_client()
self._namespace = 'ovs_celery_beat'
self._mutex = VolatileMutex('celery_beat')
self._has_lock = False
super(DistributedScheduler, self).__init__(*args, **kwargs)
logger.debug('DS init')
def create_cluster(cluster_name, ip, base_dir, plugins=None, locked=True):
"""
Creates a cluster
:param locked: Indicates whether the create should run in a locked context (e.g. to prevent port conflicts)
:param plugins: Plugins that should be added to the configuration file
:param base_dir: Base directory that should contain the data and tlogs
:param ip: IP address of the first node of the new cluster
:param cluster_name: Name of the cluster
"""
logger.debug('Creating cluster {0} on {1}'.format(cluster_name, ip))
base_dir = base_dir.rstrip('/')
client = SSHClient(ip)
if ArakoonInstaller.is_running(cluster_name, client):
logger.info('Arakoon service running for cluster {0}'.format(cluster_name))
config = ArakoonClusterConfig(cluster_name, plugins)
config.load_config()
for node in config.nodes:
if node.ip == ip:
return {'client_port': node.client_port,
'messaging_port': node.messaging_port}
node_name = System.get_my_machine_id(client)
home_dir = ArakoonInstaller.ARAKOON_HOME_DIR.format(base_dir, cluster_name)
log_dir = ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name)
tlog_dir = ArakoonInstaller.ARAKOON_TLOG_DIR.format(base_dir, cluster_name)
ArakoonInstaller.clean_leftover_arakoon_data(ip, {log_dir: True,
home_dir: False,
tlog_dir: False})
port_mutex = None
try:
if locked is True:
from ovs.extensions.generic.volatilemutex import VolatileMutex
port_mutex = VolatileMutex('arakoon_install_ports_{0}'.format(ip))
port_mutex.acquire(wait=60)
ports = ArakoonInstaller._get_free_ports(client)
config = ArakoonClusterConfig(cluster_name, plugins)
config.nodes.append(ArakoonNodeConfig(name=node_name,
ip=ip,
client_port=ports[0],
messaging_port=ports[1],
log_dir=log_dir,
home=home_dir,
tlog_dir=tlog_dir))
ArakoonInstaller._deploy(config)
finally:
if port_mutex is not None:
port_mutex.release()
logger.debug('Creating cluster {0} on {1} completed'.format(cluster_name, ip))
return {'client_port': ports[0],
'messaging_port': ports[1]}
def new_function(*args, **kw):
"""
Executes the decorated function in a locked context
"""
filemutex = FileMutex('messaging')
try:
filemutex.acquire(wait=5)
mutex = VolatileMutex('messaging')
try:
mutex.acquire(wait=5)
return f(*args, **kw)
finally:
mutex.release()
finally:
filemutex.release()
def extend_cluster(master_ip, new_ip, cluster_name, base_dir):
"""
Extends a cluster to a given new node
:param base_dir: Base directory that will hold the db and tlogs
:param cluster_name: Name of the cluster to be extended
:param new_ip: IP address of the node to be added
:param master_ip: IP of one of the already existing nodes
"""
logger.debug('Extending cluster {0} from {1} to {2}'.format(cluster_name, master_ip, new_ip))
base_dir = base_dir.rstrip('/')
from ovs.extensions.generic.volatilemutex import VolatileMutex
port_mutex = VolatileMutex('arakoon_install_ports_{0}'.format(new_ip))
config = ArakoonClusterConfig(cluster_name)
config.load_config()
client = SSHClient(new_ip)
node_name = System.get_my_machine_id(client)
home_dir = ArakoonInstaller.ARAKOON_HOME_DIR.format(base_dir, cluster_name)
log_dir = ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name)
tlog_dir = ArakoonInstaller.ARAKOON_TLOG_DIR.format(base_dir, cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(new_ip, home_dir,
ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir), cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(new_ip, log_dir,
ArakoonInstaller.ARAKOON_LOG_DIR.format(''), cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(new_ip, tlog_dir,
ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir), cluster_name)
try:
port_mutex.acquire(wait=60)
ports = ArakoonInstaller._get_free_ports(client)
if node_name not in [node.name for node in config.nodes]:
config.nodes.append(ArakoonNodeConfig(name=node_name,
ip=new_ip,
client_port=ports[0],
messaging_port=ports[1],
log_dir=log_dir,
home=home_dir,
tlog_dir=tlog_dir))
ArakoonInstaller._deploy(config)
finally:
port_mutex.release()
logger.debug('Extending cluster {0} from {1} to {2} completed'.format(cluster_name, master_ip, new_ip))
return {'client_port': ports[0],
'messaging_port': ports[1]}
def new_function(*args, **kwargs):
"""
Wrapped function
"""
request = _find_request(args)
now = time.time()
key = 'ovs_api_limit_{0}.{1}_{2}'.format(
f.__module__, f.__name__, request.META['HTTP_X_REAL_IP'])
client = VolatileFactory.get_client()
with VolatileMutex(key):
rate_info = client.get(key, {'calls': [], 'timeout': None})
active_timeout = rate_info['timeout']
if active_timeout is not None:
if active_timeout > now:
logger.warning(
'Call {0} is being throttled with a wait of {1}'.
format(key, active_timeout - now))
raise Throttled(wait=active_timeout - now)
else:
rate_info['timeout'] = None
rate_info['calls'] = [
call for call in rate_info['calls'] if call > (now - per)
] + [now]
calls = len(rate_info['calls'])
if calls > amount:
rate_info['timeout'] = now + timeout
client.set(key, rate_info)
logger.warning(
'Call {0} is being throttled with a wait of {1}'.
format(key, timeout))
raise Throttled(wait=timeout)
client.set(key, rate_info)
return f(*args, **kwargs)
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
# Load dummy stores
PersistentFactory.store = DummyPersistentStore()
VolatileFactory.store = DummyVolatileStore()
# Replace mocked classes
sys.modules[
'ovs.extensions.storageserver.storagedriver'] = StorageDriver
# Import required modules/classes after mocking is done
from ovs.dal.hybrids.vmachine import VMachine
from ovs.dal.hybrids.vdisk import VDisk
from ovs.extensions.generic.volatilemutex import VolatileMutex
from ovs.lib.vmachine import VMachineController
from ovs.lib.vdisk import VDiskController
from ovs.lib.scheduledtask import ScheduledTaskController
# Globalize mocked classes
global VDisk
global VMachine
global VolatileMutex
global VMachineController
global VDiskController
global ScheduledTaskController
_ = VDisk(), VolatileMutex('dummy'), VMachine(
), VMachineController, VDiskController, ScheduledTaskController
# Cleaning storage
VolatileFactory.store.clean()
PersistentFactory.store.clean()
def __init__(self, *args, **kwargs):
"""
Initializes the distributed scheduler
"""
self._persistent = PersistentFactory.get_client()
self._namespace = 'ovs_celery_beat'
self._mutex = VolatileMutex('celery_beat')
self._has_lock = False
super(DistributedScheduler, self).__init__(*args, **kwargs)
logger.debug('DS init')
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
# Load dummy stores
PersistentFactory.store = DummyPersistentStore()
VolatileFactory.store = DummyVolatileStore()
# Replace mocked classes
sys.modules[
'ovs.extensions.storageserver.storagedriver'] = StorageDriverModule
# Import required modules/classes after mocking is done
from ovs.dal.hybrids.vdisk import VDisk
from ovs.dal.hybrids.service import Service
from ovs.dal.hybrids.vpool import VPool
from ovs.dal.hybrids.storagerouter import StorageRouter
from ovs.dal.hybrids.pmachine import PMachine
from ovs.dal.hybrids.servicetype import ServiceType
from ovs.dal.hybrids.storagedriver import StorageDriver
from ovs.dal.hybrids.backendtype import BackendType
from ovs.dal.hybrids.j_mdsservice import MDSService
from ovs.dal.hybrids.j_mdsservicevdisk import MDSServiceVDisk
from ovs.extensions.generic.volatilemutex import VolatileMutex
from ovs.lib.mdsservice import MDSServiceController
# Globalize mocked classes
global VDisk
global VPool
global Service
global StorageRouter
global StorageDriver
global BackendType
global PMachine
global MDSService
global ServiceType
global MDSServiceVDisk
global VolatileMutex
global MDSServiceController
_ = VDisk(), VPool(), Service(), MDSService(), MDSServiceVDisk(), ServiceType(), \
StorageRouter(), StorageDriver(), BackendType(), PMachine(), \
VolatileMutex('dummy'), MDSServiceController
# Configuration
def _get(key):
c = PersistentFactory.get_client()
if c.exists(key):
return c.get(key)
return None
Configuration.get = staticmethod(_get)
# Cleaning storage
VolatileFactory.store.clean()
PersistentFactory.store.clean()
def update_vdisk_name(volume_id, old_name, new_name):
"""
Update a vDisk name using Management Center: set new name
"""
vdisk = None
for mgmt_center in MgmtCenterList.get_mgmtcenters():
mgmt = Factory.get_mgmtcenter(mgmt_center=mgmt_center)
try:
disk_info = mgmt.get_vdisk_device_info(volume_id)
device_path = disk_info['device_path']
vpool_name = disk_info['vpool_name']
vp = VPoolList.get_vpool_by_name(vpool_name)
file_name = os.path.basename(device_path)
vdisk = VDiskList.get_by_devicename_and_vpool(file_name, vp)
if vdisk:
break
except Exception as ex:
logger.info(
'Trying to get mgmt center failed for disk {0} with volume_id {1}. {2}'
.format(old_name, volume_id, ex))
if not vdisk:
logger.error('No vdisk found for name {0}'.format(old_name))
return
vpool = vdisk.vpool
mutex = VolatileMutex('{}_{}'.format(
old_name, vpool.guid if vpool is not None else 'none'))
try:
mutex.acquire(wait=5)
vdisk.name = new_name
vdisk.save()
finally:
mutex.release()
def resize_from_voldrv(volumename, volumesize, volumepath,
storagedriver_id):
"""
Resize a disk
Triggered by volumedriver messages on the queue
@param volumepath: path on hypervisor to the volume
@param volumename: volume id of the disk
@param volumesize: size of the volume
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
hypervisor = Factory.get(pmachine)
volumepath = hypervisor.clean_backing_disk_filename(volumepath)
mutex = VolatileMutex('{}_{}'.format(volumename, volumepath))
try:
mutex.acquire(wait=30)
disk = VDiskList.get_vdisk_by_volume_id(volumename)
if disk is None:
disk = VDiskList.get_by_devicename_and_vpool(
volumepath, storagedriver.vpool)
if disk is None:
disk = VDisk()
finally:
mutex.release()
disk.devicename = volumepath
disk.volume_id = volumename
disk.size = volumesize
disk.vpool = storagedriver.vpool
disk.save()
def _backend_property(self, function, dynamic):
"""
Handles the internal caching of dynamic properties
"""
caller_name = dynamic.name
cache_key = '{0}_{1}'.format(self._key, caller_name)
mutex = VolatileMutex(cache_key)
try:
cached_data = self._volatile.get(cache_key)
if cached_data is None:
if dynamic.locked:
mutex.acquire()
cached_data = self._volatile.get(cache_key)
if cached_data is None:
function_info = inspect.getargspec(function)
if 'dynamic' in function_info.args:
cached_data = function(dynamic=dynamic) # Load data from backend
else:
cached_data = function()
if cached_data is not None:
correct, allowed_types, given_type = Toolbox.check_type(cached_data, dynamic.return_type)
if not correct:
raise TypeError('Dynamic property {0} allows types {1}. {2} given'.format(
caller_name, str(allowed_types), given_type
))
if dynamic.timeout > 0:
self._volatile.set(cache_key, cached_data, dynamic.timeout)
return cached_data
finally:
mutex.release()
def new_function(self, request, *args, **kwargs):
"""
Wrapped function
"""
now = time.time()
key = 'ovs_api_limit_{0}.{1}_{2}'.format(
f.__module__, f.__name__, request.META['HTTP_X_REAL_IP'])
client = VolatileFactory.get_client()
mutex = VolatileMutex(key)
try:
mutex.acquire()
rate_info = client.get(key, {'calls': [], 'timeout': None})
active_timeout = rate_info['timeout']
if active_timeout is not None:
if active_timeout > now:
raise Throttled(wait=active_timeout - now)
else:
rate_info['timeout'] = None
rate_info['calls'] = [
call for call in rate_info['calls'] if call > (now - per)
] + [now]
calls = len(rate_info['calls'])
if calls > amount:
rate_info['timeout'] = now + timeout
client.set(key, rate_info)
raise Throttled(wait=timeout)
client.set(key, rate_info)
finally:
mutex.release()
return f(self, request, *args, **kwargs)
def update_vmachine_name(instance_id, old_name, new_name):
"""
Update a vMachine name: find vmachine by management center instance id, set new name
:param instance_id: ID for the virtual machine known by management center
:param old_name: Old name of the virtual machine
:param new_name: New name for the virtual machine
"""
vmachine = None
for mgmt_center in MgmtCenterList.get_mgmtcenters():
mgmt = Factory.get_mgmtcenter(mgmt_center = mgmt_center)
try:
machine_info = mgmt.get_vmachine_device_info(instance_id)
file_name = machine_info['file_name']
host_name = machine_info['host_name']
vpool_name = machine_info['vpool_name']
storage_router = StorageRouterList.get_by_name(host_name)
machine_id = storage_router.machine_id
device_name = '{0}/{1}'.format(machine_id, file_name)
vp = VPoolList.get_vpool_by_name(vpool_name)
vmachine = VMachineList.get_by_devicename_and_vpool(device_name, vp)
if vmachine:
break
vmachine = VMachineList.get_by_devicename_and_vpool(device_name, None)
if vmachine:
break
except Exception as ex:
logger.info('Trying to get mgmt center failed for vmachine {0}. {1}'.format(old_name, ex))
if not vmachine:
logger.error('No vmachine found for name {0}'.format(old_name))
return
vpool = vmachine.vpool
mutex = VolatileMutex('{0}_{1}'.format(old_name, vpool.guid if vpool is not None else 'none'))
try:
mutex.acquire(wait=5)
vmachine.name = new_name
vmachine.save()
finally:
mutex.release()
def create_cluster(cluster_name, ip, base_dir, plugins=None, locked=True):
"""
Creates a cluster
:param locked: Indicates whether the create should run in a locked context (e.g. to prevent port conflicts)
:param plugins: Plugins that should be added to the configuration file
:param base_dir: Base directory that should contain the data and tlogs
:param ip: IP address of the first node of the new cluster
:param cluster_name: Name of the cluster
"""
logger.debug('Creating cluster {0} on {1}'.format(cluster_name, ip))
base_dir = base_dir.rstrip('/')
client = SSHClient(ip)
node_name = System.get_my_machine_id(client)
home_dir = ArakoonInstaller.ARAKOON_HOME_DIR.format(
base_dir, cluster_name)
log_dir = ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name)
tlog_dir = ArakoonInstaller.ARAKOON_TLOG_DIR.format(
base_dir, cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(
ip, home_dir, ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir),
cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(
ip, log_dir, ArakoonInstaller.ARAKOON_LOG_DIR.format(''),
cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(
ip, tlog_dir, ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir),
cluster_name)
port_mutex = None
try:
if locked is True:
from ovs.extensions.generic.volatilemutex import VolatileMutex
port_mutex = VolatileMutex(
'arakoon_install_ports_{0}'.format(ip))
port_mutex.acquire(wait=60)
ports = ArakoonInstaller._get_free_ports(client)
config = ArakoonClusterConfig(cluster_name, plugins)
config.nodes.append(
ArakoonNodeConfig(name=node_name,
ip=ip,
client_port=ports[0],
messaging_port=ports[1],
log_dir=log_dir,
home=home_dir,
tlog_dir=tlog_dir))
ArakoonInstaller._deploy(config)
finally:
if port_mutex is not None:
port_mutex.release()
logger.debug('Creating cluster {0} on {1} completed'.format(
cluster_name, ip))
return {'client_port': ports[0], 'messaging_port': ports[1]}
def extend_cluster(master_ip, new_ip, cluster_name, base_dir):
"""
Extends a cluster to a given new node
:param base_dir: Base directory that will hold the db and tlogs
:param cluster_name: Name of the cluster to be extended
:param new_ip: IP address of the node to be added
:param master_ip: IP of one of the already existing nodes
"""
logger.debug('Extending cluster {0} from {1} to {2}'.format(
cluster_name, master_ip, new_ip))
base_dir = base_dir.rstrip('/')
from ovs.extensions.generic.volatilemutex import VolatileMutex
port_mutex = VolatileMutex('arakoon_install_ports_{0}'.format(new_ip))
config = ArakoonClusterConfig(cluster_name)
config.load_config()
client = SSHClient(new_ip)
node_name = System.get_my_machine_id(client)
home_dir = ArakoonInstaller.ARAKOON_HOME_DIR.format(
base_dir, cluster_name)
log_dir = ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name)
tlog_dir = ArakoonInstaller.ARAKOON_TLOG_DIR.format(
base_dir, cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(
new_ip, home_dir,
ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir), cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(
new_ip, log_dir, ArakoonInstaller.ARAKOON_LOG_DIR.format(''),
cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(
new_ip, tlog_dir,
ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir), cluster_name)
try:
port_mutex.acquire(wait=60)
ports = ArakoonInstaller._get_free_ports(client)
if node_name not in [node.name for node in config.nodes]:
config.nodes.append(
ArakoonNodeConfig(name=node_name,
ip=new_ip,
client_port=ports[0],
messaging_port=ports[1],
log_dir=log_dir,
home=home_dir,
tlog_dir=tlog_dir))
ArakoonInstaller._deploy(config)
finally:
port_mutex.release()
logger.debug('Extending cluster {0} from {1} to {2} completed'.format(
cluster_name, master_ip, new_ip))
return {'client_port': ports[0], 'messaging_port': ports[1]}
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
# Load dummy stores
PersistentFactory.store = DummyPersistentStore()
VolatileFactory.store = DummyVolatileStore()
# Replace mocked classes
sys.modules[
'ovs.extensions.storageserver.storagedriver'] = StorageDriverModule
# Import required modules/classes after mocking is done
from ovs.dal.hybrids.backendtype import BackendType
from ovs.dal.hybrids.disk import Disk
from ovs.dal.hybrids.diskpartition import DiskPartition
from ovs.dal.hybrids.failuredomain import FailureDomain
from ovs.dal.hybrids.pmachine import PMachine
from ovs.dal.hybrids.storagerouter import StorageRouter
from ovs.dal.hybrids.vdisk import VDisk
from ovs.dal.hybrids.vmachine import VMachine
from ovs.dal.hybrids.vpool import VPool
from ovs.extensions.generic.volatilemutex import VolatileMutex
from ovs.lib.vmachine import VMachineController
from ovs.lib.vdisk import VDiskController
from ovs.lib.scheduledtask import ScheduledTaskController
# Globalize mocked classes
global Disk
global VDisk
global VMachine
global PMachine
global VPool
global BackendType
global DiskPartition
global FailureDomain
global StorageRouter
global VolatileMutex
global VMachineController
global VDiskController
global ScheduledTaskController
_ = VDisk(), VolatileMutex('dummy'), VMachine(), PMachine(), VPool(), BackendType(), FailureDomain(), \
VMachineController, VDiskController, ScheduledTaskController, StorageRouter(), Disk(), DiskPartition()
# Cleaning storage
VolatileFactory.store.clean()
PersistentFactory.store.clean()
def invalidate_dynamics(self, properties=None):
"""
Invalidates all dynamic property caches. Use with caution, as this action can introduce
a short performance hit.
"""
for dynamic in self._dynamics:
if properties is None or dynamic.name in properties:
key = '{0}_{1}'.format(self._key, dynamic.name)
mutex = VolatileMutex(key)
try:
if dynamic.locked:
mutex.acquire()
self._volatile.delete(key)
finally:
mutex.release()
def new_function(*args, **kw):
"""
Executes the decorated function in a locked context
"""
filemutex = FileMutex('messaging')
try:
filemutex.acquire(wait=5)
mutex = VolatileMutex('messaging')
try:
mutex.acquire(wait=5)
return f(*args, **kw)
finally:
mutex.release()
finally:
filemutex.release()
def new_function(self, request, *args, **kwargs):
"""
Wrapped function
"""
now = time.time()
key = 'ovs_api_limit_{0}.{1}_{2}'.format(
f.__module__, f.__name__, request.META['HTTP_X_REAL_IP'])
client = VolatileFactory.get_client()
mutex = VolatileMutex(key)
try:
mutex.acquire()
rate_info = client.get(key, {'calls': [], 'timeout': None})
active_timeout = rate_info['timeout']
if active_timeout is not None:
if active_timeout > now:
logger.warning(
'Call {0} is being throttled with a wait of {1}'.
format(key, active_timeout - now))
return HttpResponse, {
'error_code':
'rate_limit_timeout',
'error':
'Rate limit timeout ({0}s remaining)'.format(
round(active_timeout - now, 2))
}, 429
else:
rate_info['timeout'] = None
rate_info['calls'] = [
call for call in rate_info['calls'] if call > (now - per)
] + [now]
calls = len(rate_info['calls'])
if calls > amount:
rate_info['timeout'] = now + timeout
client.set(key, rate_info)
logger.warning(
'Call {0} is being throttled with a wait of {1}'.
format(key, timeout))
return HttpResponse, {
'error_code':
'rate_limit_reached',
'error':
'Rate limit reached ({0} in last {1}s)'.format(
calls, per)
}, 429
client.set(key, rate_info)
finally:
mutex.release()
return f(self, request, *args, **kwargs)
def update_value(key, append, value_to_store=None):
"""
Store the specified value in the PersistentFactory
:param key: Key to store the value for
:param append: If True, the specified value will be appended else element at index 0 will be popped
:param value_to_store: Value to append to the list
:return: Updated value
"""
with VolatileMutex(name=key, wait=5):
if persistent_client.exists(key):
val = persistent_client.get(key)
if append is True and value_to_store is not None:
val['values'].append(value_to_store)
elif append is False and len(val['values']) > 0:
val['values'].pop(0)
else:
log_message('Setting initial value for key {0}'.format(
persistent_key))
val = {'mode': mode, 'values': []}
persistent_client.set(key, val)
return val
def update_vmachine_name(instance_id, old_name, new_name):
"""
Update a vMachine name: find vmachine by management center instance id, set new name
:param instance_id: ID for the virtual machine known by management center
:param old_name: Old name of the virtual machine
:param new_name: New name for the virtual machine
"""
vmachine = None
for mgmt_center in MgmtCenterList.get_mgmtcenters():
mgmt = Factory.get_mgmtcenter(mgmt_center=mgmt_center)
try:
machine_info = mgmt.get_vmachine_device_info(instance_id)
file_name = machine_info['file_name']
host_name = machine_info['host_name']
vpool_name = machine_info['vpool_name']
storage_router = StorageRouterList.get_by_name(host_name)
machine_id = storage_router.machine_id
device_name = '{0}/{1}'.format(machine_id, file_name)
vp = VPoolList.get_vpool_by_name(vpool_name)
vmachine = VMachineList.get_by_devicename_and_vpool(
device_name, vp)
if vmachine:
break
vmachine = VMachineList.get_by_devicename_and_vpool(
device_name, None)
if vmachine:
break
except Exception as ex:
logger.info(
'Trying to get mgmt center failed for vmachine {0}. {1}'.
format(old_name, ex))
if not vmachine:
logger.error('No vmachine found for name {0}'.format(old_name))
return
vpool = vmachine.vpool
mutex = VolatileMutex('{0}_{1}'.format(
old_name, vpool.guid if vpool is not None else 'none'))
try:
mutex.acquire(wait=5)
vmachine.name = new_name
vmachine.save()
finally:
mutex.release()
def delete_from_voldrv(volumename, storagedriver_id):
"""
Delete a disk
Triggered by volumedriver messages on the queue
@param volumename: volume id of the disk
"""
_ = storagedriver_id # For logging purposes
disk = VDiskList.get_vdisk_by_volume_id(volumename)
if disk is not None:
mutex = VolatileMutex('{}_{}'.format(volumename, disk.devicename))
try:
mutex.acquire(wait=20)
pmachine = None
try:
pmachine = PMachineList.get_by_storagedriver_id(
disk.storagedriver_id)
except RuntimeError as ex:
if 'could not be found' not in str(ex):
raise
# else: pmachine can't be loaded, because the volumedriver doesn't know about it anymore
if pmachine is not None:
limit = 5
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
hypervisor = Factory.get(pmachine)
exists = hypervisor.file_exists(storagedriver,
disk.devicename)
while limit > 0 and exists is True:
time.sleep(1)
exists = hypervisor.file_exists(
storagedriver, disk.devicename)
limit -= 1
if exists is True:
logger.info(
'Disk {0} still exists, ignoring delete'.format(
disk.devicename))
return
logger.info('Delete disk {}'.format(disk.name))
for mds_service in disk.mds_services:
mds_service.delete()
disk.delete()
finally:
mutex.release()
def __init__(self, guid=None, data=None, datastore_wins=False, volatile=False, _hook=None):
"""
Loads an object with a given guid. If no guid is given, a new object
is generated with a new guid.
* guid: The guid indicating which object should be loaded
* datastoreWins: Optional boolean indicating save conflict resolve management.
** True: when saving, external modified fields will not be saved
** False: when saving, all changed data will be saved, regardless of external updates
** None: in case changed field were also changed externally, an error will be raised
"""
# Initialize super class
super(DataObject, self).__init__()
# Initialize internal fields
self._frozen = False
self._datastore_wins = datastore_wins
self._guid = None # Guid identifier of the object
self._original = {} # Original data copy
self._metadata = {} # Some metadata, mainly used for unit testing
self._data = {} # Internal data storage
self._objects = {} # Internal objects storage
# Initialize public fields
self.dirty = False
self.volatile = volatile
# Worker fields/objects
self._classname = self.__class__.__name__.lower()
self._namespace = 'ovs_data' # Namespace of the object
self._mutex_listcache = VolatileMutex('listcache_{0}'.format(self._classname))
self._mutex_reverseindex = VolatileMutex('reverseindex')
# Rebuild _relation types
hybrid_structure = HybridRunner.get_hybrids()
for relation in self._relations:
if relation.foreign_type is not None:
identifier = Descriptor(relation.foreign_type).descriptor['identifier']
if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']:
relation.foreign_type = Descriptor().load(hybrid_structure[identifier]).get_object()
# Init guid
self._new = False
if guid is None:
self._guid = str(uuid.uuid4())
self._new = True
else:
guid = str(guid).lower()
if re.match('^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', guid) is not None:
self._guid = str(guid)
else:
raise ValueError('The given guid is invalid: {0}'.format(guid))
# Build base keys
self._key = '{0}_{1}_{2}'.format(self._namespace, self._classname, self._guid)
# Worker mutexes
self._mutex_version = VolatileMutex('ovs_dataversion_{0}_{1}'.format(self._classname, self._guid))
# Load data from cache or persistent backend where appropriate
self._volatile = VolatileFactory.get_client()
self._persistent = PersistentFactory.get_client()
self._metadata['cache'] = None
if self._new:
self._data = {}
else:
self._data = self._volatile.get(self._key)
if self._data is None:
Toolbox.log_cache_hit('object_load', False)
self._metadata['cache'] = False
try:
self._data = self._persistent.get(self._key)
except KeyNotFoundException:
raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format(
self.__class__.__name__, self._guid
))
else:
Toolbox.log_cache_hit('object_load', True)
self._metadata['cache'] = True
# Set default values on new fields
for prop in self._properties:
if prop.name not in self._data:
self._data[prop.name] = prop.default
self._add_property(prop)
# Load relations
for relation in self._relations:
if relation.name not in self._data:
if relation.foreign_type is None:
cls = self.__class__
else:
cls = relation.foreign_type
self._data[relation.name] = Descriptor(cls).descriptor
self._add_relation_property(relation)
# Add wrapped properties
for dynamic in self._dynamics:
self._add_dynamic_property(dynamic)
# Load foreign keys
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, info in relations.iteritems():
self._objects[key] = {'info': info,
'data': None}
self._add_list_property(key, info['list'])
# Store original data
self._original = copy.deepcopy(self._data)
if _hook is not None and hasattr(_hook, '__call__'):
_hook()
if not self._new:
# Re-cache the object, if required
if self._metadata['cache'] is False:
# The data wasn't loaded from the cache, so caching is required now
try:
self._mutex_version.acquire(30)
this_version = self._data['_version']
store_version = self._persistent.get(self._key)['_version']
if this_version == store_version:
self._volatile.set(self._key, self._data)
except KeyNotFoundException:
raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format(
self.__class__.__name__, self._guid
))
finally:
self._mutex_version.release()
# Freeze property creation
self._frozen = True
# Optionally, initialize some fields
if data is not None:
for prop in self._properties:
if prop.name in data:
setattr(self, prop.name, data[prop.name])
def new_function(*args, **kwargs):
"""
Wrapped function
:param args: Arguments without default values
:param kwargs: Arguments with default values
"""
def log_message(message, level='info'):
"""
Log a message with some additional information
:param message: Message to log
:param level: Log level
:return: None
"""
if level not in ('info', 'warning', 'debug', 'error'):
raise ValueError(
'Unsupported log level "{0}" specified'.format(level))
complete_message = 'Ensure single {0} mode - ID {1} - {2}'.format(
mode, now, message)
getattr(logger, level)(complete_message)
def update_value(key, append, value_to_store=None):
"""
Store the specified value in the PersistentFactory
:param key: Key to store the value for
:param append: If True, the specified value will be appended else element at index 0 will be popped
:param value_to_store: Value to append to the list
:return: Updated value
"""
with VolatileMutex(name=key, wait=5):
if persistent_client.exists(key):
val = persistent_client.get(key)
if append is True and value_to_store is not None:
val['values'].append(value_to_store)
elif append is False and len(val['values']) > 0:
val['values'].pop(0)
else:
log_message('Setting initial value for key {0}'.format(
persistent_key))
val = {'mode': mode, 'values': []}
persistent_client.set(key, val)
return val
now = '{0}_{1}'.format(
int(time.time()), ''.join(
random.choice(string.ascii_letters + string.digits)
for _ in range(10)))
task_names = [
task_name
] if extra_task_names is None else [task_name] + extra_task_names
persistent_key = '{0}_{1}'.format(ENSURE_SINGLE_KEY, task_name)
persistent_client = PersistentFactory.get_client()
if mode == 'DEFAULT':
with VolatileMutex(persistent_key, wait=5):
for task in task_names:
key_to_check = '{0}_{1}'.format(
ENSURE_SINGLE_KEY, task)
if persistent_client.exists(key_to_check):
log_message(
'Execution of task {0} discarded'.format(
task_name))
return None
log_message('Setting key {0}'.format(persistent_key))
persistent_client.set(persistent_key, {'mode': mode})
try:
output = function(*args, **kwargs)
log_message(
'Task {0} finished successfully'.format(task_name))
return output
finally:
with VolatileMutex(persistent_key, wait=5):
if persistent_client.exists(persistent_key):
log_message(
'Deleting key {0}'.format(persistent_key))
persistent_client.delete(persistent_key)
elif mode == 'CHAINED':
if extra_task_names is not None:
log_message('Extra tasks are not allowed in this mode',
level='error')
raise ValueError(
'Ensure single {0} mode - ID {1} - Extra tasks are not allowed in this mode'
.format(mode, now))
# 1. Create key to be stored in arakoon and update kwargs with args
timeout = kwargs.pop(
'chain_timeout'
) if 'chain_timeout' in kwargs else global_timeout
function_info = inspect.getargspec(function)
kwargs_dict = {}
for index, arg in enumerate(args):
kwargs_dict[function_info.args[index]] = arg
kwargs_dict.update(kwargs)
params_info = 'with params {0}'.format(
kwargs_dict) if kwargs_dict else 'with default params'
# 2. Set the key in arakoon if non-existent
value = update_value(key=persistent_key, append=True)
# 3. Validate whether another job with same params is being executed, skip if so
for item in value['values'][
1:]: # 1st element is processing job, we check all other queued jobs for identical params
if item['kwargs'] == kwargs_dict:
log_message(
'Execution of task {0} {1} discarded because of identical parameters'
.format(task_name, params_info))
return None
log_message('New task {0} {1} scheduled for execution'.format(
task_name, params_info))
update_value(key=persistent_key,
append=True,
value_to_store={
'kwargs': kwargs_dict,
'timestamp': now
})
# 4. Poll the arakoon to see whether this call is the first in list, if so --> execute, else wait
first_element = None
counter = 0
while first_element != now and counter < timeout:
if persistent_client.exists(persistent_key):
value = persistent_client.get(persistent_key)
first_element = value['values'][0]['timestamp']
if first_element == now:
try:
if counter != 0:
current_time = int(time.time())
starting_time = int(now.split('_')[0])
log_message(
'Task {0} {1} had to wait {2} seconds before being able to start'
.format(task_name, params_info,
current_time - starting_time))
output = function(*args, **kwargs)
log_message(
'Task {0} finished successfully'.format(
task_name))
finally:
update_value(key=persistent_key, append=False)
return output
counter += 1
time.sleep(1)
if counter == timeout:
update_value(key=persistent_key, append=False)
log_message(
'Could not start task {0} {1}, within expected time ({2}s). Removed it from queue'
.format(task_name, params_info, timeout),
level='error')
raise EnsureSingleTimeoutReached(
'Ensure single {0} mode - ID {1} - Task {2} could not be started within timeout of {3}s'
.format(mode, now, task_name, timeout))
else:
raise ValueError(
'Unsupported mode "{0}" provided'.format(mode))
def update_from_voldrv(name, storagedriver_id):
"""
This method will update/create a vmachine based on a given vmx/xml file
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
if pmachine.hvtype not in ['VMWARE', 'KVM']:
return
hypervisor = Factory.get(pmachine)
name = hypervisor.clean_vmachine_filename(name)
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
vpool = storagedriver.vpool
machine_ids = [
storagedriver.storagerouter.machine_id
for storagedriver in vpool.storagedrivers
]
if hypervisor.should_process(name, machine_ids=machine_ids):
if pmachine.hvtype == 'VMWARE':
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
vpool = storagedriver.vpool
else:
vpool = None
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
mutex = VolatileMutex('{}_{}'.format(
name, vpool.guid if vpool is not None else 'none'))
try:
mutex.acquire(wait=120)
limit = 5
exists = hypervisor.file_exists(storagedriver, name)
while limit > 0 and exists is False:
time.sleep(1)
exists = hypervisor.file_exists(storagedriver, name)
limit -= 1
if exists is False:
logger.info(
'Could not locate vmachine with name {0} on vpool {1}'.
format(name, vpool.name))
vmachine = VMachineList.get_by_devicename_and_vpool(
name, vpool)
if vmachine is not None:
VMachineController.delete_from_voldrv(
name, storagedriver_id=storagedriver_id)
return
finally:
mutex.release()
try:
mutex.acquire(wait=5)
vmachine = VMachineList.get_by_devicename_and_vpool(
name, vpool)
if not vmachine:
vmachine = VMachine()
vmachine.vpool = vpool
vmachine.pmachine = pmachine
vmachine.status = 'CREATED'
vmachine.devicename = name
vmachine.save()
finally:
mutex.release()
if pmachine.hvtype == 'KVM':
try:
VMachineController.sync_with_hypervisor(
vmachine.guid, storagedriver_id=storagedriver_id)
vmachine.status = 'SYNC'
except:
vmachine.status = 'SYNC_NOK'
vmachine.save()
else:
logger.info('Ignored invalid file {0}'.format(name))
class DataObject(object):
"""
This base class contains all logic to support our multiple backends and the caching
- Storage backends:
- Persistent backend for persistent storage (key-value store)
- Volatile backend for volatile but fast storage (key-value store)
- Storage backends are abstracted and injected into this class, making it possible to use
fake backends
- Features:
- Hybrid property access:
- Persistent backend
- 3rd party component for "live" properties
- Individual cache settings for "live" properties
- 1-n relations with automatic property propagation
- Recursive save
"""
__metaclass__ = MetaClass
#######################
# Attributes
#######################
# Properties that needs to be overwritten by implementation
_properties = [] # Blueprint data of the object type
_dynamics = [] # Timeout of readonly object properties cache
_relations = [] # Blueprint for relations
#######################
# Constructor
#######################
def __new__(cls, *args, **kwargs):
"""
Initializes the class
"""
hybrid_structure = HybridRunner.get_hybrids()
identifier = Descriptor(cls).descriptor['identifier']
if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']:
new_class = Descriptor().load(hybrid_structure[identifier]).get_object()
return super(cls, new_class).__new__(new_class, *args, **kwargs)
return super(DataObject, cls).__new__(cls)
def __init__(self, guid=None, data=None, datastore_wins=False, volatile=False, _hook=None):
"""
Loads an object with a given guid. If no guid is given, a new object
is generated with a new guid.
* guid: The guid indicating which object should be loaded
* datastoreWins: Optional boolean indicating save conflict resolve management.
** True: when saving, external modified fields will not be saved
** False: when saving, all changed data will be saved, regardless of external updates
** None: in case changed field were also changed externally, an error will be raised
"""
# Initialize super class
super(DataObject, self).__init__()
# Initialize internal fields
self._frozen = False
self._datastore_wins = datastore_wins
self._guid = None # Guid identifier of the object
self._original = {} # Original data copy
self._metadata = {} # Some metadata, mainly used for unit testing
self._data = {} # Internal data storage
self._objects = {} # Internal objects storage
# Initialize public fields
self.dirty = False
self.volatile = volatile
# Worker fields/objects
self._classname = self.__class__.__name__.lower()
self._namespace = 'ovs_data' # Namespace of the object
self._mutex_listcache = VolatileMutex('listcache_{0}'.format(self._classname))
self._mutex_reverseindex = VolatileMutex('reverseindex')
# Rebuild _relation types
hybrid_structure = HybridRunner.get_hybrids()
for relation in self._relations:
if relation.foreign_type is not None:
identifier = Descriptor(relation.foreign_type).descriptor['identifier']
if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']:
relation.foreign_type = Descriptor().load(hybrid_structure[identifier]).get_object()
# Init guid
self._new = False
if guid is None:
self._guid = str(uuid.uuid4())
self._new = True
else:
guid = str(guid).lower()
if re.match('^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', guid) is not None:
self._guid = str(guid)
else:
raise ValueError('The given guid is invalid: {0}'.format(guid))
# Build base keys
self._key = '{0}_{1}_{2}'.format(self._namespace, self._classname, self._guid)
# Worker mutexes
self._mutex_version = VolatileMutex('ovs_dataversion_{0}_{1}'.format(self._classname, self._guid))
# Load data from cache or persistent backend where appropriate
self._volatile = VolatileFactory.get_client()
self._persistent = PersistentFactory.get_client()
self._metadata['cache'] = None
if self._new:
self._data = {}
else:
self._data = self._volatile.get(self._key)
if self._data is None:
Toolbox.log_cache_hit('object_load', False)
self._metadata['cache'] = False
try:
self._data = self._persistent.get(self._key)
except KeyNotFoundException:
raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format(
self.__class__.__name__, self._guid
))
else:
Toolbox.log_cache_hit('object_load', True)
self._metadata['cache'] = True
# Set default values on new fields
for prop in self._properties:
if prop.name not in self._data:
self._data[prop.name] = prop.default
self._add_property(prop)
# Load relations
for relation in self._relations:
if relation.name not in self._data:
if relation.foreign_type is None:
cls = self.__class__
else:
cls = relation.foreign_type
self._data[relation.name] = Descriptor(cls).descriptor
self._add_relation_property(relation)
# Add wrapped properties
for dynamic in self._dynamics:
self._add_dynamic_property(dynamic)
# Load foreign keys
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, info in relations.iteritems():
self._objects[key] = {'info': info,
'data': None}
self._add_list_property(key, info['list'])
# Store original data
self._original = copy.deepcopy(self._data)
if _hook is not None and hasattr(_hook, '__call__'):
_hook()
if not self._new:
# Re-cache the object, if required
if self._metadata['cache'] is False:
# The data wasn't loaded from the cache, so caching is required now
try:
self._mutex_version.acquire(30)
this_version = self._data['_version']
store_version = self._persistent.get(self._key)['_version']
if this_version == store_version:
self._volatile.set(self._key, self._data)
except KeyNotFoundException:
raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format(
self.__class__.__name__, self._guid
))
finally:
self._mutex_version.release()
# Freeze property creation
self._frozen = True
# Optionally, initialize some fields
if data is not None:
for prop in self._properties:
if prop.name in data:
setattr(self, prop.name, data[prop.name])
#######################
# Helper methods for dynamic getting and setting
#######################
def _add_property(self, prop):
"""
Adds a simple property to the object
"""
# pylint: disable=protected-access
fget = lambda s: s._get_property(prop)
fset = lambda s, v: s._set_property(prop, v)
# pylint: enable=protected-access
setattr(self.__class__, prop.name, property(fget, fset))
def _add_relation_property(self, relation):
"""
Adds a complex property to the object (hybrids)
"""
# pylint: disable=protected-access
fget = lambda s: s._get_relation_property(relation)
fset = lambda s, v: s._set_relation_property(relation, v)
gget = lambda s: s._get_guid_property(relation)
# pylint: enable=protected-access
setattr(self.__class__, relation.name, property(fget, fset))
setattr(self.__class__, '{0}_guid'.format(relation.name), property(gget))
def _add_list_property(self, attribute, list):
"""
Adds a list (readonly) property to the object
"""
# pylint: disable=protected-access
fget = lambda s: s._get_list_property(attribute)
gget = lambda s: s._get_list_guid_property(attribute)
# pylint: enable=protected-access
setattr(self.__class__, attribute, property(fget))
setattr(self.__class__, ('{0}_guids' if list else '{0}_guid').format(attribute), property(gget))
def _add_dynamic_property(self, dynamic):
"""
Adds a dynamic property to the object
"""
# pylint: disable=protected-access
fget = lambda s: s._get_dynamic_property(dynamic)
# pylint: enable=protected-access
setattr(self.__class__, dynamic.name, property(fget))
# Helper method spporting property fetching
def _get_property(self, prop):
"""
Getter for a simple property
"""
return self._data[prop.name]
def _get_relation_property(self, relation):
"""
Getter for a complex property (hybrid)
It will only load the object once and caches it for the lifetime of this object
"""
attribute = relation.name
if attribute not in self._objects:
descriptor = Descriptor().load(self._data[attribute])
self._objects[attribute] = descriptor.get_object(instantiate=True)
return self._objects[attribute]
def _get_guid_property(self, relation):
"""
Getter for a foreign key property
"""
attribute = relation.name
return self._data[attribute]['guid']
def _get_list_property(self, attribute):
"""
Getter for the list property
It will execute the related query every time to return a list of hybrid objects that
refer to this object. The resulting data will be stored or merged into the cached list
preserving as much already loaded objects as possible
"""
info = self._objects[attribute]['info']
remote_class = Descriptor().load(info['class']).get_object()
remote_key = info['key']
datalist = DataList.get_relation_set(remote_class, remote_key, self.__class__, attribute, self.guid)
if self._objects[attribute]['data'] is None:
self._objects[attribute]['data'] = DataObjectList(datalist.data, remote_class)
else:
self._objects[attribute]['data'].update(datalist.data)
if info['list'] is True:
return self._objects[attribute]['data']
else:
data = self._objects[attribute]['data']
if len(data) > 1:
raise InvalidRelationException('More than one element found in {0}'.format(attribute))
return data[0] if len(data) == 1 else None
def _get_list_guid_property(self, attribute):
"""
Getter for guid list property
"""
dataobjectlist = getattr(self, attribute)
if dataobjectlist is None:
return None
if hasattr(dataobjectlist, '_guids'):
return dataobjectlist._guids
return dataobjectlist.guid
def _get_dynamic_property(self, dynamic):
"""
Getter for dynamic property, wrapping the internal data loading property
in a caching layer
"""
data_loader = getattr(self, '_{0}'.format(dynamic.name))
return self._backend_property(data_loader, dynamic)
# Helper method supporting property setting
def _set_property(self, prop, value):
"""
Setter for a simple property that will validate the type
"""
self.dirty = True
if value is None:
self._data[prop.name] = value
else:
correct, allowed_types, given_type = Toolbox.check_type(value, prop.property_type)
if correct:
self._data[prop.name] = value
else:
raise TypeError('Property {0} allows types {1}. {2} given'.format(
prop.name, str(allowed_types), given_type
))
def _set_relation_property(self, relation, value):
"""
Setter for a complex property (hybrid) that will validate the type
"""
self.dirty = True
attribute = relation.name
if value is None:
self._objects[attribute] = None
self._data[attribute]['guid'] = None
else:
descriptor = Descriptor(value.__class__).descriptor
if descriptor['identifier'] != self._data[attribute]['identifier']:
if descriptor['type'] == self._data[attribute]['type']:
logger.error('Corrupt descriptors: {0} vs {1}'.format(descriptor, self._data[attribute]))
raise TypeError('An invalid type was given: {0} instead of {1}'.format(
descriptor['type'], self._data[attribute]['type']
))
self._objects[attribute] = value
self._data[attribute]['guid'] = value.guid
def __setattr__(self, key, value):
"""
__setattr__ hook that will block creating on the fly new properties, except
the predefined ones
"""
if not hasattr(self, '_frozen') or not self._frozen:
allowed = True
else:
# If our object structure is frozen (which is after __init__), we only allow known
# property updates: items that are in __dict__ and our own blueprinting dicts
allowed = key in self.__dict__ \
or key in (prop.name for prop in self._properties) \
or key in (relation.name for relation in self._relations) \
or key in (dynamic.name for dynamic in self._dynamics)
if allowed:
super(DataObject, self).__setattr__(key, value)
else:
raise RuntimeError('Property {0} does not exist on this object.'.format(key))
#######################
# Saving data to persistent store and invalidating volatile store
#######################
def save(self, recursive=False, skip=None, _hook=None):
"""
Save the object to the persistent backend and clear cache, making use
of the specified conflict resolve settings.
It will also invalidate certain caches if required. For example lists pointing towards this
object
"""
if self.volatile is True:
raise VolatileObjectException()
tries = 0
successful = False
while successful is False:
invalid_fields = []
for prop in self._properties:
if prop.mandatory is True and self._data[prop.name] is None:
invalid_fields.append(prop.name)
for relation in self._relations:
if relation.mandatory is True and self._data[relation.name]['guid'] is None:
invalid_fields.append(relation.name)
if len(invalid_fields) > 0:
raise MissingMandatoryFieldsException('Missing fields on {0}: {1}'.format(self._classname, ', '.join(invalid_fields)))
if recursive:
# Save objects that point to us (e.g. disk.vmachine - if this is disk)
for relation in self._relations:
if relation.name != skip: # disks will be skipped
item = getattr(self, relation.name)
if item is not None:
item.save(recursive=True, skip=relation.foreign_key)
# Save object we point at (e.g. machine.vdisks - if this is machine)
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, info in relations.iteritems():
if key != skip: # machine will be skipped
if info['list'] is True:
for item in getattr(self, key).iterloaded():
item.save(recursive=True, skip=info['key'])
else:
item = getattr(self, key)
if item is not None:
item.save(recursive=True, skip=info['key'])
for relation in self._relations:
if self._data[relation.name]['guid'] is not None:
if relation.foreign_type is None:
cls = self.__class__
else:
cls = relation.foreign_type
_ = cls(self._data[relation.name]['guid'])
try:
data = self._persistent.get(self._key)
except KeyNotFoundException:
if self._new:
data = {'_version': 0}
else:
raise ObjectNotFoundException('{0} with guid \'{1}\' was deleted'.format(
self.__class__.__name__, self._guid
))
changed_fields = []
data_conflicts = []
for attribute in self._data.keys():
if attribute == '_version':
continue
if self._data[attribute] != self._original[attribute]:
# We changed this value
changed_fields.append(attribute)
if attribute in data and self._original[attribute] != data[attribute]:
# Some other process also wrote to the database
if self._datastore_wins is None:
# In case we didn't set a policy, we raise the conflicts
data_conflicts.append(attribute)
elif self._datastore_wins is False:
# If the datastore should not win, we just overwrite the data
data[attribute] = self._data[attribute]
# If the datastore should win, we discard/ignore our change
else:
# Normal scenario, saving data
data[attribute] = self._data[attribute]
elif attribute not in data:
data[attribute] = self._data[attribute]
for attribute in data.keys():
if attribute == '_version':
continue
if attribute not in self._data:
del data[attribute]
if data_conflicts:
raise ConcurrencyException('Got field conflicts while saving {0}. Conflicts: {1}'.format(
self._classname, ', '.join(data_conflicts)
))
# Refresh internal data structure
self._data = copy.deepcopy(data)
caching_keys = []
try:
# First, update reverse index
try:
self._mutex_reverseindex.acquire(60)
for relation in self._relations:
key = relation.name
original_guid = self._original[key]['guid']
new_guid = self._data[key]['guid']
if original_guid != new_guid:
if relation.foreign_type is None:
classname = self.__class__.__name__.lower()
else:
classname = relation.foreign_type.__name__.lower()
if original_guid is not None:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, original_guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is not None:
if relation.foreign_key in reverse_index:
entries = reverse_index[relation.foreign_key]
if self.guid in entries:
entries.remove(self.guid)
reverse_index[relation.foreign_key] = entries
caching_keys.append(reverse_key)
self._volatile.set(reverse_key, reverse_index)
if new_guid is not None:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, new_guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is not None:
if relation.foreign_key in reverse_index:
entries = reverse_index[relation.foreign_key]
if self.guid not in entries:
entries.append(self.guid)
reverse_index[relation.foreign_key] = entries
caching_keys.append(reverse_key)
self._volatile.set(reverse_key, reverse_index)
else:
reverse_index[relation.foreign_key] = [self.guid]
caching_keys.append(reverse_key)
self._volatile.set(reverse_key, reverse_index)
else:
reverse_index = {relation.foreign_key: [self.guid]}
caching_keys.append(reverse_key)
self._volatile.set(reverse_key, reverse_index)
if self._new is True:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(self._classname, self.guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, _ in relations.iteritems():
reverse_index[key] = []
caching_keys.append(reverse_key)
self._volatile.set(reverse_key, reverse_index)
finally:
self._mutex_reverseindex.release()
# Second, invalidate property lists
try:
self._mutex_listcache.acquire(60)
cache_key = '{0}_{1}'.format(DataList.cachelink, self._classname)
cache_list = Toolbox.try_get(cache_key, {})
change = False
for list_key in cache_list.keys():
fields = cache_list[list_key]
if ('__all' in fields and self._new) or list(set(fields) & set(changed_fields)):
change = True
self._volatile.delete(list_key)
del cache_list[list_key]
if change is True:
self._volatile.set(cache_key, cache_list)
self._persistent.set(cache_key, cache_list)
finally:
self._mutex_listcache.release()
if _hook is not None and hasattr(_hook, '__call__'):
_hook()
# Save the data
try:
self._mutex_version.acquire(30)
this_version = self._data['_version']
try:
store_version = self._persistent.get(self._key)['_version']
except KeyNotFoundException:
store_version = 0
if this_version == store_version:
self._data['_version'] = this_version + 1
self._persistent.set(self._key, self._data)
self._volatile.delete(self._key)
successful = True
else:
tries += 1
finally:
self._mutex_version.release()
if tries > 5:
raise SaveRaceConditionException()
except:
for key in caching_keys:
self._volatile.delete(key)
raise
self.invalidate_dynamics()
self._original = copy.deepcopy(self._data)
self.dirty = False
self._new = False
#######################
# Other CRUDs
#######################
def delete(self, abandon=None, _hook=None):
"""
Delete the given object. It also invalidates certain lists
"""
if self.volatile is True:
raise VolatileObjectException()
# Check foreign relations
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, info in relations.iteritems():
items = getattr(self, key)
if info['list'] is True:
if len(items) > 0:
if abandon is not None and (key in abandon or '_all' in abandon):
for item in items.itersafe():
setattr(item, info['key'], None)
try:
item.save()
except ObjectNotFoundException:
pass
else:
multi = 'are {0} items'.format(len(items)) if len(items) > 1 else 'is 1 item'
raise LinkedObjectException('There {0} left in self.{1}'.format(multi, key))
elif items is not None:
# No list (so a 1-to-1 relation), so there should be an object, or None
item = items # More clear naming
if abandon is not None and (key in abandon or '_all' in abandon):
setattr(item, info['key'], None)
try:
item.save()
except ObjectNotFoundException:
pass
else:
raise LinkedObjectException('There is still an item linked in self.{0}'.format(key))
# Delete the object out of the persistent store
try:
self._persistent.delete(self._key)
except KeyNotFoundException:
pass
# First, update reverse index
try:
self._mutex_reverseindex.acquire(60)
for relation in self._relations:
key = relation.name
original_guid = self._original[key]['guid']
if original_guid is not None:
if relation.foreign_type is None:
classname = self.__class__.__name__.lower()
else:
classname = relation.foreign_type.__name__.lower()
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, original_guid)
reverse_index = self._volatile.get(reverse_key)
if reverse_index is not None:
if relation.foreign_key in reverse_index:
entries = reverse_index[relation.foreign_key]
if self.guid in entries:
entries.remove(self.guid)
reverse_index[relation.foreign_key] = entries
self._volatile.set(reverse_key, reverse_index)
self._volatile.delete('ovs_reverseindex_{0}_{1}'.format(self._classname, self.guid))
finally:
self._mutex_reverseindex.release()
# Second, invalidate property lists
try:
self._mutex_listcache.acquire(60)
cache_key = '{0}_{1}'.format(DataList.cachelink, self._classname)
cache_list = Toolbox.try_get(cache_key, {})
change = False
for list_key in cache_list.keys():
fields = cache_list[list_key]
if '__all' in fields:
change = True
self._volatile.delete(list_key)
del cache_list[list_key]
if change is True:
self._volatile.set(cache_key, cache_list)
self._persistent.set(cache_key, cache_list)
finally:
self._mutex_listcache.release()
# Delete the object and its properties out of the volatile store
self.invalidate_dynamics()
self._volatile.delete(self._key)
# Discard all pending changes
def discard(self):
"""
Discard all pending changes, reloading the data from the persistent backend
"""
self.__init__(guid=self._guid,
datastore_wins=self._datastore_wins)
def invalidate_dynamics(self, properties=None):
"""
Invalidates all dynamic property caches. Use with caution, as this action can introduce
a short performance hit.
"""
for dynamic in self._dynamics:
if properties is None or dynamic.name in properties:
key = '{0}_{1}'.format(self._key, dynamic.name)
mutex = VolatileMutex(key)
try:
if dynamic.locked:
mutex.acquire()
self._volatile.delete(key)
finally:
mutex.release()
def invalidate_cached_objects(self):
"""
Invalidates cached objects so they are reloaded when used.
"""
for relation in self._relations:
if relation.name in self._objects:
del self._objects[relation.name]
def export(self):
"""
Exports this object's data for import in another object
"""
data = {}
for prop in self._properties:
data[prop.name] = self._data[prop.name]
return data
def serialize(self, depth=0):
"""
Serializes the internal data, getting rid of certain metadata like descriptors
"""
data = {'guid': self.guid}
for relation in self._relations:
key = relation.name
if depth == 0:
data['{0}_guid'.format(key)] = self._data[key]['guid']
else:
instance = getattr(self, key)
if instance is not None:
data[key] = getattr(self, key).serialize(depth=(depth - 1))
else:
data[key] = None
for prop in self._properties:
data[prop.name] = self._data[prop.name]
for dynamic in self._dynamics:
data[dynamic.name] = getattr(self, dynamic.name)
return data
def copy(self, other_object, include=None, exclude=None, include_relations=False):
"""
Copies all _properties (and optionally _relations) properties over from a given hybrid to
self. One can pass in a list of properties that should be copied, or a list of properties
that should not be copied. Exclude > Include
"""
if include is not None and not isinstance(include, list):
raise TypeError('Argument include should be None or a list of strings')
if exclude is not None and not isinstance(exclude, list):
raise TypeError('Argument exclude should be None or a list of strings')
if self.__class__.__name__ != other_object.__class__.__name__:
raise TypeError('Properties can only be loaded from hybrids of the same type')
all_properties = [prop.name for prop in self._properties]
all_relations = [relation.name for relation in self._relations]
if include:
properties_to_copy = include
else:
properties_to_copy = all_properties
if include_relations:
properties_to_copy += all_relations
if exclude:
properties_to_copy = [p for p in properties_to_copy if p not in exclude]
possible_options = all_properties + (all_relations if include_relations else [])
properties_to_copy = [p for p in properties_to_copy if p in possible_options]
for key in properties_to_copy:
setattr(self, key, getattr(other_object, key))
def updated_on_datastore(self):
"""
Checks whether this object has been modified on the datastore
"""
if self.volatile is True:
return False
this_version = self._data['_version']
cached_object = self._volatile.get(self._key)
if cached_object is None:
try:
backend_version = self._persistent.get(self._key)['_version']
except KeyNotFoundException:
backend_version = -1
else:
backend_version = cached_object['_version']
return this_version != backend_version
#######################
# Properties
#######################
@property
def guid(self):
"""
The primary key of the object
"""
return self._guid
#######################
# Helper methods
#######################
def _backend_property(self, function, dynamic):
"""
Handles the internal caching of dynamic properties
"""
caller_name = dynamic.name
cache_key = '{0}_{1}'.format(self._key, caller_name)
mutex = VolatileMutex(cache_key)
try:
cached_data = self._volatile.get(cache_key)
if cached_data is None:
if dynamic.locked:
mutex.acquire()
cached_data = self._volatile.get(cache_key)
if cached_data is None:
function_info = inspect.getargspec(function)
if 'dynamic' in function_info.args:
cached_data = function(dynamic=dynamic) # Load data from backend
else:
cached_data = function()
if cached_data is not None:
correct, allowed_types, given_type = Toolbox.check_type(cached_data, dynamic.return_type)
if not correct:
raise TypeError('Dynamic property {0} allows types {1}. {2} given'.format(
caller_name, str(allowed_types), given_type
))
if dynamic.timeout > 0:
self._volatile.set(cache_key, cached_data, dynamic.timeout)
return cached_data
finally:
mutex.release()
def __str__(self):
"""
The string representation of a DataObject is the serialized value
"""
return json.dumps(self.serialize(), indent=4, cls=DataObjectAttributeEncoder)
def __hash__(self):
"""
Defines a hashing equivalent for a given object. The key (object type and guid) is considered to be identifying
"""
return hash(self._key)
def __eq__(self, other):
"""
Checks whether two objects are the same.
"""
if not Descriptor.isinstance(other, self.__class__):
return False
return self.__hash__() == other.__hash__()
def __ne__(self, other):
"""
Checks whether to objects are not the same.
"""
if not Descriptor.isinstance(other, self.__class__):
return True
return not self.__eq__(other)
@staticmethod
def enumerator(name, items):
"""
Generates an enumerator
"""
class Enumerator(dict):
def __init__(self, *args, **kwargs):
super(Enumerator, self).__init__(*args, **kwargs)
if isinstance(items, list):
enumerator = Enumerator(zip(items, items))
elif isinstance(items, dict):
enumerator = Enumerator(**items)
else:
raise ValueError("Argument 'items' should be a list or a dict. A '{0}' was given".format(type(items)))
enumerator.__name__ = name
for item in enumerator:
setattr(enumerator, item, enumerator[item])
return enumerator
def update_vmachine_config(vmachine, vm_object, pmachine=None):
"""
Update a vMachine configuration with a given vMachine configuration
"""
try:
vdisks_synced = 0
if vmachine.name is None:
MessageController.fire(MessageController.Type.EVENT,
{'type': 'vmachine_created',
'metadata': {'name': vm_object['name']}})
elif vmachine.name != vm_object['name']:
MessageController.fire(MessageController.Type.EVENT,
{'type': 'vmachine_renamed',
'metadata': {'old_name': vmachine.name,
'new_name': vm_object['name']}})
if pmachine is not None:
vmachine.pmachine = pmachine
vmachine.name = vm_object['name']
vmachine.hypervisor_id = vm_object['id']
vmachine.devicename = vm_object['backing']['filename']
vmachine.save()
# Updating and linking disks
storagedrivers = StorageDriverList.get_storagedrivers()
datastores = dict([('{0}:{1}'.format(storagedriver.storage_ip, storagedriver.mountpoint), storagedriver) for storagedriver in storagedrivers])
vdisk_guids = []
mutex = VolatileMutex('{0}_{1}'.format(vmachine.name, vmachine.devicename))
for disk in vm_object['disks']:
ensure_safety = False
if disk['datastore'] in vm_object['datastores']:
datastore = vm_object['datastores'][disk['datastore']]
if datastore in datastores:
try:
mutex.acquire(wait=10)
vdisk = VDiskList.get_by_devicename_and_vpool(disk['filename'], datastores[datastore].vpool)
if vdisk is None:
# The disk couldn't be located, but is in our datastore. We might be in a recovery scenario
vdisk = VDisk()
vdisk.vpool = datastores[datastore].vpool
vdisk.reload_client()
vdisk.devicename = disk['filename']
vdisk.volume_id = vdisk.storagedriver_client.get_volume_id(str(disk['backingfilename']))
vdisk.size = vdisk.info['volume_size']
# Create the disk in a locked context, but don't execute long running-task in same context
vdisk.save()
ensure_safety = True
finally:
mutex.release()
if ensure_safety:
MDSServiceController.ensure_safety(vdisk)
# Update the disk with information from the hypervisor
if vdisk.vmachine is None:
MessageController.fire(MessageController.Type.EVENT,
{'type': 'vdisk_attached',
'metadata': {'vmachine_name': vmachine.name,
'vdisk_name': disk['name']}})
vdisk.vmachine = vmachine
vdisk.name = disk['name']
vdisk.order = disk['order']
vdisk.save()
vdisk_guids.append(vdisk.guid)
vdisks_synced += 1
for vdisk in vmachine.vdisks:
if vdisk.guid not in vdisk_guids:
MessageController.fire(MessageController.Type.EVENT,
{'type': 'vdisk_detached',
'metadata': {'vmachine_name': vmachine.name,
'vdisk_name': vdisk.name}})
vdisk.vmachine = None
vdisk.save()
logger.info('Updating vMachine finished (name {0}, {1} vdisks (re)linked)'.format(
vmachine.name, vdisks_synced
))
except Exception as ex:
logger.info('Error during vMachine update: {0}'.format(str(ex)))
raise
class DistributedScheduler(Scheduler):
"""
Distributed scheduler that can run on multiple nodes at the same time.
"""
TIMEOUT = 60 * 30
def __init__(self, *args, **kwargs):
"""
Initializes the distributed scheduler
"""
self._persistent = PersistentFactory.get_client()
self._namespace = 'ovs_celery_beat'
self._mutex = VolatileMutex('celery_beat')
self._has_lock = False
super(DistributedScheduler, self).__init__(*args, **kwargs)
logger.debug('DS init')
def setup_schedule(self):
"""
Setups the schedule
"""
logger.debug('DS setting up schedule')
self._load_schedule()
self.merge_inplace(DistributedScheduler._discover_schedule())
self.install_default_entries(self.schedule)
for entry in self.schedule:
logger.debug('* {0}'.format(entry))
logger.debug('DS setting up schedule - done')
@staticmethod
def _discover_schedule():
schedule = {}
path = os.path.join(os.path.dirname(__file__), 'lib')
for filename in os.listdir(path):
if os.path.isfile(os.path.join(
path, filename)) and filename.endswith(
'.py') and filename != '__init__.py':
name = filename.replace('.py', '')
module = imp.load_source(name, os.path.join(path, filename))
for member in inspect.getmembers(module):
if inspect.isclass(member[1]) \
and member[1].__module__ == name \
and 'object' in [base.__name__ for base in member[1].__bases__]:
for submember in inspect.getmembers(member[1]):
if hasattr(submember[1],
'schedule') and isinstance(
submember[1].schedule, crontab):
schedule[submember[1].name] = {
'task': submember[1].name,
'schedule': submember[1].schedule,
'args': []
}
return schedule
def _load_schedule(self):
"""
Loads the most recent schedule from the persistent store
"""
self.schedule = {}
try:
logger.debug('DS loading schedule entries')
self._mutex.acquire(wait=10)
try:
self.schedule = cPickle.loads(
str(
self._persistent.get('{0}_entries'.format(
self._namespace))))
except:
# In case an exception occurs during loading the schedule, it is ignored and the default schedule
# will be used/restored.
pass
finally:
self._mutex.release()
def sync(self):
if self._has_lock is True:
try:
logger.debug('DS syncing schedule entries')
self._mutex.acquire(wait=10)
self._persistent.set('{0}_entries'.format(self._namespace),
cPickle.dumps(self.schedule))
finally:
self._mutex.release()
else:
logger.debug('DS skipping sync: lock is not ours')
def tick(self):
"""
Runs one iteration of the scheduler. This is guarded with a distributed lock
"""
self._has_lock = False
try:
logger.debug('DS executing tick')
self._mutex.acquire(wait=10)
node_now = current_app._get_current_object().now()
node_timestamp = time.mktime(node_now.timetuple())
node_name = System.get_my_machine_id()
try:
lock = self._persistent.get('{0}_lock'.format(self._namespace))
except KeyNotFoundException:
lock = None
if lock is None:
# There is no lock yet, so the lock is acquired
self._has_lock = True
logger.debug('DS there was no lock in tick')
else:
if lock['name'] == node_name:
# The current node holds the lock
logger.debug('DS keeps own lock')
self._has_lock = True
elif node_timestamp - lock[
'timestamp'] > DistributedScheduler.TIMEOUT:
# The current lock is timed out, so the lock is stolen
logger.debug('DS last lock refresh is {0}s old'.format(
node_timestamp - lock['timestamp']))
logger.debug('DS stealing lock from {0}'.format(
lock['name']))
self._load_schedule()
self._has_lock = True
else:
logger.debug('DS lock is not ours')
if self._has_lock is True:
lock = {'name': node_name, 'timestamp': node_timestamp}
logger.debug('DS refreshing lock')
self._persistent.set('{0}_lock'.format(self._namespace), lock)
finally:
self._mutex.release()
if self._has_lock is True:
logger.debug('DS executing tick workload')
remaining_times = []
try:
for entry in self.schedule.itervalues():
next_time_to_run = self.maybe_due(entry, self.publisher)
if next_time_to_run:
remaining_times.append(next_time_to_run)
except RuntimeError:
pass
logger.debug('DS executing tick workload - done')
return min(remaining_times + [self.max_interval])
else:
return self.max_interval
class DistributedScheduler(Scheduler):
"""
Distributed scheduler that can run on multiple nodes at the same time.
"""
TIMEOUT = 60 * 30
def __init__(self, *args, **kwargs):
"""
Initializes the distributed scheduler
"""
self._persistent = PersistentFactory.get_client()
self._namespace = 'ovs_celery_beat'
self._mutex = VolatileMutex('celery_beat')
self._has_lock = False
super(DistributedScheduler, self).__init__(*args, **kwargs)
logger.debug('DS init')
def setup_schedule(self):
"""
Setups the schedule
"""
logger.debug('DS setting up schedule')
self._load_schedule()
self.merge_inplace(self.app.conf.CELERYBEAT_SCHEDULE)
self.install_default_entries(self.schedule)
logger.debug('DS setting up schedule - done')
def _load_schedule(self):
"""
Loads the most recent schedule from the persistent store
"""
self.schedule = {}
try:
logger.debug('DS loading schedule entries')
self._mutex.acquire(wait=10)
try:
self.schedule = cPickle.loads(
str(self._persistent.get('{0}_entries'.format(self._namespace))))
except:
# In case an exception occurs during loading the schedule, it is ignored and the default schedule
# will be used/restored.
pass
finally:
self._mutex.release()
def sync(self):
if self._has_lock is True:
try:
logger.debug('DS syncing schedule entries')
self._mutex.acquire(wait=10)
self._persistent.set('{0}_entries'.format(
self._namespace), cPickle.dumps(self.schedule))
finally:
self._mutex.release()
else:
logger.debug('DS skipping sync: lock is not ours')
def tick(self):
"""
Runs one iteration of the scheduler. This is guarded with a distributed lock
"""
self._has_lock = False
try:
logger.debug('DS executing tick')
self._mutex.acquire(wait=10)
node_now = current_app._get_current_object().now()
node_timestamp = time.mktime(node_now.timetuple())
node_name = Configuration.get('ovs.core.uniqueid')
try:
lock = self._persistent.get('{0}_lock'.format(self._namespace))
except KeyNotFoundException:
lock = None
if lock is None:
# There is no lock yet, so the lock is acquired
self._has_lock = True
logger.debug('DS there was no lock in tick')
else:
if lock['name'] == node_name:
# The current node holds the lock
logger.debug('DS keeps own lock')
self._has_lock = True
elif node_timestamp - lock['timestamp'] > DistributedScheduler.TIMEOUT:
# The current lock is timed out, so the lock is stolen
logger.debug('DS last lock refresh is {0}s old'.format(
node_timestamp - lock['timestamp']))
logger.debug(
'DS stealing lock from {0}'.format(lock['name']))
self._load_schedule()
self._has_lock = True
else:
logger.debug('DS lock is not ours')
if self._has_lock is True:
lock = {'name': node_name,
'timestamp': node_timestamp}
logger.debug('DS refreshing lock')
self._persistent.set('{0}_lock'.format(self._namespace), lock)
finally:
self._mutex.release()
if self._has_lock is True:
logger.debug('DS executing tick workload')
remaining_times = []
try:
for entry in self.schedule.itervalues():
next_time_to_run = self.maybe_due(entry, self.publisher)
if next_time_to_run:
remaining_times.append(next_time_to_run)
except RuntimeError:
pass
logger.debug('DS executing tick workload - done')
return min(remaining_times + [self.max_interval])
else:
return self.max_interval
def _load(self):
"""
Tries to load the result for the given key from the volatile cache, or executes the query
if not yet available. Afterwards (if a key is given), the result will be (re)cached
"""
self.data = self._volatile.get(self._key) if self._key is not None else None
if self.data is None:
# The query should be a dictionary:
# {'object': Disk, # Object on which the query should be executed
# 'data' : DataList.select.XYZ, # The requested result
# 'query' : <query>} # The actual query
# Where <query> is a query(group) dictionary:
# {'type' : DataList.where_operator.ABC, # Whether the items should be AND/OR
# 'items': <items>} # The items in the group
# Where the <items> is any combination of one or more <filter> or <query>
# A <filter> tuple example:
# (<field>, DataList.operator.GHI, <value>) # For example EQUALS
# The field is any property you would also find on the given object. In case of
# properties, you can dot as far as you like. This means you can combine AND and OR
# in any possible combination
Toolbox.log_cache_hit('datalist', False)
hybrid_structure = HybridRunner.get_hybrids()
items = self._query['query']['items']
query_type = self._query['query']['type']
query_data = self._query['data']
query_object = self._query['object']
query_object_id = Descriptor(query_object).descriptor['identifier']
if query_object_id in hybrid_structure and query_object_id != hybrid_structure[query_object_id]['identifier']:
query_object = Descriptor().load(hybrid_structure[query_object_id]).get_object()
invalidations = {query_object.__name__.lower(): ['__all']}
DataList._build_invalidations(invalidations, query_object, items)
for class_name in invalidations:
key = '{0}_{1}'.format(DataList.cachelink, class_name)
mutex = VolatileMutex('listcache_{0}'.format(class_name))
try:
mutex.acquire(60)
cache_list = Toolbox.try_get(key, {})
current_fields = cache_list.get(self._key, [])
current_fields = list(set(current_fields + ['__all'] + invalidations[class_name]))
cache_list[self._key] = current_fields
self._volatile.set(key, cache_list)
self._persistent.set(key, cache_list)
finally:
mutex.release()
self.from_cache = False
namespace = query_object()._namespace
name = query_object.__name__.lower()
guids = DataList.get_pks(namespace, name)
if query_data == DataList.select.COUNT:
self.data = 0
else:
self.data = []
for guid in guids:
try:
instance = query_object(guid)
if query_type == DataList.where_operator.AND:
include = self._exec_and(instance, items)
elif query_type == DataList.where_operator.OR:
include = self._exec_or(instance, items)
else:
raise NotImplementedError('The given operator is not yet implemented.')
if include:
if query_data == DataList.select.COUNT:
self.data += 1
elif query_data == DataList.select.GUIDS:
self.data.append(guid)
else:
raise NotImplementedError('The given selector type is not implemented')
except ObjectNotFoundException:
pass
if 'post_query' in DataList.test_hooks:
DataList.test_hooks['post_query'](self)
if self._key is not None and len(guids) > 0 and self._can_cache:
invalidated = False
for class_name in invalidations:
key = '{0}_{1}'.format(DataList.cachelink, class_name)
cache_list = Toolbox.try_get(key, {})
if self._key not in cache_list:
invalidated = True
# If the key under which the list should be saved was already invalidated since the invalidations
# were saved, the returned list is most likely outdated. This is OK for this result, but the list
# won't get cached
if invalidated is False:
self._volatile.set(self._key, self.data, 300 + randint(0, 300)) # Cache between 5 and 10 minutes
else:
Toolbox.log_cache_hit('datalist', True)
self.from_cache = True
return self
def clone(machineguid, timestamp, name):
"""
Clone a vmachine using the disk snapshot based on a snapshot timestamp
@param machineguid: guid of the machine to clone
@param timestamp: timestamp of the disk snapshots to use for the clone
@param name: name for the new machine
"""
machine = VMachine(machineguid)
timestamp = str(timestamp)
if timestamp not in (snap['timestamp'] for snap in machine.snapshots):
raise RuntimeError(
'Invalid timestamp provided, not a valid snapshot of this vmachine.'
)
vpool = None
storagerouter = None
if machine.pmachine is not None and machine.pmachine.hvtype == 'VMWARE':
for vdisk in machine.vdisks:
if vdisk.vpool is not None:
vpool = vdisk.vpool
break
for vdisk in machine.vdisks:
if vdisk.storagerouter_guid:
storagerouter = StorageRouter(vdisk.storagerouter_guid)
break
hv = Factory.get(machine.pmachine)
vm_path = hv.get_vmachine_path(
name,
storagerouter.machine_id if storagerouter is not None else '')
# mutex in sync_with_hypervisor uses "None" for KVM hvtype
mutex = VolatileMutex('{0}_{1}'.format(
hv.clean_vmachine_filename(vm_path),
vpool.guid if vpool is not None else 'none'))
disks = {}
for snapshot in machine.snapshots:
if snapshot['timestamp'] == timestamp:
for diskguid, snapshotguid in snapshot['snapshots'].iteritems(
):
disks[diskguid] = snapshotguid
try:
mutex.acquire(wait=120)
new_machine = VMachine()
new_machine.copy(machine)
new_machine.name = name
new_machine.devicename = hv.clean_vmachine_filename(vm_path)
new_machine.pmachine = machine.pmachine
new_machine.save()
finally:
mutex.release
new_disk_guids = []
vm_disks = []
mountpoint = None
disks_by_order = sorted(machine.vdisks, key=lambda x: x.order)
try:
for currentDisk in disks_by_order:
if machine.is_vtemplate and currentDisk.templatesnapshot:
snapshotid = currentDisk.templatesnapshot
else:
snapshotid = disks[currentDisk.guid]
prefix = '%s-clone' % currentDisk.name
result = VDiskController.clone(
diskguid=currentDisk.guid,
snapshotid=snapshotid,
devicename=prefix,
pmachineguid=new_machine.pmachine_guid,
machinename=new_machine.name,
machineguid=new_machine.guid)
new_disk_guids.append(result['diskguid'])
mountpoint = StorageDriverList.get_by_storagedriver_id(
currentDisk.storagedriver_id).mountpoint
vm_disks.append(result)
except Exception as ex:
logger.error('Failed to clone disks. {0}'.format(ex))
VMachineController.delete(machineguid=new_machine.guid)
raise
try:
result = hv.clone_vm(machine.hypervisor_id, name, vm_disks,
mountpoint)
except Exception as ex:
logger.error('Failed to clone vm. {0}'.format(ex))
VMachineController.delete(machineguid=new_machine.guid)
raise
try:
mutex.acquire(wait=120)
new_machine.hypervisor_id = result
new_machine.save()
finally:
mutex.release()
return new_machine.guid
def update_from_voldrv(name, storagedriver_id):
"""
This method will update/create a vmachine based on a given vmx/xml file
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
if pmachine.hvtype not in ['VMWARE', 'KVM']:
return
hypervisor = Factory.get(pmachine)
name = hypervisor.clean_vmachine_filename(name)
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
vpool = storagedriver.vpool
machine_ids = [storagedriver.storagerouter.machine_id for storagedriver in vpool.storagedrivers]
if hypervisor.should_process(name, machine_ids=machine_ids):
if pmachine.hvtype == 'VMWARE':
storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id)
vpool = storagedriver.vpool
else:
vpool = None
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
mutex = VolatileMutex('{0}_{1}'.format(name, vpool.guid if vpool is not None else 'none'))
try:
mutex.acquire(wait=120)
limit = 5
exists = hypervisor.file_exists(storagedriver, name)
while limit > 0 and exists is False:
time.sleep(1)
exists = hypervisor.file_exists(storagedriver, name)
limit -= 1
if exists is False:
logger.info('Could not locate vmachine with name {0} on vpool {1}'.format(name, vpool.name))
vmachine = VMachineList.get_by_devicename_and_vpool(name, vpool)
if vmachine is not None:
VMachineController.delete_from_voldrv(name, storagedriver_id=storagedriver_id)
return
finally:
mutex.release()
try:
mutex.acquire(wait=5)
vmachine = VMachineList.get_by_devicename_and_vpool(name, vpool)
if not vmachine:
vmachine = VMachine()
vmachine.vpool = vpool
vmachine.pmachine = pmachine
vmachine.status = 'CREATED'
vmachine.devicename = name
vmachine.save()
finally:
mutex.release()
if pmachine.hvtype == 'KVM':
try:
VMachineController.sync_with_hypervisor(vmachine.guid, storagedriver_id=storagedriver_id)
vmachine.status = 'SYNC'
except:
vmachine.status = 'SYNC_NOK'
vmachine.save()
else:
logger.info('Ignored invalid file {0}'.format(name))
class DistributedScheduler(Scheduler):
"""
Distributed scheduler that can run on multiple nodes at the same time.
"""
TIMEOUT = 60 * 30
def __init__(self, *args, **kwargs):
"""
Initializes the distributed scheduler
"""
self._persistent = PersistentFactory.get_client()
self._namespace = 'ovs_celery_beat'
self._mutex = VolatileMutex('celery_beat')
self._has_lock = False
super(DistributedScheduler, self).__init__(*args, **kwargs)
logger.debug('DS init')
def setup_schedule(self):
"""
Setups the schedule
"""
logger.debug('DS setting up schedule')
self._load_schedule()
self.merge_inplace(DistributedScheduler._discover_schedule())
self.install_default_entries(self.schedule)
for entry in self.schedule:
logger.debug('* {0}'.format(entry))
logger.debug('DS setting up schedule - done')
@staticmethod
def _discover_schedule():
schedule = {}
path = os.path.join(os.path.dirname(__file__), 'lib')
for filename in os.listdir(path):
if os.path.isfile(os.path.join(path, filename)) and filename.endswith('.py') and filename != '__init__.py':
name = filename.replace('.py', '')
module = imp.load_source(name, os.path.join(path, filename))
for member in inspect.getmembers(module):
if inspect.isclass(member[1]) \
and member[1].__module__ == name \
and 'object' in [base.__name__ for base in member[1].__bases__]:
for submember in inspect.getmembers(member[1]):
if hasattr(submember[1], 'schedule') and isinstance(submember[1].schedule, crontab):
schedule[submember[1].name] = {'task': submember[1].name,
'schedule': submember[1].schedule,
'args': []}
return schedule
def _load_schedule(self):
"""
Loads the most recent schedule from the persistent store
"""
self.schedule = {}
try:
logger.debug('DS loading schedule entries')
self._mutex.acquire(wait=10)
try:
self.schedule = cPickle.loads(str(self._persistent.get('{0}_entries'.format(self._namespace))))
except:
# In case an exception occurs during loading the schedule, it is ignored and the default schedule
# will be used/restored.
pass
finally:
self._mutex.release()
def sync(self):
if self._has_lock is True:
try:
logger.debug('DS syncing schedule entries')
self._mutex.acquire(wait=10)
self._persistent.set('{0}_entries'.format(
self._namespace), cPickle.dumps(self.schedule))
finally:
self._mutex.release()
else:
logger.debug('DS skipping sync: lock is not ours')
def tick(self):
"""
Runs one iteration of the scheduler. This is guarded with a distributed lock
"""
self._has_lock = False
try:
logger.debug('DS executing tick')
self._mutex.acquire(wait=10)
node_now = current_app._get_current_object().now()
node_timestamp = time.mktime(node_now.timetuple())
node_name = System.get_my_machine_id()
try:
lock = self._persistent.get('{0}_lock'.format(self._namespace))
except KeyNotFoundException:
lock = None
if lock is None:
# There is no lock yet, so the lock is acquired
self._has_lock = True
logger.debug('DS there was no lock in tick')
else:
if lock['name'] == node_name:
# The current node holds the lock
logger.debug('DS keeps own lock')
self._has_lock = True
elif node_timestamp - lock['timestamp'] > DistributedScheduler.TIMEOUT:
# The current lock is timed out, so the lock is stolen
logger.debug('DS last lock refresh is {0}s old'.format(
node_timestamp - lock['timestamp']))
logger.debug(
'DS stealing lock from {0}'.format(lock['name']))
self._load_schedule()
self._has_lock = True
else:
logger.debug('DS lock is not ours')
if self._has_lock is True:
lock = {'name': node_name,
'timestamp': node_timestamp}
logger.debug('DS refreshing lock')
self._persistent.set('{0}_lock'.format(self._namespace), lock)
finally:
self._mutex.release()
if self._has_lock is True:
logger.debug('DS executing tick workload')
remaining_times = []
try:
for entry in self.schedule.itervalues():
next_time_to_run = self.maybe_due(entry, self.publisher)
if next_time_to_run:
remaining_times.append(next_time_to_run)
except RuntimeError:
pass
logger.debug('DS executing tick workload - done')
return min(remaining_times + [self.max_interval])
else:
return self.max_interval
def clone(machineguid, timestamp, name):
"""
Clone a vmachine using the disk snapshot based on a snapshot timestamp
@param machineguid: guid of the machine to clone
@param timestamp: timestamp of the disk snapshots to use for the clone
@param name: name for the new machine
"""
machine = VMachine(machineguid)
timestamp = str(timestamp)
if timestamp not in (snap['timestamp'] for snap in machine.snapshots):
raise RuntimeError('Invalid timestamp provided, not a valid snapshot of this vmachine.')
vpool = None
storagerouter = None
if machine.pmachine is not None and machine.pmachine.hvtype == 'VMWARE':
for vdisk in machine.vdisks:
if vdisk.vpool is not None:
vpool = vdisk.vpool
break
for vdisk in machine.vdisks:
if vdisk.storagerouter_guid:
storagerouter = StorageRouter(vdisk.storagerouter_guid)
break
hv = Factory.get(machine.pmachine)
vm_path = hv.get_vmachine_path(name, storagerouter.machine_id if storagerouter is not None else '')
# mutex in sync_with_hypervisor uses "None" for KVM hvtype
mutex = VolatileMutex('{0}_{1}'.format(hv.clean_vmachine_filename(vm_path), vpool.guid if vpool is not None else 'none'))
disks = {}
for snapshot in machine.snapshots:
if snapshot['timestamp'] == timestamp:
for diskguid, snapshotguid in snapshot['snapshots'].iteritems():
disks[diskguid] = snapshotguid
try:
mutex.acquire(wait=120)
new_machine = VMachine()
new_machine.copy(machine)
new_machine.name = name
new_machine.devicename = hv.clean_vmachine_filename(vm_path)
new_machine.pmachine = machine.pmachine
new_machine.save()
finally:
mutex.release
new_disk_guids = []
vm_disks = []
mountpoint = None
disks_by_order = sorted(machine.vdisks, key=lambda x: x.order)
try:
for currentDisk in disks_by_order:
if machine.is_vtemplate and currentDisk.templatesnapshot:
snapshotid = currentDisk.templatesnapshot
else:
snapshotid = disks[currentDisk.guid]
prefix = '%s-clone' % currentDisk.name
result = VDiskController.clone(diskguid=currentDisk.guid,
snapshotid=snapshotid,
devicename=prefix,
pmachineguid=new_machine.pmachine_guid,
machinename=new_machine.name,
machineguid=new_machine.guid)
new_disk_guids.append(result['diskguid'])
mountpoint = StorageDriverList.get_by_storagedriver_id(currentDisk.storagedriver_id).mountpoint
vm_disks.append(result)
except Exception as ex:
logger.error('Failed to clone disks. {0}'.format(ex))
VMachineController.delete(machineguid=new_machine.guid)
raise
try:
result = hv.clone_vm(machine.hypervisor_id, name, vm_disks, mountpoint)
except Exception as ex:
logger.error('Failed to clone vm. {0}'.format(ex))
VMachineController.delete(machineguid=new_machine.guid)
raise
try:
mutex.acquire(wait=120)
new_machine.hypervisor_id = result
new_machine.save()
finally:
mutex.release()
return new_machine.guid
def get_relation_set(remote_class, remote_key, own_class, own_key,
own_guid):
"""
This method will get a DataList for a relation.
On a cache miss, the relation DataList will be rebuild and due to the nature of the full table scan, it will
update all relations in the mean time.
"""
# Example:
# * remote_class = vDisk
# * remote_key = vmachine
# * own_class = vMachine
# * own_key = vdisks
# Called to load the vMachine.vdisks list (resulting in a possible scan of vDisk objects)
# * own_guid = this vMachine object's guid
volatile = VolatileFactory.get_client()
own_name = own_class.__name__.lower()
datalist = DataList({},
'{0}_{1}_{2}'.format(own_name, own_guid,
remote_key),
load=False)
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid)
# Check whether the requested information is available in cache
reverse_index = volatile.get(reverse_key)
if reverse_index is not None and own_key in reverse_index:
Toolbox.log_cache_hit('datalist', True)
datalist.data = reverse_index[own_key]
datalist.from_cache = True
return datalist
Toolbox.log_cache_hit('datalist', False)
mutex = VolatileMutex('reverseindex')
remote_name = remote_class.__name__.lower()
blueprint_object = remote_class() # vDisk object
foreign_guids = {}
remote_namespace = blueprint_object._namespace
remote_keys = DataList.get_pks(remote_namespace, remote_name)
handled_flows = []
for guid in remote_keys:
try:
instance = remote_class(guid)
for relation in blueprint_object._relations: # E.g. vmachine or vpool relation
if relation.foreign_type is None:
classname = remote_name
foreign_namespace = blueprint_object._namespace
else:
classname = relation.foreign_type.__name__.lower()
foreign_namespace = relation.foreign_type()._namespace
if classname not in foreign_guids:
foreign_guids[classname] = DataList.get_pks(
foreign_namespace, classname)
flow = '{0}_{1}'.format(classname, relation.foreign_key)
if flow not in handled_flows:
handled_flows.append(flow)
try:
mutex.acquire(60)
for foreign_guid in foreign_guids[classname]:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(
classname, foreign_guid)
reverse_index = volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
if relation.foreign_key not in reverse_index:
reverse_index[relation.foreign_key] = []
volatile.set(reverse_key, reverse_index)
finally:
mutex.release()
key = getattr(instance, '{0}_guid'.format(relation.name))
if key is not None:
try:
mutex.acquire(60)
reverse_index = volatile.get(
'ovs_reverseindex_{0}_{1}'.format(
classname, key))
if reverse_index is None:
reverse_index = {}
if relation.foreign_key not in reverse_index:
reverse_index[relation.foreign_key] = []
if guid not in reverse_index[relation.foreign_key]:
reverse_index[relation.foreign_key].append(
guid)
volatile.set(
'ovs_reverseindex_{0}_{1}'.format(
classname, key), reverse_index)
finally:
mutex.release()
except ObjectNotFoundException:
pass
try:
mutex.acquire(60)
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid)
reverse_index = volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
if own_key not in reverse_index:
reverse_index[own_key] = []
volatile.set(reverse_key, reverse_index)
datalist.data = reverse_index[own_key]
datalist.from_cache = False
finally:
mutex.release()
return datalist
def _load(self):
"""
Tries to load the result for the given key from the volatile cache, or executes the query
if not yet available. Afterwards (if a key is given), the result will be (re)cached
"""
self.data = self._volatile.get(
self._key) if self._key is not None else None
if self.data is None:
# The query should be a dictionary:
# {'object': Disk, # Object on which the query should be executed
# 'data' : DataList.select.XYZ, # The requested result
# 'query' : <query>} # The actual query
# Where <query> is a query(group) dictionary:
# {'type' : DataList.where_operator.ABC, # Whether the items should be AND/OR
# 'items': <items>} # The items in the group
# Where the <items> is any combination of one or more <filter> or <query>
# A <filter> tuple example:
# (<field>, DataList.operator.GHI, <value>) # For example EQUALS
# The field is any property you would also find on the given object. In case of
# properties, you can dot as far as you like. This means you can combine AND and OR
# in any possible combination
Toolbox.log_cache_hit('datalist', False)
hybrid_structure = HybridRunner.get_hybrids()
items = self._query['query']['items']
query_type = self._query['query']['type']
query_data = self._query['data']
query_object = self._query['object']
query_object_id = Descriptor(query_object).descriptor['identifier']
if query_object_id in hybrid_structure and query_object_id != hybrid_structure[
query_object_id]['identifier']:
query_object = Descriptor().load(
hybrid_structure[query_object_id]).get_object()
invalidations = {query_object.__name__.lower(): ['__all']}
DataList._build_invalidations(invalidations, query_object, items)
for class_name in invalidations:
key = '{0}_{1}'.format(DataList.cachelink, class_name)
mutex = VolatileMutex('listcache_{0}'.format(class_name))
try:
mutex.acquire(60)
cache_list = Toolbox.try_get(key, {})
current_fields = cache_list.get(self._key, [])
current_fields = list(
set(current_fields + ['__all'] +
invalidations[class_name]))
cache_list[self._key] = current_fields
self._volatile.set(key, cache_list)
self._persistent.set(key, cache_list)
finally:
mutex.release()
self.from_cache = False
namespace = query_object()._namespace
name = query_object.__name__.lower()
guids = DataList.get_pks(namespace, name)
if query_data == DataList.select.COUNT:
self.data = 0
else:
self.data = []
for guid in guids:
try:
instance = query_object(guid)
if query_type == DataList.where_operator.AND:
include = self._exec_and(instance, items)
elif query_type == DataList.where_operator.OR:
include = self._exec_or(instance, items)
else:
raise NotImplementedError(
'The given operator is not yet implemented.')
if include:
if query_data == DataList.select.COUNT:
self.data += 1
elif query_data == DataList.select.GUIDS:
self.data.append(guid)
else:
raise NotImplementedError(
'The given selector type is not implemented')
except ObjectNotFoundException:
pass
if self._post_query_hook is not None:
self._post_query_hook(self)
if self._key is not None and len(guids) > 0 and self._can_cache:
invalidated = False
for class_name in invalidations:
key = '{0}_{1}'.format(DataList.cachelink, class_name)
cache_list = Toolbox.try_get(key, {})
if self._key not in cache_list:
invalidated = True
# If the key under which the list should be saved was already invalidated since the invalidations
# were saved, the returned list is most likely outdated. This is OK for this result, but the list
# won't get cached
if invalidated is False:
self._volatile.set(
self._key, self.data, 300 +
randint(0, 300)) # Cache between 5 and 10 minutes
else:
Toolbox.log_cache_hit('datalist', True)
self.from_cache = True
return self
def get_relation_set(remote_class, remote_key, own_class, own_key, own_guid):
"""
This method will get a DataList for a relation.
On a cache miss, the relation DataList will be rebuild and due to the nature of the full table scan, it will
update all relations in the mean time.
"""
# Example:
# * remote_class = vDisk
# * remote_key = vmachine
# * own_class = vMachine
# * own_key = vdisks
# Called to load the vMachine.vdisks list (resulting in a possible scan of vDisk objects)
# * own_guid = this vMachine object's guid
volatile = VolatileFactory.get_client()
own_name = own_class.__name__.lower()
datalist = DataList({}, '{0}_{1}_{2}'.format(own_name, own_guid, remote_key), load=False)
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid)
# Check whether the requested information is available in cache
reverse_index = volatile.get(reverse_key)
if reverse_index is not None and own_key in reverse_index:
Toolbox.log_cache_hit('datalist', True)
datalist.data = reverse_index[own_key]
datalist.from_cache = True
return datalist
Toolbox.log_cache_hit('datalist', False)
mutex = VolatileMutex('reverseindex')
remote_name = remote_class.__name__.lower()
blueprint_object = remote_class() # vDisk object
foreign_guids = {}
remote_namespace = blueprint_object._namespace
for relation in blueprint_object._relations: # E.g. vmachine or vpool relation
if relation.foreign_type is None:
classname = remote_name
foreign_namespace = blueprint_object._namespace
else:
classname = relation.foreign_type.__name__.lower()
foreign_namespace = relation.foreign_type()._namespace
if classname not in foreign_guids:
foreign_guids[classname] = DataList.get_pks(foreign_namespace, classname)
try:
mutex.acquire(60)
for foreign_guid in foreign_guids[classname]:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, foreign_guid)
reverse_index = volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
if relation.foreign_key not in reverse_index:
reverse_index[relation.foreign_key] = []
volatile.set(reverse_key, reverse_index)
finally:
mutex.release()
remote_keys = DataList.get_pks(remote_namespace, remote_name)
for guid in remote_keys:
try:
instance = remote_class(guid)
for relation in blueprint_object._relations: # E.g. vmachine or vpool relation
if relation.foreign_type is None:
classname = remote_name
else:
classname = relation.foreign_type.__name__.lower()
key = getattr(instance, '{0}_guid'.format(relation.name))
if key is not None:
try:
mutex.acquire(60)
reverse_index = volatile.get('ovs_reverseindex_{0}_{1}'.format(classname, key))
if reverse_index is None:
reverse_index = {}
if relation.foreign_key not in reverse_index:
reverse_index[relation.foreign_key] = []
if guid not in reverse_index[relation.foreign_key]:
if instance.updated_on_datastore():
raise ConcurrencyException()
reverse_index[relation.foreign_key].append(guid)
volatile.set('ovs_reverseindex_{0}_{1}'.format(classname, key), reverse_index)
finally:
mutex.release()
except ObjectNotFoundException:
pass
except ConcurrencyException:
pass
try:
mutex.acquire(60)
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid)
reverse_index = volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
if own_key not in reverse_index:
reverse_index[own_key] = []
volatile.set(reverse_key, reverse_index)
datalist.data = reverse_index[own_key]
datalist.from_cache = False
finally:
mutex.release()
return datalist
def update_vmachine_config(vmachine, vm_object, pmachine=None):
"""
Update a vMachine configuration with a given vMachine configuration
:param vmachine: Virtual Machine to update
:param vm_object: New virtual machine info
:param pmachine: Physical machine of the virtual machine
"""
try:
vdisks_synced = 0
if vmachine.name is None:
MessageController.fire(
MessageController.Type.EVENT, {
'type': 'vmachine_created',
'metadata': {
'name': vm_object['name']
}
})
elif vmachine.name != vm_object['name']:
MessageController.fire(
MessageController.Type.EVENT, {
'type': 'vmachine_renamed',
'metadata': {
'old_name': vmachine.name,
'new_name': vm_object['name']
}
})
if pmachine is not None:
vmachine.pmachine = pmachine
vmachine.name = vm_object['name']
vmachine.hypervisor_id = vm_object['id']
vmachine.devicename = vm_object['backing']['filename']
vmachine.save()
# Updating and linking disks
storagedrivers = StorageDriverList.get_storagedrivers()
datastores = dict([('{0}:{1}'.format(storagedriver.storage_ip,
storagedriver.mountpoint),
storagedriver)
for storagedriver in storagedrivers])
vdisk_guids = []
mutex = VolatileMutex('{0}_{1}'.format(vmachine.name,
vmachine.devicename))
for disk in vm_object['disks']:
ensure_safety = False
if disk['datastore'] in vm_object['datastores']:
datastore = vm_object['datastores'][disk['datastore']]
if datastore in datastores:
try:
mutex.acquire(wait=10)
vdisk = VDiskList.get_by_devicename_and_vpool(
disk['filename'], datastores[datastore].vpool)
if vdisk is None:
# The disk couldn't be located, but is in our datastore. We might be in a recovery scenario
vdisk = VDisk()
vdisk.vpool = datastores[datastore].vpool
vdisk.reload_client()
vdisk.devicename = disk['filename']
vdisk.volume_id = vdisk.storagedriver_client.get_volume_id(
str(disk['backingfilename']))
vdisk.size = vdisk.info['volume_size']
# Create the disk in a locked context, but don't execute long running-task in same context
vdisk.save()
ensure_safety = True
finally:
mutex.release()
if ensure_safety:
MDSServiceController.ensure_safety(vdisk)
VDiskController.dtl_checkup(vdisk_guid=vdisk.guid)
# Update the disk with information from the hypervisor
if vdisk.vmachine is None:
MessageController.fire(
MessageController.Type.EVENT, {
'type': 'vdisk_attached',
'metadata': {
'vmachine_name': vmachine.name,
'vdisk_name': disk['name']
}
})
vdisk.vmachine = vmachine
vdisk.name = disk['name']
vdisk.order = disk['order']
vdisk.save()
vdisk_guids.append(vdisk.guid)
vdisks_synced += 1
for vdisk in vmachine.vdisks:
if vdisk.guid not in vdisk_guids:
MessageController.fire(
MessageController.Type.EVENT, {
'type': 'vdisk_detached',
'metadata': {
'vmachine_name': vmachine.name,
'vdisk_name': vdisk.name
}
})
vdisk.vmachine = None
vdisk.save()
logger.info(
'Updating vMachine finished (name {0}, {1} vdisks (re)linked)'.
format(vmachine.name, vdisks_synced))
except Exception as ex:
logger.info('Error during vMachine update: {0}'.format(str(ex)))
raise
def test_volatiemutex(self):
"""
Validates the volatile mutex
"""
mutex = VolatileMutex('test')
mutex.acquire()
mutex.acquire() # Should not raise errors
mutex.release()
mutex.release() # Should not raise errors
mutex._volatile.add(mutex.key(), 1, 10)
with self.assertRaises(RuntimeError):
mutex.acquire(wait=1)
mutex._volatile.delete(mutex.key())
mutex.acquire()
time.sleep(0.5)
mutex.release()
