def new_function(self, request, *args, **kwargs):
"""
Wrapped function
"""
now = time.time()
key = 'ovs_api_limit_{0}.{1}_{2}'.format(
f.__module__, f.__name__, request.META['HTTP_X_REAL_IP'])
client = VolatileFactory.get_client()
mutex = VolatileMutex(key)
try:
mutex.acquire()
rate_info = client.get(key, {'calls': [], 'timeout': None})
active_timeout = rate_info['timeout']
if active_timeout is not None:
if active_timeout > now:
raise Throttled(wait=active_timeout - now)
else:
rate_info['timeout'] = None
rate_info['calls'] = [
call for call in rate_info['calls'] if call > (now - per)
] + [now]
calls = len(rate_info['calls'])
if calls > amount:
rate_info['timeout'] = now + timeout
client.set(key, rate_info)
raise Throttled(wait=timeout)
client.set(key, rate_info)
finally:
mutex.release()
return f(self, request, *args, **kwargs)
def _backend_property(self, function, dynamic):
"""
Handles the internal caching of dynamic properties
"""
caller_name = dynamic.name
cache_key = '{0}_{1}'.format(self._key, caller_name)
mutex = VolatileMutex(cache_key)
try:
cached_data = self._volatile.get(cache_key)
if cached_data is None:
if dynamic.locked:
mutex.acquire()
cached_data = self._volatile.get(cache_key)
if cached_data is None:
function_info = inspect.getargspec(function)
if 'dynamic' in function_info.args:
cached_data = function(dynamic=dynamic) # Load data from backend
else:
cached_data = function()
if cached_data is not None:
correct, allowed_types, given_type = Toolbox.check_type(cached_data, dynamic.return_type)
if not correct:
raise TypeError('Dynamic property {0} allows types {1}. {2} given'.format(
caller_name, str(allowed_types), given_type
))
if dynamic.timeout > 0:
self._volatile.set(cache_key, cached_data, dynamic.timeout)
return cached_data
finally:
mutex.release()
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
# Load dummy stores
PersistentFactory.store = DummyPersistentStore()
VolatileFactory.store = DummyVolatileStore()
# Replace mocked classes
sys.modules[
'ovs.extensions.storageserver.storagedriver'] = StorageDriver
# Import required modules/classes after mocking is done
from ovs.dal.hybrids.vmachine import VMachine
from ovs.dal.hybrids.vdisk import VDisk
from ovs.extensions.generic.volatilemutex import VolatileMutex
from ovs.lib.vmachine import VMachineController
from ovs.lib.vdisk import VDiskController
from ovs.lib.scheduledtask import ScheduledTaskController
# Globalize mocked classes
global VDisk
global VMachine
global VolatileMutex
global VMachineController
global VDiskController
global ScheduledTaskController
_ = VDisk(), VolatileMutex('dummy'), VMachine(
), VMachineController, VDiskController, ScheduledTaskController
# Cleaning storage
VolatileFactory.store.clean()
PersistentFactory.store.clean()
def resize_from_voldrv(volumename, volumesize, volumepath,
storagedriver_id):
"""
Resize a disk
Triggered by volumedriver messages on the queue
@param volumepath: path on hypervisor to the volume
@param volumename: volume id of the disk
@param volumesize: size of the volume
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
hypervisor = Factory.get(pmachine)
volumepath = hypervisor.clean_backing_disk_filename(volumepath)
mutex = VolatileMutex('{}_{}'.format(volumename, volumepath))
try:
mutex.acquire(wait=30)
disk = VDiskList.get_vdisk_by_volume_id(volumename)
if disk is None:
disk = VDiskList.get_by_devicename_and_vpool(
volumepath, storagedriver.vpool)
if disk is None:
disk = VDisk()
finally:
mutex.release()
disk.devicename = volumepath
disk.volume_id = volumename
disk.size = volumesize
disk.vpool = storagedriver.vpool
disk.save()
def update_vdisk_name(volume_id, old_name, new_name):
"""
Update a vDisk name using Management Center: set new name
"""
vdisk = None
for mgmt_center in MgmtCenterList.get_mgmtcenters():
mgmt = Factory.get_mgmtcenter(mgmt_center=mgmt_center)
try:
disk_info = mgmt.get_vdisk_device_info(volume_id)
device_path = disk_info['device_path']
vpool_name = disk_info['vpool_name']
vp = VPoolList.get_vpool_by_name(vpool_name)
file_name = os.path.basename(device_path)
vdisk = VDiskList.get_by_devicename_and_vpool(file_name, vp)
if vdisk:
break
except Exception as ex:
logger.info(
'Trying to get mgmt center failed for disk {0} with volume_id {1}. {2}'
.format(old_name, volume_id, ex))
if not vdisk:
logger.error('No vdisk found for name {0}'.format(old_name))
return
vpool = vdisk.vpool
mutex = VolatileMutex('{}_{}'.format(
old_name, vpool.guid if vpool is not None else 'none'))
try:
mutex.acquire(wait=5)
vdisk.name = new_name
vdisk.save()
finally:
mutex.release()
def new_function(*args, **kwargs):
"""
Wrapped function
"""
request = _find_request(args)
now = time.time()
key = 'ovs_api_limit_{0}.{1}_{2}'.format(
f.__module__, f.__name__, request.META['HTTP_X_REAL_IP'])
client = VolatileFactory.get_client()
with VolatileMutex(key):
rate_info = client.get(key, {'calls': [], 'timeout': None})
active_timeout = rate_info['timeout']
if active_timeout is not None:
if active_timeout > now:
logger.warning(
'Call {0} is being throttled with a wait of {1}'.
format(key, active_timeout - now))
raise Throttled(wait=active_timeout - now)
else:
rate_info['timeout'] = None
rate_info['calls'] = [
call for call in rate_info['calls'] if call > (now - per)
] + [now]
calls = len(rate_info['calls'])
if calls > amount:
rate_info['timeout'] = now + timeout
client.set(key, rate_info)
logger.warning(
'Call {0} is being throttled with a wait of {1}'.
format(key, timeout))
raise Throttled(wait=timeout)
client.set(key, rate_info)
return f(*args, **kwargs)
def __init__(self, *args, **kwargs):
"""
Initializes the distributed scheduler
"""
self._persistent = PersistentFactory.get_client()
self._namespace = 'ovs_celery_beat'
self._mutex = VolatileMutex('celery_beat')
self._has_lock = False
super(DistributedScheduler, self).__init__(*args, **kwargs)
logger.debug('DS init')
def extend_cluster(master_ip, new_ip, cluster_name, base_dir):
"""
Extends a cluster to a given new node
:param base_dir: Base directory that will hold the db and tlogs
:param cluster_name: Name of the cluster to be extended
:param new_ip: IP address of the node to be added
:param master_ip: IP of one of the already existing nodes
"""
logger.debug('Extending cluster {0} from {1} to {2}'.format(
cluster_name, master_ip, new_ip))
base_dir = base_dir.rstrip('/')
from ovs.extensions.generic.volatilemutex import VolatileMutex
port_mutex = VolatileMutex('arakoon_install_ports_{0}'.format(new_ip))
config = ArakoonClusterConfig(cluster_name)
config.load_config()
client = SSHClient(new_ip)
node_name = System.get_my_machine_id(client)
home_dir = ArakoonInstaller.ARAKOON_HOME_DIR.format(
base_dir, cluster_name)
log_dir = ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name)
tlog_dir = ArakoonInstaller.ARAKOON_TLOG_DIR.format(
base_dir, cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(
new_ip, home_dir,
ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir), cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(
new_ip, log_dir, ArakoonInstaller.ARAKOON_LOG_DIR.format(''),
cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(
new_ip, tlog_dir,
ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir), cluster_name)
try:
port_mutex.acquire(wait=60)
ports = ArakoonInstaller._get_free_ports(client)
if node_name not in [node.name for node in config.nodes]:
config.nodes.append(
ArakoonNodeConfig(name=node_name,
ip=new_ip,
client_port=ports[0],
messaging_port=ports[1],
log_dir=log_dir,
home=home_dir,
tlog_dir=tlog_dir))
ArakoonInstaller._deploy(config)
finally:
port_mutex.release()
logger.debug('Extending cluster {0} from {1} to {2} completed'.format(
cluster_name, master_ip, new_ip))
return {'client_port': ports[0], 'messaging_port': ports[1]}
def create_cluster(cluster_name, ip, base_dir, plugins=None, locked=True):
"""
Creates a cluster
:param locked: Indicates whether the create should run in a locked context (e.g. to prevent port conflicts)
:param plugins: Plugins that should be added to the configuration file
:param base_dir: Base directory that should contain the data and tlogs
:param ip: IP address of the first node of the new cluster
:param cluster_name: Name of the cluster
"""
logger.debug('Creating cluster {0} on {1}'.format(cluster_name, ip))
base_dir = base_dir.rstrip('/')
client = SSHClient(ip)
node_name = System.get_my_machine_id(client)
home_dir = ArakoonInstaller.ARAKOON_HOME_DIR.format(
base_dir, cluster_name)
log_dir = ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name)
tlog_dir = ArakoonInstaller.ARAKOON_TLOG_DIR.format(
base_dir, cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(
ip, home_dir, ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir),
cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(
ip, log_dir, ArakoonInstaller.ARAKOON_LOG_DIR.format(''),
cluster_name)
ArakoonInstaller.archive_existing_arakoon_data(
ip, tlog_dir, ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir),
cluster_name)
port_mutex = None
try:
if locked is True:
from ovs.extensions.generic.volatilemutex import VolatileMutex
port_mutex = VolatileMutex(
'arakoon_install_ports_{0}'.format(ip))
port_mutex.acquire(wait=60)
ports = ArakoonInstaller._get_free_ports(client)
config = ArakoonClusterConfig(cluster_name, plugins)
config.nodes.append(
ArakoonNodeConfig(name=node_name,
ip=ip,
client_port=ports[0],
messaging_port=ports[1],
log_dir=log_dir,
home=home_dir,
tlog_dir=tlog_dir))
ArakoonInstaller._deploy(config)
finally:
if port_mutex is not None:
port_mutex.release()
logger.debug('Creating cluster {0} on {1} completed'.format(
cluster_name, ip))
return {'client_port': ports[0], 'messaging_port': ports[1]}
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
# Load dummy stores
PersistentFactory.store = DummyPersistentStore()
VolatileFactory.store = DummyVolatileStore()
# Replace mocked classes
sys.modules[
'ovs.extensions.storageserver.storagedriver'] = StorageDriverModule
# Import required modules/classes after mocking is done
from ovs.dal.hybrids.vdisk import VDisk
from ovs.dal.hybrids.service import Service
from ovs.dal.hybrids.vpool import VPool
from ovs.dal.hybrids.storagerouter import StorageRouter
from ovs.dal.hybrids.pmachine import PMachine
from ovs.dal.hybrids.servicetype import ServiceType
from ovs.dal.hybrids.storagedriver import StorageDriver
from ovs.dal.hybrids.backendtype import BackendType
from ovs.dal.hybrids.j_mdsservice import MDSService
from ovs.dal.hybrids.j_mdsservicevdisk import MDSServiceVDisk
from ovs.extensions.generic.volatilemutex import VolatileMutex
from ovs.lib.mdsservice import MDSServiceController
# Globalize mocked classes
global VDisk
global VPool
global Service
global StorageRouter
global StorageDriver
global BackendType
global PMachine
global MDSService
global ServiceType
global MDSServiceVDisk
global VolatileMutex
global MDSServiceController
_ = VDisk(), VPool(), Service(), MDSService(), MDSServiceVDisk(), ServiceType(), \
StorageRouter(), StorageDriver(), BackendType(), PMachine(), \
VolatileMutex('dummy'), MDSServiceController
# Configuration
def _get(key):
c = PersistentFactory.get_client()
if c.exists(key):
return c.get(key)
return None
Configuration.get = staticmethod(_get)
# Cleaning storage
VolatileFactory.store.clean()
PersistentFactory.store.clean()
def new_function(*args, **kw):
"""
Executes the decorated function in a locked context
"""
filemutex = FileMutex('messaging')
try:
filemutex.acquire(wait=5)
mutex = VolatileMutex('messaging')
try:
mutex.acquire(wait=5)
return f(*args, **kw)
finally:
mutex.release()
finally:
filemutex.release()
def invalidate_dynamics(self, properties=None):
"""
Invalidates all dynamic property caches. Use with caution, as this action can introduce
a short performance hit.
"""
for dynamic in self._dynamics:
if properties is None or dynamic.name in properties:
key = '{0}_{1}'.format(self._key, dynamic.name)
mutex = VolatileMutex(key)
try:
if dynamic.locked:
mutex.acquire()
self._volatile.delete(key)
finally:
mutex.release()
def new_function(self, request, *args, **kwargs):
"""
Wrapped function
"""
now = time.time()
key = 'ovs_api_limit_{0}.{1}_{2}'.format(
f.__module__, f.__name__, request.META['HTTP_X_REAL_IP'])
client = VolatileFactory.get_client()
mutex = VolatileMutex(key)
try:
mutex.acquire()
rate_info = client.get(key, {'calls': [], 'timeout': None})
active_timeout = rate_info['timeout']
if active_timeout is not None:
if active_timeout > now:
logger.warning(
'Call {0} is being throttled with a wait of {1}'.
format(key, active_timeout - now))
return HttpResponse, {
'error_code':
'rate_limit_timeout',
'error':
'Rate limit timeout ({0}s remaining)'.format(
round(active_timeout - now, 2))
}, 429
else:
rate_info['timeout'] = None
rate_info['calls'] = [
call for call in rate_info['calls'] if call > (now - per)
] + [now]
calls = len(rate_info['calls'])
if calls > amount:
rate_info['timeout'] = now + timeout
client.set(key, rate_info)
logger.warning(
'Call {0} is being throttled with a wait of {1}'.
format(key, timeout))
return HttpResponse, {
'error_code':
'rate_limit_reached',
'error':
'Rate limit reached ({0} in last {1}s)'.format(
calls, per)
}, 429
client.set(key, rate_info)
finally:
mutex.release()
return f(self, request, *args, **kwargs)
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
# Load dummy stores
PersistentFactory.store = DummyPersistentStore()
VolatileFactory.store = DummyVolatileStore()
# Replace mocked classes
sys.modules[
'ovs.extensions.storageserver.storagedriver'] = StorageDriverModule
# Import required modules/classes after mocking is done
from ovs.dal.hybrids.backendtype import BackendType
from ovs.dal.hybrids.disk import Disk
from ovs.dal.hybrids.diskpartition import DiskPartition
from ovs.dal.hybrids.failuredomain import FailureDomain
from ovs.dal.hybrids.pmachine import PMachine
from ovs.dal.hybrids.storagerouter import StorageRouter
from ovs.dal.hybrids.vdisk import VDisk
from ovs.dal.hybrids.vmachine import VMachine
from ovs.dal.hybrids.vpool import VPool
from ovs.extensions.generic.volatilemutex import VolatileMutex
from ovs.lib.vmachine import VMachineController
from ovs.lib.vdisk import VDiskController
from ovs.lib.scheduledtask import ScheduledTaskController
# Globalize mocked classes
global Disk
global VDisk
global VMachine
global PMachine
global VPool
global BackendType
global DiskPartition
global FailureDomain
global StorageRouter
global VolatileMutex
global VMachineController
global VDiskController
global ScheduledTaskController
_ = VDisk(), VolatileMutex('dummy'), VMachine(), PMachine(), VPool(), BackendType(), FailureDomain(), \
VMachineController, VDiskController, ScheduledTaskController, StorageRouter(), Disk(), DiskPartition()
# Cleaning storage
VolatileFactory.store.clean()
PersistentFactory.store.clean()
def update_vmachine_name(instance_id, old_name, new_name):
"""
Update a vMachine name: find vmachine by management center instance id, set new name
:param instance_id: ID for the virtual machine known by management center
:param old_name: Old name of the virtual machine
:param new_name: New name for the virtual machine
"""
vmachine = None
for mgmt_center in MgmtCenterList.get_mgmtcenters():
mgmt = Factory.get_mgmtcenter(mgmt_center=mgmt_center)
try:
machine_info = mgmt.get_vmachine_device_info(instance_id)
file_name = machine_info['file_name']
host_name = machine_info['host_name']
vpool_name = machine_info['vpool_name']
storage_router = StorageRouterList.get_by_name(host_name)
machine_id = storage_router.machine_id
device_name = '{0}/{1}'.format(machine_id, file_name)
vp = VPoolList.get_vpool_by_name(vpool_name)
vmachine = VMachineList.get_by_devicename_and_vpool(
device_name, vp)
if vmachine:
break
vmachine = VMachineList.get_by_devicename_and_vpool(
device_name, None)
if vmachine:
break
except Exception as ex:
logger.info(
'Trying to get mgmt center failed for vmachine {0}. {1}'.
format(old_name, ex))
if not vmachine:
logger.error('No vmachine found for name {0}'.format(old_name))
return
vpool = vmachine.vpool
mutex = VolatileMutex('{0}_{1}'.format(
old_name, vpool.guid if vpool is not None else 'none'))
try:
mutex.acquire(wait=5)
vmachine.name = new_name
vmachine.save()
finally:
mutex.release()
def delete_from_voldrv(volumename, storagedriver_id):
"""
Delete a disk
Triggered by volumedriver messages on the queue
@param volumename: volume id of the disk
"""
_ = storagedriver_id # For logging purposes
disk = VDiskList.get_vdisk_by_volume_id(volumename)
if disk is not None:
mutex = VolatileMutex('{}_{}'.format(volumename, disk.devicename))
try:
mutex.acquire(wait=20)
pmachine = None
try:
pmachine = PMachineList.get_by_storagedriver_id(
disk.storagedriver_id)
except RuntimeError as ex:
if 'could not be found' not in str(ex):
raise
# else: pmachine can't be loaded, because the volumedriver doesn't know about it anymore
if pmachine is not None:
limit = 5
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
hypervisor = Factory.get(pmachine)
exists = hypervisor.file_exists(storagedriver,
disk.devicename)
while limit > 0 and exists is True:
time.sleep(1)
exists = hypervisor.file_exists(
storagedriver, disk.devicename)
limit -= 1
if exists is True:
logger.info(
'Disk {0} still exists, ignoring delete'.format(
disk.devicename))
return
logger.info('Delete disk {}'.format(disk.name))
for mds_service in disk.mds_services:
mds_service.delete()
disk.delete()
finally:
mutex.release()
def update_value(key, append, value_to_store=None):
"""
Store the specified value in the PersistentFactory
:param key: Key to store the value for
:param append: If True, the specified value will be appended else element at index 0 will be popped
:param value_to_store: Value to append to the list
:return: Updated value
"""
with VolatileMutex(name=key, wait=5):
if persistent_client.exists(key):
val = persistent_client.get(key)
if append is True and value_to_store is not None:
val['values'].append(value_to_store)
elif append is False and len(val['values']) > 0:
val['values'].pop(0)
else:
log_message('Setting initial value for key {0}'.format(
persistent_key))
val = {'mode': mode, 'values': []}
persistent_client.set(key, val)
return val
def update_from_voldrv(name, storagedriver_id):
"""
This method will update/create a vmachine based on a given vmx/xml file
"""
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
if pmachine.hvtype not in ['VMWARE', 'KVM']:
return
hypervisor = Factory.get(pmachine)
name = hypervisor.clean_vmachine_filename(name)
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
vpool = storagedriver.vpool
machine_ids = [
storagedriver.storagerouter.machine_id
for storagedriver in vpool.storagedrivers
]
if hypervisor.should_process(name, machine_ids=machine_ids):
if pmachine.hvtype == 'VMWARE':
storagedriver = StorageDriverList.get_by_storagedriver_id(
storagedriver_id)
vpool = storagedriver.vpool
else:
vpool = None
pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id)
mutex = VolatileMutex('{}_{}'.format(
name, vpool.guid if vpool is not None else 'none'))
try:
mutex.acquire(wait=120)
limit = 5
exists = hypervisor.file_exists(storagedriver, name)
while limit > 0 and exists is False:
time.sleep(1)
exists = hypervisor.file_exists(storagedriver, name)
limit -= 1
if exists is False:
logger.info(
'Could not locate vmachine with name {0} on vpool {1}'.
format(name, vpool.name))
vmachine = VMachineList.get_by_devicename_and_vpool(
name, vpool)
if vmachine is not None:
VMachineController.delete_from_voldrv(
name, storagedriver_id=storagedriver_id)
return
finally:
mutex.release()
try:
mutex.acquire(wait=5)
vmachine = VMachineList.get_by_devicename_and_vpool(
name, vpool)
if not vmachine:
vmachine = VMachine()
vmachine.vpool = vpool
vmachine.pmachine = pmachine
vmachine.status = 'CREATED'
vmachine.devicename = name
vmachine.save()
finally:
mutex.release()
if pmachine.hvtype == 'KVM':
try:
VMachineController.sync_with_hypervisor(
vmachine.guid, storagedriver_id=storagedriver_id)
vmachine.status = 'SYNC'
except:
vmachine.status = 'SYNC_NOK'
vmachine.save()
else:
logger.info('Ignored invalid file {0}'.format(name))
def __init__(self, guid=None, data=None, datastore_wins=False, volatile=False, _hook=None):
"""
Loads an object with a given guid. If no guid is given, a new object
is generated with a new guid.
* guid: The guid indicating which object should be loaded
* datastoreWins: Optional boolean indicating save conflict resolve management.
** True: when saving, external modified fields will not be saved
** False: when saving, all changed data will be saved, regardless of external updates
** None: in case changed field were also changed externally, an error will be raised
"""
# Initialize super class
super(DataObject, self).__init__()
# Initialize internal fields
self._frozen = False
self._datastore_wins = datastore_wins
self._guid = None # Guid identifier of the object
self._original = {} # Original data copy
self._metadata = {} # Some metadata, mainly used for unit testing
self._data = {} # Internal data storage
self._objects = {} # Internal objects storage
# Initialize public fields
self.dirty = False
self.volatile = volatile
# Worker fields/objects
self._classname = self.__class__.__name__.lower()
self._namespace = 'ovs_data' # Namespace of the object
self._mutex_listcache = VolatileMutex('listcache_{0}'.format(self._classname))
self._mutex_reverseindex = VolatileMutex('reverseindex')
# Rebuild _relation types
hybrid_structure = HybridRunner.get_hybrids()
for relation in self._relations:
if relation.foreign_type is not None:
identifier = Descriptor(relation.foreign_type).descriptor['identifier']
if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']:
relation.foreign_type = Descriptor().load(hybrid_structure[identifier]).get_object()
# Init guid
self._new = False
if guid is None:
self._guid = str(uuid.uuid4())
self._new = True
else:
guid = str(guid).lower()
if re.match('^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', guid) is not None:
self._guid = str(guid)
else:
raise ValueError('The given guid is invalid: {0}'.format(guid))
# Build base keys
self._key = '{0}_{1}_{2}'.format(self._namespace, self._classname, self._guid)
# Worker mutexes
self._mutex_version = VolatileMutex('ovs_dataversion_{0}_{1}'.format(self._classname, self._guid))
# Load data from cache or persistent backend where appropriate
self._volatile = VolatileFactory.get_client()
self._persistent = PersistentFactory.get_client()
self._metadata['cache'] = None
if self._new:
self._data = {}
else:
self._data = self._volatile.get(self._key)
if self._data is None:
Toolbox.log_cache_hit('object_load', False)
self._metadata['cache'] = False
try:
self._data = self._persistent.get(self._key)
except KeyNotFoundException:
raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format(
self.__class__.__name__, self._guid
))
else:
Toolbox.log_cache_hit('object_load', True)
self._metadata['cache'] = True
# Set default values on new fields
for prop in self._properties:
if prop.name not in self._data:
self._data[prop.name] = prop.default
self._add_property(prop)
# Load relations
for relation in self._relations:
if relation.name not in self._data:
if relation.foreign_type is None:
cls = self.__class__
else:
cls = relation.foreign_type
self._data[relation.name] = Descriptor(cls).descriptor
self._add_relation_property(relation)
# Add wrapped properties
for dynamic in self._dynamics:
self._add_dynamic_property(dynamic)
# Load foreign keys
relations = RelationMapper.load_foreign_relations(self.__class__)
if relations is not None:
for key, info in relations.iteritems():
self._objects[key] = {'info': info,
'data': None}
self._add_list_property(key, info['list'])
# Store original data
self._original = copy.deepcopy(self._data)
if _hook is not None and hasattr(_hook, '__call__'):
_hook()
if not self._new:
# Re-cache the object, if required
if self._metadata['cache'] is False:
# The data wasn't loaded from the cache, so caching is required now
try:
self._mutex_version.acquire(30)
this_version = self._data['_version']
store_version = self._persistent.get(self._key)['_version']
if this_version == store_version:
self._volatile.set(self._key, self._data)
except KeyNotFoundException:
raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format(
self.__class__.__name__, self._guid
))
finally:
self._mutex_version.release()
# Freeze property creation
self._frozen = True
# Optionally, initialize some fields
if data is not None:
for prop in self._properties:
if prop.name in data:
setattr(self, prop.name, data[prop.name])
def update_vmachine_config(vmachine, vm_object, pmachine=None):
"""
Update a vMachine configuration with a given vMachine configuration
:param vmachine: Virtual Machine to update
:param vm_object: New virtual machine info
:param pmachine: Physical machine of the virtual machine
"""
try:
vdisks_synced = 0
if vmachine.name is None:
MessageController.fire(
MessageController.Type.EVENT, {
'type': 'vmachine_created',
'metadata': {
'name': vm_object['name']
}
})
elif vmachine.name != vm_object['name']:
MessageController.fire(
MessageController.Type.EVENT, {
'type': 'vmachine_renamed',
'metadata': {
'old_name': vmachine.name,
'new_name': vm_object['name']
}
})
if pmachine is not None:
vmachine.pmachine = pmachine
vmachine.name = vm_object['name']
vmachine.hypervisor_id = vm_object['id']
vmachine.devicename = vm_object['backing']['filename']
vmachine.save()
# Updating and linking disks
storagedrivers = StorageDriverList.get_storagedrivers()
datastores = dict([('{0}:{1}'.format(storagedriver.storage_ip,
storagedriver.mountpoint),
storagedriver)
for storagedriver in storagedrivers])
vdisk_guids = []
mutex = VolatileMutex('{0}_{1}'.format(vmachine.name,
vmachine.devicename))
for disk in vm_object['disks']:
ensure_safety = False
if disk['datastore'] in vm_object['datastores']:
datastore = vm_object['datastores'][disk['datastore']]
if datastore in datastores:
try:
mutex.acquire(wait=10)
vdisk = VDiskList.get_by_devicename_and_vpool(
disk['filename'], datastores[datastore].vpool)
if vdisk is None:
# The disk couldn't be located, but is in our datastore. We might be in a recovery scenario
vdisk = VDisk()
vdisk.vpool = datastores[datastore].vpool
vdisk.reload_client()
vdisk.devicename = disk['filename']
vdisk.volume_id = vdisk.storagedriver_client.get_volume_id(
str(disk['backingfilename']))
vdisk.size = vdisk.info['volume_size']
# Create the disk in a locked context, but don't execute long running-task in same context
vdisk.save()
ensure_safety = True
finally:
mutex.release()
if ensure_safety:
MDSServiceController.ensure_safety(vdisk)
VDiskController.dtl_checkup(vdisk_guid=vdisk.guid)
# Update the disk with information from the hypervisor
if vdisk.vmachine is None:
MessageController.fire(
MessageController.Type.EVENT, {
'type': 'vdisk_attached',
'metadata': {
'vmachine_name': vmachine.name,
'vdisk_name': disk['name']
}
})
vdisk.vmachine = vmachine
vdisk.name = disk['name']
vdisk.order = disk['order']
vdisk.save()
vdisk_guids.append(vdisk.guid)
vdisks_synced += 1
for vdisk in vmachine.vdisks:
if vdisk.guid not in vdisk_guids:
MessageController.fire(
MessageController.Type.EVENT, {
'type': 'vdisk_detached',
'metadata': {
'vmachine_name': vmachine.name,
'vdisk_name': vdisk.name
}
})
vdisk.vmachine = None
vdisk.save()
logger.info(
'Updating vMachine finished (name {0}, {1} vdisks (re)linked)'.
format(vmachine.name, vdisks_synced))
except Exception as ex:
logger.info('Error during vMachine update: {0}'.format(str(ex)))
raise
def new_function(*args, **kwargs):
"""
Wrapped function
:param args: Arguments without default values
:param kwargs: Arguments with default values
"""
def log_message(message, level='info'):
"""
Log a message with some additional information
:param message: Message to log
:param level: Log level
:return: None
"""
if level not in ('info', 'warning', 'debug', 'error'):
raise ValueError(
'Unsupported log level "{0}" specified'.format(level))
complete_message = 'Ensure single {0} mode - ID {1} - {2}'.format(
mode, now, message)
getattr(logger, level)(complete_message)
def update_value(key, append, value_to_store=None):
"""
Store the specified value in the PersistentFactory
:param key: Key to store the value for
:param append: If True, the specified value will be appended else element at index 0 will be popped
:param value_to_store: Value to append to the list
:return: Updated value
"""
with VolatileMutex(name=key, wait=5):
if persistent_client.exists(key):
val = persistent_client.get(key)
if append is True and value_to_store is not None:
val['values'].append(value_to_store)
elif append is False and len(val['values']) > 0:
val['values'].pop(0)
else:
log_message('Setting initial value for key {0}'.format(
persistent_key))
val = {'mode': mode, 'values': []}
persistent_client.set(key, val)
return val
now = '{0}_{1}'.format(
int(time.time()), ''.join(
random.choice(string.ascii_letters + string.digits)
for _ in range(10)))
task_names = [
task_name
] if extra_task_names is None else [task_name] + extra_task_names
persistent_key = '{0}_{1}'.format(ENSURE_SINGLE_KEY, task_name)
persistent_client = PersistentFactory.get_client()
if mode == 'DEFAULT':
with VolatileMutex(persistent_key, wait=5):
for task in task_names:
key_to_check = '{0}_{1}'.format(
ENSURE_SINGLE_KEY, task)
if persistent_client.exists(key_to_check):
log_message(
'Execution of task {0} discarded'.format(
task_name))
return None
log_message('Setting key {0}'.format(persistent_key))
persistent_client.set(persistent_key, {'mode': mode})
try:
output = function(*args, **kwargs)
log_message(
'Task {0} finished successfully'.format(task_name))
return output
finally:
with VolatileMutex(persistent_key, wait=5):
if persistent_client.exists(persistent_key):
log_message(
'Deleting key {0}'.format(persistent_key))
persistent_client.delete(persistent_key)
elif mode == 'CHAINED':
if extra_task_names is not None:
log_message('Extra tasks are not allowed in this mode',
level='error')
raise ValueError(
'Ensure single {0} mode - ID {1} - Extra tasks are not allowed in this mode'
.format(mode, now))
# 1. Create key to be stored in arakoon and update kwargs with args
timeout = kwargs.pop(
'chain_timeout'
) if 'chain_timeout' in kwargs else global_timeout
function_info = inspect.getargspec(function)
kwargs_dict = {}
for index, arg in enumerate(args):
kwargs_dict[function_info.args[index]] = arg
kwargs_dict.update(kwargs)
params_info = 'with params {0}'.format(
kwargs_dict) if kwargs_dict else 'with default params'
# 2. Set the key in arakoon if non-existent
value = update_value(key=persistent_key, append=True)
# 3. Validate whether another job with same params is being executed, skip if so
for item in value['values'][
1:]: # 1st element is processing job, we check all other queued jobs for identical params
if item['kwargs'] == kwargs_dict:
log_message(
'Execution of task {0} {1} discarded because of identical parameters'
.format(task_name, params_info))
return None
log_message('New task {0} {1} scheduled for execution'.format(
task_name, params_info))
update_value(key=persistent_key,
append=True,
value_to_store={
'kwargs': kwargs_dict,
'timestamp': now
})
# 4. Poll the arakoon to see whether this call is the first in list, if so --> execute, else wait
first_element = None
counter = 0
while first_element != now and counter < timeout:
if persistent_client.exists(persistent_key):
value = persistent_client.get(persistent_key)
first_element = value['values'][0]['timestamp']
if first_element == now:
try:
if counter != 0:
current_time = int(time.time())
starting_time = int(now.split('_')[0])
log_message(
'Task {0} {1} had to wait {2} seconds before being able to start'
.format(task_name, params_info,
current_time - starting_time))
output = function(*args, **kwargs)
log_message(
'Task {0} finished successfully'.format(
task_name))
finally:
update_value(key=persistent_key, append=False)
return output
counter += 1
time.sleep(1)
if counter == timeout:
update_value(key=persistent_key, append=False)
log_message(
'Could not start task {0} {1}, within expected time ({2}s). Removed it from queue'
.format(task_name, params_info, timeout),
level='error')
raise EnsureSingleTimeoutReached(
'Ensure single {0} mode - ID {1} - Task {2} could not be started within timeout of {3}s'
.format(mode, now, task_name, timeout))
else:
raise ValueError(
'Unsupported mode "{0}" provided'.format(mode))
def get_relation_set(remote_class, remote_key, own_class, own_key,
own_guid):
"""
This method will get a DataList for a relation.
On a cache miss, the relation DataList will be rebuild and due to the nature of the full table scan, it will
update all relations in the mean time.
"""
# Example:
# * remote_class = vDisk
# * remote_key = vmachine
# * own_class = vMachine
# * own_key = vdisks
# Called to load the vMachine.vdisks list (resulting in a possible scan of vDisk objects)
# * own_guid = this vMachine object's guid
volatile = VolatileFactory.get_client()
own_name = own_class.__name__.lower()
datalist = DataList({},
'{0}_{1}_{2}'.format(own_name, own_guid,
remote_key),
load=False)
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid)
# Check whether the requested information is available in cache
reverse_index = volatile.get(reverse_key)
if reverse_index is not None and own_key in reverse_index:
Toolbox.log_cache_hit('datalist', True)
datalist.data = reverse_index[own_key]
datalist.from_cache = True
return datalist
Toolbox.log_cache_hit('datalist', False)
mutex = VolatileMutex('reverseindex')
remote_name = remote_class.__name__.lower()
blueprint_object = remote_class() # vDisk object
foreign_guids = {}
remote_namespace = blueprint_object._namespace
remote_keys = DataList.get_pks(remote_namespace, remote_name)
handled_flows = []
for guid in remote_keys:
try:
instance = remote_class(guid)
for relation in blueprint_object._relations: # E.g. vmachine or vpool relation
if relation.foreign_type is None:
classname = remote_name
foreign_namespace = blueprint_object._namespace
else:
classname = relation.foreign_type.__name__.lower()
foreign_namespace = relation.foreign_type()._namespace
if classname not in foreign_guids:
foreign_guids[classname] = DataList.get_pks(
foreign_namespace, classname)
flow = '{0}_{1}'.format(classname, relation.foreign_key)
if flow not in handled_flows:
handled_flows.append(flow)
try:
mutex.acquire(60)
for foreign_guid in foreign_guids[classname]:
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(
classname, foreign_guid)
reverse_index = volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
if relation.foreign_key not in reverse_index:
reverse_index[relation.foreign_key] = []
volatile.set(reverse_key, reverse_index)
finally:
mutex.release()
key = getattr(instance, '{0}_guid'.format(relation.name))
if key is not None:
try:
mutex.acquire(60)
reverse_index = volatile.get(
'ovs_reverseindex_{0}_{1}'.format(
classname, key))
if reverse_index is None:
reverse_index = {}
if relation.foreign_key not in reverse_index:
reverse_index[relation.foreign_key] = []
if guid not in reverse_index[relation.foreign_key]:
reverse_index[relation.foreign_key].append(
guid)
volatile.set(
'ovs_reverseindex_{0}_{1}'.format(
classname, key), reverse_index)
finally:
mutex.release()
except ObjectNotFoundException:
pass
try:
mutex.acquire(60)
reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid)
reverse_index = volatile.get(reverse_key)
if reverse_index is None:
reverse_index = {}
if own_key not in reverse_index:
reverse_index[own_key] = []
volatile.set(reverse_key, reverse_index)
datalist.data = reverse_index[own_key]
datalist.from_cache = False
finally:
mutex.release()
return datalist
def _load(self):
"""
Tries to load the result for the given key from the volatile cache, or executes the query
if not yet available. Afterwards (if a key is given), the result will be (re)cached
"""
self.data = self._volatile.get(
self._key) if self._key is not None else None
if self.data is None:
# The query should be a dictionary:
# {'object': Disk, # Object on which the query should be executed
# 'data' : DataList.select.XYZ, # The requested result
# 'query' : <query>} # The actual query
# Where <query> is a query(group) dictionary:
# {'type' : DataList.where_operator.ABC, # Whether the items should be AND/OR
# 'items': <items>} # The items in the group
# Where the <items> is any combination of one or more <filter> or <query>
# A <filter> tuple example:
# (<field>, DataList.operator.GHI, <value>) # For example EQUALS
# The field is any property you would also find on the given object. In case of
# properties, you can dot as far as you like. This means you can combine AND and OR
# in any possible combination
Toolbox.log_cache_hit('datalist', False)
hybrid_structure = HybridRunner.get_hybrids()
items = self._query['query']['items']
query_type = self._query['query']['type']
query_data = self._query['data']
query_object = self._query['object']
query_object_id = Descriptor(query_object).descriptor['identifier']
if query_object_id in hybrid_structure and query_object_id != hybrid_structure[
query_object_id]['identifier']:
query_object = Descriptor().load(
hybrid_structure[query_object_id]).get_object()
invalidations = {query_object.__name__.lower(): ['__all']}
DataList._build_invalidations(invalidations, query_object, items)
for class_name in invalidations:
key = '{0}_{1}'.format(DataList.cachelink, class_name)
mutex = VolatileMutex('listcache_{0}'.format(class_name))
try:
mutex.acquire(60)
cache_list = Toolbox.try_get(key, {})
current_fields = cache_list.get(self._key, [])
current_fields = list(
set(current_fields + ['__all'] +
invalidations[class_name]))
cache_list[self._key] = current_fields
self._volatile.set(key, cache_list)
self._persistent.set(key, cache_list)
finally:
mutex.release()
self.from_cache = False
namespace = query_object()._namespace
name = query_object.__name__.lower()
guids = DataList.get_pks(namespace, name)
if query_data == DataList.select.COUNT:
self.data = 0
else:
self.data = []
for guid in guids:
try:
instance = query_object(guid)
if query_type == DataList.where_operator.AND:
include = self._exec_and(instance, items)
elif query_type == DataList.where_operator.OR:
include = self._exec_or(instance, items)
else:
raise NotImplementedError(
'The given operator is not yet implemented.')
if include:
if query_data == DataList.select.COUNT:
self.data += 1
elif query_data == DataList.select.GUIDS:
self.data.append(guid)
else:
raise NotImplementedError(
'The given selector type is not implemented')
except ObjectNotFoundException:
pass
if self._post_query_hook is not None:
self._post_query_hook(self)
if self._key is not None and len(guids) > 0 and self._can_cache:
invalidated = False
for class_name in invalidations:
key = '{0}_{1}'.format(DataList.cachelink, class_name)
cache_list = Toolbox.try_get(key, {})
if self._key not in cache_list:
invalidated = True
# If the key under which the list should be saved was already invalidated since the invalidations
# were saved, the returned list is most likely outdated. This is OK for this result, but the list
# won't get cached
if invalidated is False:
self._volatile.set(
self._key, self.data, 300 +
randint(0, 300)) # Cache between 5 and 10 minutes
else:
Toolbox.log_cache_hit('datalist', True)
self.from_cache = True
return self
def clone(machineguid, timestamp, name):
"""
Clone a vmachine using the disk snapshot based on a snapshot timestamp
@param machineguid: guid of the machine to clone
@param timestamp: timestamp of the disk snapshots to use for the clone
@param name: name for the new machine
"""
machine = VMachine(machineguid)
timestamp = str(timestamp)
if timestamp not in (snap['timestamp'] for snap in machine.snapshots):
raise RuntimeError(
'Invalid timestamp provided, not a valid snapshot of this vmachine.'
)
vpool = None
storagerouter = None
if machine.pmachine is not None and machine.pmachine.hvtype == 'VMWARE':
for vdisk in machine.vdisks:
if vdisk.vpool is not None:
vpool = vdisk.vpool
break
for vdisk in machine.vdisks:
if vdisk.storagerouter_guid:
storagerouter = StorageRouter(vdisk.storagerouter_guid)
break
hv = Factory.get(machine.pmachine)
vm_path = hv.get_vmachine_path(
name,
storagerouter.machine_id if storagerouter is not None else '')
# mutex in sync_with_hypervisor uses "None" for KVM hvtype
mutex = VolatileMutex('{0}_{1}'.format(
hv.clean_vmachine_filename(vm_path),
vpool.guid if vpool is not None else 'none'))
disks = {}
for snapshot in machine.snapshots:
if snapshot['timestamp'] == timestamp:
for diskguid, snapshotguid in snapshot['snapshots'].iteritems(
):
disks[diskguid] = snapshotguid
try:
mutex.acquire(wait=120)
new_machine = VMachine()
new_machine.copy(machine)
new_machine.name = name
new_machine.devicename = hv.clean_vmachine_filename(vm_path)
new_machine.pmachine = machine.pmachine
new_machine.save()
finally:
mutex.release
new_disk_guids = []
vm_disks = []
mountpoint = None
disks_by_order = sorted(machine.vdisks, key=lambda x: x.order)
try:
for currentDisk in disks_by_order:
if machine.is_vtemplate and currentDisk.templatesnapshot:
snapshotid = currentDisk.templatesnapshot
else:
snapshotid = disks[currentDisk.guid]
prefix = '%s-clone' % currentDisk.name
result = VDiskController.clone(
diskguid=currentDisk.guid,
snapshotid=snapshotid,
devicename=prefix,
pmachineguid=new_machine.pmachine_guid,
machinename=new_machine.name,
machineguid=new_machine.guid)
new_disk_guids.append(result['diskguid'])
mountpoint = StorageDriverList.get_by_storagedriver_id(
currentDisk.storagedriver_id).mountpoint
vm_disks.append(result)
except Exception as ex:
logger.error('Failed to clone disks. {0}'.format(ex))
VMachineController.delete(machineguid=new_machine.guid)
raise
try:
result = hv.clone_vm(machine.hypervisor_id, name, vm_disks,
mountpoint)
except Exception as ex:
logger.error('Failed to clone vm. {0}'.format(ex))
VMachineController.delete(machineguid=new_machine.guid)
raise
try:
mutex.acquire(wait=120)
new_machine.hypervisor_id = result
new_machine.save()
finally:
mutex.release()
return new_machine.guid
