def update_vdisk_name(volume_id, old_name, new_name): """ Update a vDisk name using Management Center: set new name """ vdisk = None for mgmt_center in MgmtCenterList.get_mgmtcenters(): mgmt = Factory.get_mgmtcenter(mgmt_center = mgmt_center) try: disk_info = mgmt.get_vdisk_device_info(volume_id) device_path = disk_info['device_path'] vpool_name = disk_info['vpool_name'] vp = VPoolList.get_vpool_by_name(vpool_name) file_name = os.path.basename(device_path) vdisk = VDiskList.get_by_devicename_and_vpool(file_name, vp) if vdisk: break except Exception as ex: logger.info('Trying to get mgmt center failed for disk {0} with volume_id {1}. {2}'.format(old_name, volume_id, ex)) if not vdisk: logger.error('No vdisk found for name {0}'.format(old_name)) return vpool = vdisk.vpool mutex = VolatileMutex('{}_{}'.format(old_name, vpool.guid if vpool is not None else 'none')) try: mutex.acquire(wait=5) vdisk.name = new_name vdisk.save() finally: mutex.release()
def sync_with_reality(storagerouter_guid=None, max_attempts=3): """ Try to run sync_with_reality, retry in case of failure always run sync, as tasks calling this expect this to be sync :param storagerouter_guid: :return: """ cache = VolatileFactory.get_client() mutex = VolatileMutex('ovs_disk_sync_with_reality_{0}'.format(storagerouter_guid)) key = 'ovs_dedupe_sync_with_reality_{0}'.format(storagerouter_guid) attempt = 1 while attempt < max_attempts: task_id = cache.get(key) if task_id: revoke(task_id) try: mutex.acquire(wait=120) return DiskController._sync_with_reality(storagerouter_guid) except Exception as ex: logger.warning('Sync with reality failed. {0}'.format(ex)) attempt += 1 time.sleep(attempt*30) finally: mutex.release() raise RuntimeError('Sync with reality failed after 3 attempts')
def new_function(self, request, *args, **kwargs): """ Wrapped function """ now = time.time() key = 'ovs_api_limit_{0}.{1}_{2}'.format( f.__module__, f.__name__, request.META['HTTP_X_REAL_IP'] ) client = VolatileFactory.get_client() mutex = VolatileMutex(key) try: mutex.acquire() rate_info = client.get(key, {'calls': [], 'timeout': None}) active_timeout = rate_info['timeout'] if active_timeout is not None: if active_timeout > now: return HttpResponse, {'error_code': 'rate_limit_timeout', 'error': 'Rate limit timeout ({0}s remaining)'.format(round(active_timeout - now, 2))}, 429 else: rate_info['timeout'] = None rate_info['calls'] = [call for call in rate_info['calls'] if call > (now - per)] + [now] calls = len(rate_info['calls']) if calls > amount: rate_info['timeout'] = now + timeout client.set(key, rate_info) return HttpResponse, {'error_code': 'rate_limit_reached', 'error': 'Rate limit reached ({0} in last {1}s)'.format(calls, per)}, 429 client.set(key, rate_info) finally: mutex.release() return f(self, request, *args, **kwargs)
def delete_from_voldrv(volumename, storagedriver_id): """ Delete a disk Triggered by volumedriver messages on the queue @param volumename: volume id of the disk """ _ = storagedriver_id # For logging purposes disk = VDiskList.get_vdisk_by_volume_id(volumename) if disk is not None: mutex = VolatileMutex('{}_{}'.format(volumename, disk.devicename)) try: mutex.acquire(wait=20) pmachine = None try: pmachine = PMachineList.get_by_storagedriver_id(disk.storagedriver_id) except RuntimeError as ex: if 'could not be found' not in str(ex): raise # else: pmachine can't be loaded, because the volumedriver doesn't know about it anymore if pmachine is not None: limit = 5 hypervisor = Factory.get(pmachine) exists = hypervisor.file_exists(disk.vpool, disk.devicename) while limit > 0 and exists is True: time.sleep(1) exists = hypervisor.file_exists(disk.vpool, disk.devicename) limit -= 1 if exists is True: logger.info('Disk {0} still exists, ignoring delete'.format(disk.devicename)) return logger.info('Delete disk {}'.format(disk.name)) disk.delete() finally: mutex.release()
def new_function(self, request, *args, **kwargs): """ Wrapped function """ now = time.time() key = 'ovs_api_limit_{0}.{1}_{2}'.format( f.__module__, f.__name__, request.META['HTTP_X_REAL_IP'] ) client = VolatileFactory.get_client() mutex = VolatileMutex(key) try: mutex.acquire() rate_info = client.get(key, {'calls': [], 'timeout': None}) active_timeout = rate_info['timeout'] if active_timeout is not None: if active_timeout > now: raise Throttled(wait=active_timeout - now) else: rate_info['timeout'] = None rate_info['calls'] = [call for call in rate_info['calls'] if call > (now - per)] + [now] calls = len(rate_info['calls']) if calls > amount: rate_info['timeout'] = now + timeout client.set(key, rate_info) raise Throttled(wait=timeout) client.set(key, rate_info) finally: mutex.release() return f(self, request, *args, **kwargs)
def resize_from_voldrv(volumename, volumesize, volumepath, storagedriver_id): """ Resize a disk Triggered by volumedriver messages on the queue @param volumepath: path on hypervisor to the volume @param volumename: volume id of the disk @param volumesize: size of the volume """ pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id) storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id) hypervisor = Factory.get(pmachine) volumepath = hypervisor.clean_backing_disk_filename(volumepath) mutex = VolatileMutex('{}_{}'.format(volumename, volumepath)) try: mutex.acquire(wait=30) disk = VDiskList.get_vdisk_by_volume_id(volumename) if disk is None: disk = VDiskList.get_by_devicename_and_vpool(volumepath, storagedriver.vpool) if disk is None: disk = VDisk() finally: mutex.release() disk.devicename = volumepath disk.volume_id = volumename disk.size = volumesize disk.vpool = storagedriver.vpool disk.save() VDiskController.sync_with_mgmtcenter(disk, pmachine, storagedriver) MDSServiceController.ensure_safety(disk)
def __init__(self, *args, **kwargs): """ Initializes the distributed scheduler """ self._persistent = PersistentFactory.get_client() self._namespace = 'ovs_celery_beat' self._mutex = VolatileMutex('celery_beat') self._has_lock = False super(DistributedScheduler, self).__init__(*args, **kwargs) logger.debug('DS init')
def create_cluster(cluster_name, ip, base_dir, plugins=None, locked=True): """ Creates a cluster :param locked: Indicates whether the create should run in a locked context (e.g. to prevent port conflicts) :param plugins: Plugins that should be added to the configuration file :param base_dir: Base directory that should contain the data and tlogs :param ip: IP address of the first node of the new cluster :param cluster_name: Name of the cluster """ logger.debug('Creating cluster {0} on {1}'.format(cluster_name, ip)) base_dir = base_dir.rstrip('/') client = SSHClient(ip) if ArakoonInstaller.is_running(cluster_name, client): logger.info('Arakoon service running for cluster {0}'.format(cluster_name)) config = ArakoonClusterConfig(cluster_name, plugins) config.load_config() for node in config.nodes: if node.ip == ip: return {'client_port': node.client_port, 'messaging_port': node.messaging_port} node_name = System.get_my_machine_id(client) home_dir = ArakoonInstaller.ARAKOON_HOME_DIR.format(base_dir, cluster_name) log_dir = ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name) tlog_dir = ArakoonInstaller.ARAKOON_TLOG_DIR.format(base_dir, cluster_name) ArakoonInstaller.clean_leftover_arakoon_data(ip, {log_dir: True, home_dir: False, tlog_dir: False}) port_mutex = None try: if locked is True: from ovs.extensions.generic.volatilemutex import VolatileMutex port_mutex = VolatileMutex('arakoon_install_ports_{0}'.format(ip)) port_mutex.acquire(wait=60) ports = ArakoonInstaller._get_free_ports(client) config = ArakoonClusterConfig(cluster_name, plugins) config.nodes.append(ArakoonNodeConfig(name=node_name, ip=ip, client_port=ports[0], messaging_port=ports[1], log_dir=log_dir, home=home_dir, tlog_dir=tlog_dir)) ArakoonInstaller._deploy(config) finally: if port_mutex is not None: port_mutex.release() logger.debug('Creating cluster {0} on {1} completed'.format(cluster_name, ip)) return {'client_port': ports[0], 'messaging_port': ports[1]}
def new_function(*args, **kw): """ Executes the decorated function in a locked context """ filemutex = FileMutex('messaging') try: filemutex.acquire(wait=5) mutex = VolatileMutex('messaging') try: mutex.acquire(wait=5) return f(*args, **kw) finally: mutex.release() finally: filemutex.release()
def extend_cluster(master_ip, new_ip, cluster_name, base_dir): """ Extends a cluster to a given new node :param base_dir: Base directory that will hold the db and tlogs :param cluster_name: Name of the cluster to be extended :param new_ip: IP address of the node to be added :param master_ip: IP of one of the already existing nodes """ logger.debug('Extending cluster {0} from {1} to {2}'.format(cluster_name, master_ip, new_ip)) base_dir = base_dir.rstrip('/') from ovs.extensions.generic.volatilemutex import VolatileMutex port_mutex = VolatileMutex('arakoon_install_ports_{0}'.format(new_ip)) config = ArakoonClusterConfig(cluster_name) config.load_config() client = SSHClient(new_ip) node_name = System.get_my_machine_id(client) home_dir = ArakoonInstaller.ARAKOON_HOME_DIR.format(base_dir, cluster_name) log_dir = ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name) tlog_dir = ArakoonInstaller.ARAKOON_TLOG_DIR.format(base_dir, cluster_name) ArakoonInstaller.archive_existing_arakoon_data(new_ip, home_dir, ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir), cluster_name) ArakoonInstaller.archive_existing_arakoon_data(new_ip, log_dir, ArakoonInstaller.ARAKOON_LOG_DIR.format(''), cluster_name) ArakoonInstaller.archive_existing_arakoon_data(new_ip, tlog_dir, ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir), cluster_name) try: port_mutex.acquire(wait=60) ports = ArakoonInstaller._get_free_ports(client) if node_name not in [node.name for node in config.nodes]: config.nodes.append(ArakoonNodeConfig(name=node_name, ip=new_ip, client_port=ports[0], messaging_port=ports[1], log_dir=log_dir, home=home_dir, tlog_dir=tlog_dir)) ArakoonInstaller._deploy(config) finally: port_mutex.release() logger.debug('Extending cluster {0} from {1} to {2} completed'.format(cluster_name, master_ip, new_ip)) return {'client_port': ports[0], 'messaging_port': ports[1]}
def new_function(*args, **kwargs): """ Wrapped function """ request = _find_request(args) now = time.time() key = 'ovs_api_limit_{0}.{1}_{2}'.format( f.__module__, f.__name__, request.META['HTTP_X_REAL_IP']) client = VolatileFactory.get_client() with VolatileMutex(key): rate_info = client.get(key, {'calls': [], 'timeout': None}) active_timeout = rate_info['timeout'] if active_timeout is not None: if active_timeout > now: logger.warning( 'Call {0} is being throttled with a wait of {1}'. format(key, active_timeout - now)) raise Throttled(wait=active_timeout - now) else: rate_info['timeout'] = None rate_info['calls'] = [ call for call in rate_info['calls'] if call > (now - per) ] + [now] calls = len(rate_info['calls']) if calls > amount: rate_info['timeout'] = now + timeout client.set(key, rate_info) logger.warning( 'Call {0} is being throttled with a wait of {1}'. format(key, timeout)) raise Throttled(wait=timeout) client.set(key, rate_info) return f(*args, **kwargs)
def setUpClass(cls): """ Sets up the unittest, mocking a certain set of 3rd party libraries and extensions. This makes sure the unittests can be executed without those libraries installed """ # Load dummy stores PersistentFactory.store = DummyPersistentStore() VolatileFactory.store = DummyVolatileStore() # Replace mocked classes sys.modules[ 'ovs.extensions.storageserver.storagedriver'] = StorageDriver # Import required modules/classes after mocking is done from ovs.dal.hybrids.vmachine import VMachine from ovs.dal.hybrids.vdisk import VDisk from ovs.extensions.generic.volatilemutex import VolatileMutex from ovs.lib.vmachine import VMachineController from ovs.lib.vdisk import VDiskController from ovs.lib.scheduledtask import ScheduledTaskController # Globalize mocked classes global VDisk global VMachine global VolatileMutex global VMachineController global VDiskController global ScheduledTaskController _ = VDisk(), VolatileMutex('dummy'), VMachine( ), VMachineController, VDiskController, ScheduledTaskController # Cleaning storage VolatileFactory.store.clean() PersistentFactory.store.clean()
def setUpClass(cls): """ Sets up the unittest, mocking a certain set of 3rd party libraries and extensions. This makes sure the unittests can be executed without those libraries installed """ # Load dummy stores PersistentFactory.store = DummyPersistentStore() VolatileFactory.store = DummyVolatileStore() # Replace mocked classes sys.modules[ 'ovs.extensions.storageserver.storagedriver'] = StorageDriverModule # Import required modules/classes after mocking is done from ovs.dal.hybrids.vdisk import VDisk from ovs.dal.hybrids.service import Service from ovs.dal.hybrids.vpool import VPool from ovs.dal.hybrids.storagerouter import StorageRouter from ovs.dal.hybrids.pmachine import PMachine from ovs.dal.hybrids.servicetype import ServiceType from ovs.dal.hybrids.storagedriver import StorageDriver from ovs.dal.hybrids.backendtype import BackendType from ovs.dal.hybrids.j_mdsservice import MDSService from ovs.dal.hybrids.j_mdsservicevdisk import MDSServiceVDisk from ovs.extensions.generic.volatilemutex import VolatileMutex from ovs.lib.mdsservice import MDSServiceController # Globalize mocked classes global VDisk global VPool global Service global StorageRouter global StorageDriver global BackendType global PMachine global MDSService global ServiceType global MDSServiceVDisk global VolatileMutex global MDSServiceController _ = VDisk(), VPool(), Service(), MDSService(), MDSServiceVDisk(), ServiceType(), \ StorageRouter(), StorageDriver(), BackendType(), PMachine(), \ VolatileMutex('dummy'), MDSServiceController # Configuration def _get(key): c = PersistentFactory.get_client() if c.exists(key): return c.get(key) return None Configuration.get = staticmethod(_get) # Cleaning storage VolatileFactory.store.clean() PersistentFactory.store.clean()
def update_vdisk_name(volume_id, old_name, new_name): """ Update a vDisk name using Management Center: set new name """ vdisk = None for mgmt_center in MgmtCenterList.get_mgmtcenters(): mgmt = Factory.get_mgmtcenter(mgmt_center=mgmt_center) try: disk_info = mgmt.get_vdisk_device_info(volume_id) device_path = disk_info['device_path'] vpool_name = disk_info['vpool_name'] vp = VPoolList.get_vpool_by_name(vpool_name) file_name = os.path.basename(device_path) vdisk = VDiskList.get_by_devicename_and_vpool(file_name, vp) if vdisk: break except Exception as ex: logger.info( 'Trying to get mgmt center failed for disk {0} with volume_id {1}. {2}' .format(old_name, volume_id, ex)) if not vdisk: logger.error('No vdisk found for name {0}'.format(old_name)) return vpool = vdisk.vpool mutex = VolatileMutex('{}_{}'.format( old_name, vpool.guid if vpool is not None else 'none')) try: mutex.acquire(wait=5) vdisk.name = new_name vdisk.save() finally: mutex.release()
def resize_from_voldrv(volumename, volumesize, volumepath, storagedriver_id): """ Resize a disk Triggered by volumedriver messages on the queue @param volumepath: path on hypervisor to the volume @param volumename: volume id of the disk @param volumesize: size of the volume """ pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id) storagedriver = StorageDriverList.get_by_storagedriver_id( storagedriver_id) hypervisor = Factory.get(pmachine) volumepath = hypervisor.clean_backing_disk_filename(volumepath) mutex = VolatileMutex('{}_{}'.format(volumename, volumepath)) try: mutex.acquire(wait=30) disk = VDiskList.get_vdisk_by_volume_id(volumename) if disk is None: disk = VDiskList.get_by_devicename_and_vpool( volumepath, storagedriver.vpool) if disk is None: disk = VDisk() finally: mutex.release() disk.devicename = volumepath disk.volume_id = volumename disk.size = volumesize disk.vpool = storagedriver.vpool disk.save()
def _backend_property(self, function, dynamic): """ Handles the internal caching of dynamic properties """ caller_name = dynamic.name cache_key = '{0}_{1}'.format(self._key, caller_name) mutex = VolatileMutex(cache_key) try: cached_data = self._volatile.get(cache_key) if cached_data is None: if dynamic.locked: mutex.acquire() cached_data = self._volatile.get(cache_key) if cached_data is None: function_info = inspect.getargspec(function) if 'dynamic' in function_info.args: cached_data = function(dynamic=dynamic) # Load data from backend else: cached_data = function() if cached_data is not None: correct, allowed_types, given_type = Toolbox.check_type(cached_data, dynamic.return_type) if not correct: raise TypeError('Dynamic property {0} allows types {1}. {2} given'.format( caller_name, str(allowed_types), given_type )) if dynamic.timeout > 0: self._volatile.set(cache_key, cached_data, dynamic.timeout) return cached_data finally: mutex.release()
def new_function(self, request, *args, **kwargs): """ Wrapped function """ now = time.time() key = 'ovs_api_limit_{0}.{1}_{2}'.format( f.__module__, f.__name__, request.META['HTTP_X_REAL_IP']) client = VolatileFactory.get_client() mutex = VolatileMutex(key) try: mutex.acquire() rate_info = client.get(key, {'calls': [], 'timeout': None}) active_timeout = rate_info['timeout'] if active_timeout is not None: if active_timeout > now: raise Throttled(wait=active_timeout - now) else: rate_info['timeout'] = None rate_info['calls'] = [ call for call in rate_info['calls'] if call > (now - per) ] + [now] calls = len(rate_info['calls']) if calls > amount: rate_info['timeout'] = now + timeout client.set(key, rate_info) raise Throttled(wait=timeout) client.set(key, rate_info) finally: mutex.release() return f(self, request, *args, **kwargs)
def update_vmachine_name(instance_id, old_name, new_name): """ Update a vMachine name: find vmachine by management center instance id, set new name :param instance_id: ID for the virtual machine known by management center :param old_name: Old name of the virtual machine :param new_name: New name for the virtual machine """ vmachine = None for mgmt_center in MgmtCenterList.get_mgmtcenters(): mgmt = Factory.get_mgmtcenter(mgmt_center = mgmt_center) try: machine_info = mgmt.get_vmachine_device_info(instance_id) file_name = machine_info['file_name'] host_name = machine_info['host_name'] vpool_name = machine_info['vpool_name'] storage_router = StorageRouterList.get_by_name(host_name) machine_id = storage_router.machine_id device_name = '{0}/{1}'.format(machine_id, file_name) vp = VPoolList.get_vpool_by_name(vpool_name) vmachine = VMachineList.get_by_devicename_and_vpool(device_name, vp) if vmachine: break vmachine = VMachineList.get_by_devicename_and_vpool(device_name, None) if vmachine: break except Exception as ex: logger.info('Trying to get mgmt center failed for vmachine {0}. {1}'.format(old_name, ex)) if not vmachine: logger.error('No vmachine found for name {0}'.format(old_name)) return vpool = vmachine.vpool mutex = VolatileMutex('{0}_{1}'.format(old_name, vpool.guid if vpool is not None else 'none')) try: mutex.acquire(wait=5) vmachine.name = new_name vmachine.save() finally: mutex.release()
def create_cluster(cluster_name, ip, base_dir, plugins=None, locked=True): """ Creates a cluster :param locked: Indicates whether the create should run in a locked context (e.g. to prevent port conflicts) :param plugins: Plugins that should be added to the configuration file :param base_dir: Base directory that should contain the data and tlogs :param ip: IP address of the first node of the new cluster :param cluster_name: Name of the cluster """ logger.debug('Creating cluster {0} on {1}'.format(cluster_name, ip)) base_dir = base_dir.rstrip('/') client = SSHClient(ip) node_name = System.get_my_machine_id(client) home_dir = ArakoonInstaller.ARAKOON_HOME_DIR.format( base_dir, cluster_name) log_dir = ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name) tlog_dir = ArakoonInstaller.ARAKOON_TLOG_DIR.format( base_dir, cluster_name) ArakoonInstaller.archive_existing_arakoon_data( ip, home_dir, ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir), cluster_name) ArakoonInstaller.archive_existing_arakoon_data( ip, log_dir, ArakoonInstaller.ARAKOON_LOG_DIR.format(''), cluster_name) ArakoonInstaller.archive_existing_arakoon_data( ip, tlog_dir, ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir), cluster_name) port_mutex = None try: if locked is True: from ovs.extensions.generic.volatilemutex import VolatileMutex port_mutex = VolatileMutex( 'arakoon_install_ports_{0}'.format(ip)) port_mutex.acquire(wait=60) ports = ArakoonInstaller._get_free_ports(client) config = ArakoonClusterConfig(cluster_name, plugins) config.nodes.append( ArakoonNodeConfig(name=node_name, ip=ip, client_port=ports[0], messaging_port=ports[1], log_dir=log_dir, home=home_dir, tlog_dir=tlog_dir)) ArakoonInstaller._deploy(config) finally: if port_mutex is not None: port_mutex.release() logger.debug('Creating cluster {0} on {1} completed'.format( cluster_name, ip)) return {'client_port': ports[0], 'messaging_port': ports[1]}
def extend_cluster(master_ip, new_ip, cluster_name, base_dir): """ Extends a cluster to a given new node :param base_dir: Base directory that will hold the db and tlogs :param cluster_name: Name of the cluster to be extended :param new_ip: IP address of the node to be added :param master_ip: IP of one of the already existing nodes """ logger.debug('Extending cluster {0} from {1} to {2}'.format( cluster_name, master_ip, new_ip)) base_dir = base_dir.rstrip('/') from ovs.extensions.generic.volatilemutex import VolatileMutex port_mutex = VolatileMutex('arakoon_install_ports_{0}'.format(new_ip)) config = ArakoonClusterConfig(cluster_name) config.load_config() client = SSHClient(new_ip) node_name = System.get_my_machine_id(client) home_dir = ArakoonInstaller.ARAKOON_HOME_DIR.format( base_dir, cluster_name) log_dir = ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name) tlog_dir = ArakoonInstaller.ARAKOON_TLOG_DIR.format( base_dir, cluster_name) ArakoonInstaller.archive_existing_arakoon_data( new_ip, home_dir, ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir), cluster_name) ArakoonInstaller.archive_existing_arakoon_data( new_ip, log_dir, ArakoonInstaller.ARAKOON_LOG_DIR.format(''), cluster_name) ArakoonInstaller.archive_existing_arakoon_data( new_ip, tlog_dir, ArakoonInstaller.ARAKOON_BASE_DIR.format(base_dir), cluster_name) try: port_mutex.acquire(wait=60) ports = ArakoonInstaller._get_free_ports(client) if node_name not in [node.name for node in config.nodes]: config.nodes.append( ArakoonNodeConfig(name=node_name, ip=new_ip, client_port=ports[0], messaging_port=ports[1], log_dir=log_dir, home=home_dir, tlog_dir=tlog_dir)) ArakoonInstaller._deploy(config) finally: port_mutex.release() logger.debug('Extending cluster {0} from {1} to {2} completed'.format( cluster_name, master_ip, new_ip)) return {'client_port': ports[0], 'messaging_port': ports[1]}
def setUpClass(cls): """ Sets up the unittest, mocking a certain set of 3rd party libraries and extensions. This makes sure the unittests can be executed without those libraries installed """ # Load dummy stores PersistentFactory.store = DummyPersistentStore() VolatileFactory.store = DummyVolatileStore() # Replace mocked classes sys.modules[ 'ovs.extensions.storageserver.storagedriver'] = StorageDriverModule # Import required modules/classes after mocking is done from ovs.dal.hybrids.backendtype import BackendType from ovs.dal.hybrids.disk import Disk from ovs.dal.hybrids.diskpartition import DiskPartition from ovs.dal.hybrids.failuredomain import FailureDomain from ovs.dal.hybrids.pmachine import PMachine from ovs.dal.hybrids.storagerouter import StorageRouter from ovs.dal.hybrids.vdisk import VDisk from ovs.dal.hybrids.vmachine import VMachine from ovs.dal.hybrids.vpool import VPool from ovs.extensions.generic.volatilemutex import VolatileMutex from ovs.lib.vmachine import VMachineController from ovs.lib.vdisk import VDiskController from ovs.lib.scheduledtask import ScheduledTaskController # Globalize mocked classes global Disk global VDisk global VMachine global PMachine global VPool global BackendType global DiskPartition global FailureDomain global StorageRouter global VolatileMutex global VMachineController global VDiskController global ScheduledTaskController _ = VDisk(), VolatileMutex('dummy'), VMachine(), PMachine(), VPool(), BackendType(), FailureDomain(), \ VMachineController, VDiskController, ScheduledTaskController, StorageRouter(), Disk(), DiskPartition() # Cleaning storage VolatileFactory.store.clean() PersistentFactory.store.clean()
def invalidate_dynamics(self, properties=None): """ Invalidates all dynamic property caches. Use with caution, as this action can introduce a short performance hit. """ for dynamic in self._dynamics: if properties is None or dynamic.name in properties: key = '{0}_{1}'.format(self._key, dynamic.name) mutex = VolatileMutex(key) try: if dynamic.locked: mutex.acquire() self._volatile.delete(key) finally: mutex.release()
def new_function(self, request, *args, **kwargs): """ Wrapped function """ now = time.time() key = 'ovs_api_limit_{0}.{1}_{2}'.format( f.__module__, f.__name__, request.META['HTTP_X_REAL_IP']) client = VolatileFactory.get_client() mutex = VolatileMutex(key) try: mutex.acquire() rate_info = client.get(key, {'calls': [], 'timeout': None}) active_timeout = rate_info['timeout'] if active_timeout is not None: if active_timeout > now: logger.warning( 'Call {0} is being throttled with a wait of {1}'. format(key, active_timeout - now)) return HttpResponse, { 'error_code': 'rate_limit_timeout', 'error': 'Rate limit timeout ({0}s remaining)'.format( round(active_timeout - now, 2)) }, 429 else: rate_info['timeout'] = None rate_info['calls'] = [ call for call in rate_info['calls'] if call > (now - per) ] + [now] calls = len(rate_info['calls']) if calls > amount: rate_info['timeout'] = now + timeout client.set(key, rate_info) logger.warning( 'Call {0} is being throttled with a wait of {1}'. format(key, timeout)) return HttpResponse, { 'error_code': 'rate_limit_reached', 'error': 'Rate limit reached ({0} in last {1}s)'.format( calls, per) }, 429 client.set(key, rate_info) finally: mutex.release() return f(self, request, *args, **kwargs)
def update_value(key, append, value_to_store=None): """ Store the specified value in the PersistentFactory :param key: Key to store the value for :param append: If True, the specified value will be appended else element at index 0 will be popped :param value_to_store: Value to append to the list :return: Updated value """ with VolatileMutex(name=key, wait=5): if persistent_client.exists(key): val = persistent_client.get(key) if append is True and value_to_store is not None: val['values'].append(value_to_store) elif append is False and len(val['values']) > 0: val['values'].pop(0) else: log_message('Setting initial value for key {0}'.format( persistent_key)) val = {'mode': mode, 'values': []} persistent_client.set(key, val) return val
def update_vmachine_name(instance_id, old_name, new_name): """ Update a vMachine name: find vmachine by management center instance id, set new name :param instance_id: ID for the virtual machine known by management center :param old_name: Old name of the virtual machine :param new_name: New name for the virtual machine """ vmachine = None for mgmt_center in MgmtCenterList.get_mgmtcenters(): mgmt = Factory.get_mgmtcenter(mgmt_center=mgmt_center) try: machine_info = mgmt.get_vmachine_device_info(instance_id) file_name = machine_info['file_name'] host_name = machine_info['host_name'] vpool_name = machine_info['vpool_name'] storage_router = StorageRouterList.get_by_name(host_name) machine_id = storage_router.machine_id device_name = '{0}/{1}'.format(machine_id, file_name) vp = VPoolList.get_vpool_by_name(vpool_name) vmachine = VMachineList.get_by_devicename_and_vpool( device_name, vp) if vmachine: break vmachine = VMachineList.get_by_devicename_and_vpool( device_name, None) if vmachine: break except Exception as ex: logger.info( 'Trying to get mgmt center failed for vmachine {0}. {1}'. format(old_name, ex)) if not vmachine: logger.error('No vmachine found for name {0}'.format(old_name)) return vpool = vmachine.vpool mutex = VolatileMutex('{0}_{1}'.format( old_name, vpool.guid if vpool is not None else 'none')) try: mutex.acquire(wait=5) vmachine.name = new_name vmachine.save() finally: mutex.release()
def delete_from_voldrv(volumename, storagedriver_id): """ Delete a disk Triggered by volumedriver messages on the queue @param volumename: volume id of the disk """ _ = storagedriver_id # For logging purposes disk = VDiskList.get_vdisk_by_volume_id(volumename) if disk is not None: mutex = VolatileMutex('{}_{}'.format(volumename, disk.devicename)) try: mutex.acquire(wait=20) pmachine = None try: pmachine = PMachineList.get_by_storagedriver_id( disk.storagedriver_id) except RuntimeError as ex: if 'could not be found' not in str(ex): raise # else: pmachine can't be loaded, because the volumedriver doesn't know about it anymore if pmachine is not None: limit = 5 storagedriver = StorageDriverList.get_by_storagedriver_id( storagedriver_id) hypervisor = Factory.get(pmachine) exists = hypervisor.file_exists(storagedriver, disk.devicename) while limit > 0 and exists is True: time.sleep(1) exists = hypervisor.file_exists( storagedriver, disk.devicename) limit -= 1 if exists is True: logger.info( 'Disk {0} still exists, ignoring delete'.format( disk.devicename)) return logger.info('Delete disk {}'.format(disk.name)) for mds_service in disk.mds_services: mds_service.delete() disk.delete() finally: mutex.release()
def __init__(self, guid=None, data=None, datastore_wins=False, volatile=False, _hook=None): """ Loads an object with a given guid. If no guid is given, a new object is generated with a new guid. * guid: The guid indicating which object should be loaded * datastoreWins: Optional boolean indicating save conflict resolve management. ** True: when saving, external modified fields will not be saved ** False: when saving, all changed data will be saved, regardless of external updates ** None: in case changed field were also changed externally, an error will be raised """ # Initialize super class super(DataObject, self).__init__() # Initialize internal fields self._frozen = False self._datastore_wins = datastore_wins self._guid = None # Guid identifier of the object self._original = {} # Original data copy self._metadata = {} # Some metadata, mainly used for unit testing self._data = {} # Internal data storage self._objects = {} # Internal objects storage # Initialize public fields self.dirty = False self.volatile = volatile # Worker fields/objects self._classname = self.__class__.__name__.lower() self._namespace = 'ovs_data' # Namespace of the object self._mutex_listcache = VolatileMutex('listcache_{0}'.format(self._classname)) self._mutex_reverseindex = VolatileMutex('reverseindex') # Rebuild _relation types hybrid_structure = HybridRunner.get_hybrids() for relation in self._relations: if relation.foreign_type is not None: identifier = Descriptor(relation.foreign_type).descriptor['identifier'] if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']: relation.foreign_type = Descriptor().load(hybrid_structure[identifier]).get_object() # Init guid self._new = False if guid is None: self._guid = str(uuid.uuid4()) self._new = True else: guid = str(guid).lower() if re.match('^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', guid) is not None: self._guid = str(guid) else: raise ValueError('The given guid is invalid: {0}'.format(guid)) # Build base keys self._key = '{0}_{1}_{2}'.format(self._namespace, self._classname, self._guid) # Worker mutexes self._mutex_version = VolatileMutex('ovs_dataversion_{0}_{1}'.format(self._classname, self._guid)) # Load data from cache or persistent backend where appropriate self._volatile = VolatileFactory.get_client() self._persistent = PersistentFactory.get_client() self._metadata['cache'] = None if self._new: self._data = {} else: self._data = self._volatile.get(self._key) if self._data is None: Toolbox.log_cache_hit('object_load', False) self._metadata['cache'] = False try: self._data = self._persistent.get(self._key) except KeyNotFoundException: raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format( self.__class__.__name__, self._guid )) else: Toolbox.log_cache_hit('object_load', True) self._metadata['cache'] = True # Set default values on new fields for prop in self._properties: if prop.name not in self._data: self._data[prop.name] = prop.default self._add_property(prop) # Load relations for relation in self._relations: if relation.name not in self._data: if relation.foreign_type is None: cls = self.__class__ else: cls = relation.foreign_type self._data[relation.name] = Descriptor(cls).descriptor self._add_relation_property(relation) # Add wrapped properties for dynamic in self._dynamics: self._add_dynamic_property(dynamic) # Load foreign keys relations = RelationMapper.load_foreign_relations(self.__class__) if relations is not None: for key, info in relations.iteritems(): self._objects[key] = {'info': info, 'data': None} self._add_list_property(key, info['list']) # Store original data self._original = copy.deepcopy(self._data) if _hook is not None and hasattr(_hook, '__call__'): _hook() if not self._new: # Re-cache the object, if required if self._metadata['cache'] is False: # The data wasn't loaded from the cache, so caching is required now try: self._mutex_version.acquire(30) this_version = self._data['_version'] store_version = self._persistent.get(self._key)['_version'] if this_version == store_version: self._volatile.set(self._key, self._data) except KeyNotFoundException: raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format( self.__class__.__name__, self._guid )) finally: self._mutex_version.release() # Freeze property creation self._frozen = True # Optionally, initialize some fields if data is not None: for prop in self._properties: if prop.name in data: setattr(self, prop.name, data[prop.name])
def new_function(*args, **kwargs): """ Wrapped function :param args: Arguments without default values :param kwargs: Arguments with default values """ def log_message(message, level='info'): """ Log a message with some additional information :param message: Message to log :param level: Log level :return: None """ if level not in ('info', 'warning', 'debug', 'error'): raise ValueError( 'Unsupported log level "{0}" specified'.format(level)) complete_message = 'Ensure single {0} mode - ID {1} - {2}'.format( mode, now, message) getattr(logger, level)(complete_message) def update_value(key, append, value_to_store=None): """ Store the specified value in the PersistentFactory :param key: Key to store the value for :param append: If True, the specified value will be appended else element at index 0 will be popped :param value_to_store: Value to append to the list :return: Updated value """ with VolatileMutex(name=key, wait=5): if persistent_client.exists(key): val = persistent_client.get(key) if append is True and value_to_store is not None: val['values'].append(value_to_store) elif append is False and len(val['values']) > 0: val['values'].pop(0) else: log_message('Setting initial value for key {0}'.format( persistent_key)) val = {'mode': mode, 'values': []} persistent_client.set(key, val) return val now = '{0}_{1}'.format( int(time.time()), ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(10))) task_names = [ task_name ] if extra_task_names is None else [task_name] + extra_task_names persistent_key = '{0}_{1}'.format(ENSURE_SINGLE_KEY, task_name) persistent_client = PersistentFactory.get_client() if mode == 'DEFAULT': with VolatileMutex(persistent_key, wait=5): for task in task_names: key_to_check = '{0}_{1}'.format( ENSURE_SINGLE_KEY, task) if persistent_client.exists(key_to_check): log_message( 'Execution of task {0} discarded'.format( task_name)) return None log_message('Setting key {0}'.format(persistent_key)) persistent_client.set(persistent_key, {'mode': mode}) try: output = function(*args, **kwargs) log_message( 'Task {0} finished successfully'.format(task_name)) return output finally: with VolatileMutex(persistent_key, wait=5): if persistent_client.exists(persistent_key): log_message( 'Deleting key {0}'.format(persistent_key)) persistent_client.delete(persistent_key) elif mode == 'CHAINED': if extra_task_names is not None: log_message('Extra tasks are not allowed in this mode', level='error') raise ValueError( 'Ensure single {0} mode - ID {1} - Extra tasks are not allowed in this mode' .format(mode, now)) # 1. Create key to be stored in arakoon and update kwargs with args timeout = kwargs.pop( 'chain_timeout' ) if 'chain_timeout' in kwargs else global_timeout function_info = inspect.getargspec(function) kwargs_dict = {} for index, arg in enumerate(args): kwargs_dict[function_info.args[index]] = arg kwargs_dict.update(kwargs) params_info = 'with params {0}'.format( kwargs_dict) if kwargs_dict else 'with default params' # 2. Set the key in arakoon if non-existent value = update_value(key=persistent_key, append=True) # 3. Validate whether another job with same params is being executed, skip if so for item in value['values'][ 1:]: # 1st element is processing job, we check all other queued jobs for identical params if item['kwargs'] == kwargs_dict: log_message( 'Execution of task {0} {1} discarded because of identical parameters' .format(task_name, params_info)) return None log_message('New task {0} {1} scheduled for execution'.format( task_name, params_info)) update_value(key=persistent_key, append=True, value_to_store={ 'kwargs': kwargs_dict, 'timestamp': now }) # 4. Poll the arakoon to see whether this call is the first in list, if so --> execute, else wait first_element = None counter = 0 while first_element != now and counter < timeout: if persistent_client.exists(persistent_key): value = persistent_client.get(persistent_key) first_element = value['values'][0]['timestamp'] if first_element == now: try: if counter != 0: current_time = int(time.time()) starting_time = int(now.split('_')[0]) log_message( 'Task {0} {1} had to wait {2} seconds before being able to start' .format(task_name, params_info, current_time - starting_time)) output = function(*args, **kwargs) log_message( 'Task {0} finished successfully'.format( task_name)) finally: update_value(key=persistent_key, append=False) return output counter += 1 time.sleep(1) if counter == timeout: update_value(key=persistent_key, append=False) log_message( 'Could not start task {0} {1}, within expected time ({2}s). Removed it from queue' .format(task_name, params_info, timeout), level='error') raise EnsureSingleTimeoutReached( 'Ensure single {0} mode - ID {1} - Task {2} could not be started within timeout of {3}s' .format(mode, now, task_name, timeout)) else: raise ValueError( 'Unsupported mode "{0}" provided'.format(mode))
def update_from_voldrv(name, storagedriver_id): """ This method will update/create a vmachine based on a given vmx/xml file """ pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id) if pmachine.hvtype not in ['VMWARE', 'KVM']: return hypervisor = Factory.get(pmachine) name = hypervisor.clean_vmachine_filename(name) storagedriver = StorageDriverList.get_by_storagedriver_id( storagedriver_id) vpool = storagedriver.vpool machine_ids = [ storagedriver.storagerouter.machine_id for storagedriver in vpool.storagedrivers ] if hypervisor.should_process(name, machine_ids=machine_ids): if pmachine.hvtype == 'VMWARE': storagedriver = StorageDriverList.get_by_storagedriver_id( storagedriver_id) vpool = storagedriver.vpool else: vpool = None pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id) mutex = VolatileMutex('{}_{}'.format( name, vpool.guid if vpool is not None else 'none')) try: mutex.acquire(wait=120) limit = 5 exists = hypervisor.file_exists(storagedriver, name) while limit > 0 and exists is False: time.sleep(1) exists = hypervisor.file_exists(storagedriver, name) limit -= 1 if exists is False: logger.info( 'Could not locate vmachine with name {0} on vpool {1}'. format(name, vpool.name)) vmachine = VMachineList.get_by_devicename_and_vpool( name, vpool) if vmachine is not None: VMachineController.delete_from_voldrv( name, storagedriver_id=storagedriver_id) return finally: mutex.release() try: mutex.acquire(wait=5) vmachine = VMachineList.get_by_devicename_and_vpool( name, vpool) if not vmachine: vmachine = VMachine() vmachine.vpool = vpool vmachine.pmachine = pmachine vmachine.status = 'CREATED' vmachine.devicename = name vmachine.save() finally: mutex.release() if pmachine.hvtype == 'KVM': try: VMachineController.sync_with_hypervisor( vmachine.guid, storagedriver_id=storagedriver_id) vmachine.status = 'SYNC' except: vmachine.status = 'SYNC_NOK' vmachine.save() else: logger.info('Ignored invalid file {0}'.format(name))
class DataObject(object): """ This base class contains all logic to support our multiple backends and the caching - Storage backends: - Persistent backend for persistent storage (key-value store) - Volatile backend for volatile but fast storage (key-value store) - Storage backends are abstracted and injected into this class, making it possible to use fake backends - Features: - Hybrid property access: - Persistent backend - 3rd party component for "live" properties - Individual cache settings for "live" properties - 1-n relations with automatic property propagation - Recursive save """ __metaclass__ = MetaClass ####################### # Attributes ####################### # Properties that needs to be overwritten by implementation _properties = [] # Blueprint data of the object type _dynamics = [] # Timeout of readonly object properties cache _relations = [] # Blueprint for relations ####################### # Constructor ####################### def __new__(cls, *args, **kwargs): """ Initializes the class """ hybrid_structure = HybridRunner.get_hybrids() identifier = Descriptor(cls).descriptor['identifier'] if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']: new_class = Descriptor().load(hybrid_structure[identifier]).get_object() return super(cls, new_class).__new__(new_class, *args, **kwargs) return super(DataObject, cls).__new__(cls) def __init__(self, guid=None, data=None, datastore_wins=False, volatile=False, _hook=None): """ Loads an object with a given guid. If no guid is given, a new object is generated with a new guid. * guid: The guid indicating which object should be loaded * datastoreWins: Optional boolean indicating save conflict resolve management. ** True: when saving, external modified fields will not be saved ** False: when saving, all changed data will be saved, regardless of external updates ** None: in case changed field were also changed externally, an error will be raised """ # Initialize super class super(DataObject, self).__init__() # Initialize internal fields self._frozen = False self._datastore_wins = datastore_wins self._guid = None # Guid identifier of the object self._original = {} # Original data copy self._metadata = {} # Some metadata, mainly used for unit testing self._data = {} # Internal data storage self._objects = {} # Internal objects storage # Initialize public fields self.dirty = False self.volatile = volatile # Worker fields/objects self._classname = self.__class__.__name__.lower() self._namespace = 'ovs_data' # Namespace of the object self._mutex_listcache = VolatileMutex('listcache_{0}'.format(self._classname)) self._mutex_reverseindex = VolatileMutex('reverseindex') # Rebuild _relation types hybrid_structure = HybridRunner.get_hybrids() for relation in self._relations: if relation.foreign_type is not None: identifier = Descriptor(relation.foreign_type).descriptor['identifier'] if identifier in hybrid_structure and identifier != hybrid_structure[identifier]['identifier']: relation.foreign_type = Descriptor().load(hybrid_structure[identifier]).get_object() # Init guid self._new = False if guid is None: self._guid = str(uuid.uuid4()) self._new = True else: guid = str(guid).lower() if re.match('^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', guid) is not None: self._guid = str(guid) else: raise ValueError('The given guid is invalid: {0}'.format(guid)) # Build base keys self._key = '{0}_{1}_{2}'.format(self._namespace, self._classname, self._guid) # Worker mutexes self._mutex_version = VolatileMutex('ovs_dataversion_{0}_{1}'.format(self._classname, self._guid)) # Load data from cache or persistent backend where appropriate self._volatile = VolatileFactory.get_client() self._persistent = PersistentFactory.get_client() self._metadata['cache'] = None if self._new: self._data = {} else: self._data = self._volatile.get(self._key) if self._data is None: Toolbox.log_cache_hit('object_load', False) self._metadata['cache'] = False try: self._data = self._persistent.get(self._key) except KeyNotFoundException: raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format( self.__class__.__name__, self._guid )) else: Toolbox.log_cache_hit('object_load', True) self._metadata['cache'] = True # Set default values on new fields for prop in self._properties: if prop.name not in self._data: self._data[prop.name] = prop.default self._add_property(prop) # Load relations for relation in self._relations: if relation.name not in self._data: if relation.foreign_type is None: cls = self.__class__ else: cls = relation.foreign_type self._data[relation.name] = Descriptor(cls).descriptor self._add_relation_property(relation) # Add wrapped properties for dynamic in self._dynamics: self._add_dynamic_property(dynamic) # Load foreign keys relations = RelationMapper.load_foreign_relations(self.__class__) if relations is not None: for key, info in relations.iteritems(): self._objects[key] = {'info': info, 'data': None} self._add_list_property(key, info['list']) # Store original data self._original = copy.deepcopy(self._data) if _hook is not None and hasattr(_hook, '__call__'): _hook() if not self._new: # Re-cache the object, if required if self._metadata['cache'] is False: # The data wasn't loaded from the cache, so caching is required now try: self._mutex_version.acquire(30) this_version = self._data['_version'] store_version = self._persistent.get(self._key)['_version'] if this_version == store_version: self._volatile.set(self._key, self._data) except KeyNotFoundException: raise ObjectNotFoundException('{0} with guid \'{1}\' could not be found'.format( self.__class__.__name__, self._guid )) finally: self._mutex_version.release() # Freeze property creation self._frozen = True # Optionally, initialize some fields if data is not None: for prop in self._properties: if prop.name in data: setattr(self, prop.name, data[prop.name]) ####################### # Helper methods for dynamic getting and setting ####################### def _add_property(self, prop): """ Adds a simple property to the object """ # pylint: disable=protected-access fget = lambda s: s._get_property(prop) fset = lambda s, v: s._set_property(prop, v) # pylint: enable=protected-access setattr(self.__class__, prop.name, property(fget, fset)) def _add_relation_property(self, relation): """ Adds a complex property to the object (hybrids) """ # pylint: disable=protected-access fget = lambda s: s._get_relation_property(relation) fset = lambda s, v: s._set_relation_property(relation, v) gget = lambda s: s._get_guid_property(relation) # pylint: enable=protected-access setattr(self.__class__, relation.name, property(fget, fset)) setattr(self.__class__, '{0}_guid'.format(relation.name), property(gget)) def _add_list_property(self, attribute, list): """ Adds a list (readonly) property to the object """ # pylint: disable=protected-access fget = lambda s: s._get_list_property(attribute) gget = lambda s: s._get_list_guid_property(attribute) # pylint: enable=protected-access setattr(self.__class__, attribute, property(fget)) setattr(self.__class__, ('{0}_guids' if list else '{0}_guid').format(attribute), property(gget)) def _add_dynamic_property(self, dynamic): """ Adds a dynamic property to the object """ # pylint: disable=protected-access fget = lambda s: s._get_dynamic_property(dynamic) # pylint: enable=protected-access setattr(self.__class__, dynamic.name, property(fget)) # Helper method spporting property fetching def _get_property(self, prop): """ Getter for a simple property """ return self._data[prop.name] def _get_relation_property(self, relation): """ Getter for a complex property (hybrid) It will only load the object once and caches it for the lifetime of this object """ attribute = relation.name if attribute not in self._objects: descriptor = Descriptor().load(self._data[attribute]) self._objects[attribute] = descriptor.get_object(instantiate=True) return self._objects[attribute] def _get_guid_property(self, relation): """ Getter for a foreign key property """ attribute = relation.name return self._data[attribute]['guid'] def _get_list_property(self, attribute): """ Getter for the list property It will execute the related query every time to return a list of hybrid objects that refer to this object. The resulting data will be stored or merged into the cached list preserving as much already loaded objects as possible """ info = self._objects[attribute]['info'] remote_class = Descriptor().load(info['class']).get_object() remote_key = info['key'] datalist = DataList.get_relation_set(remote_class, remote_key, self.__class__, attribute, self.guid) if self._objects[attribute]['data'] is None: self._objects[attribute]['data'] = DataObjectList(datalist.data, remote_class) else: self._objects[attribute]['data'].update(datalist.data) if info['list'] is True: return self._objects[attribute]['data'] else: data = self._objects[attribute]['data'] if len(data) > 1: raise InvalidRelationException('More than one element found in {0}'.format(attribute)) return data[0] if len(data) == 1 else None def _get_list_guid_property(self, attribute): """ Getter for guid list property """ dataobjectlist = getattr(self, attribute) if dataobjectlist is None: return None if hasattr(dataobjectlist, '_guids'): return dataobjectlist._guids return dataobjectlist.guid def _get_dynamic_property(self, dynamic): """ Getter for dynamic property, wrapping the internal data loading property in a caching layer """ data_loader = getattr(self, '_{0}'.format(dynamic.name)) return self._backend_property(data_loader, dynamic) # Helper method supporting property setting def _set_property(self, prop, value): """ Setter for a simple property that will validate the type """ self.dirty = True if value is None: self._data[prop.name] = value else: correct, allowed_types, given_type = Toolbox.check_type(value, prop.property_type) if correct: self._data[prop.name] = value else: raise TypeError('Property {0} allows types {1}. {2} given'.format( prop.name, str(allowed_types), given_type )) def _set_relation_property(self, relation, value): """ Setter for a complex property (hybrid) that will validate the type """ self.dirty = True attribute = relation.name if value is None: self._objects[attribute] = None self._data[attribute]['guid'] = None else: descriptor = Descriptor(value.__class__).descriptor if descriptor['identifier'] != self._data[attribute]['identifier']: if descriptor['type'] == self._data[attribute]['type']: logger.error('Corrupt descriptors: {0} vs {1}'.format(descriptor, self._data[attribute])) raise TypeError('An invalid type was given: {0} instead of {1}'.format( descriptor['type'], self._data[attribute]['type'] )) self._objects[attribute] = value self._data[attribute]['guid'] = value.guid def __setattr__(self, key, value): """ __setattr__ hook that will block creating on the fly new properties, except the predefined ones """ if not hasattr(self, '_frozen') or not self._frozen: allowed = True else: # If our object structure is frozen (which is after __init__), we only allow known # property updates: items that are in __dict__ and our own blueprinting dicts allowed = key in self.__dict__ \ or key in (prop.name for prop in self._properties) \ or key in (relation.name for relation in self._relations) \ or key in (dynamic.name for dynamic in self._dynamics) if allowed: super(DataObject, self).__setattr__(key, value) else: raise RuntimeError('Property {0} does not exist on this object.'.format(key)) ####################### # Saving data to persistent store and invalidating volatile store ####################### def save(self, recursive=False, skip=None, _hook=None): """ Save the object to the persistent backend and clear cache, making use of the specified conflict resolve settings. It will also invalidate certain caches if required. For example lists pointing towards this object """ if self.volatile is True: raise VolatileObjectException() tries = 0 successful = False while successful is False: invalid_fields = [] for prop in self._properties: if prop.mandatory is True and self._data[prop.name] is None: invalid_fields.append(prop.name) for relation in self._relations: if relation.mandatory is True and self._data[relation.name]['guid'] is None: invalid_fields.append(relation.name) if len(invalid_fields) > 0: raise MissingMandatoryFieldsException('Missing fields on {0}: {1}'.format(self._classname, ', '.join(invalid_fields))) if recursive: # Save objects that point to us (e.g. disk.vmachine - if this is disk) for relation in self._relations: if relation.name != skip: # disks will be skipped item = getattr(self, relation.name) if item is not None: item.save(recursive=True, skip=relation.foreign_key) # Save object we point at (e.g. machine.vdisks - if this is machine) relations = RelationMapper.load_foreign_relations(self.__class__) if relations is not None: for key, info in relations.iteritems(): if key != skip: # machine will be skipped if info['list'] is True: for item in getattr(self, key).iterloaded(): item.save(recursive=True, skip=info['key']) else: item = getattr(self, key) if item is not None: item.save(recursive=True, skip=info['key']) for relation in self._relations: if self._data[relation.name]['guid'] is not None: if relation.foreign_type is None: cls = self.__class__ else: cls = relation.foreign_type _ = cls(self._data[relation.name]['guid']) try: data = self._persistent.get(self._key) except KeyNotFoundException: if self._new: data = {'_version': 0} else: raise ObjectNotFoundException('{0} with guid \'{1}\' was deleted'.format( self.__class__.__name__, self._guid )) changed_fields = [] data_conflicts = [] for attribute in self._data.keys(): if attribute == '_version': continue if self._data[attribute] != self._original[attribute]: # We changed this value changed_fields.append(attribute) if attribute in data and self._original[attribute] != data[attribute]: # Some other process also wrote to the database if self._datastore_wins is None: # In case we didn't set a policy, we raise the conflicts data_conflicts.append(attribute) elif self._datastore_wins is False: # If the datastore should not win, we just overwrite the data data[attribute] = self._data[attribute] # If the datastore should win, we discard/ignore our change else: # Normal scenario, saving data data[attribute] = self._data[attribute] elif attribute not in data: data[attribute] = self._data[attribute] for attribute in data.keys(): if attribute == '_version': continue if attribute not in self._data: del data[attribute] if data_conflicts: raise ConcurrencyException('Got field conflicts while saving {0}. Conflicts: {1}'.format( self._classname, ', '.join(data_conflicts) )) # Refresh internal data structure self._data = copy.deepcopy(data) caching_keys = [] try: # First, update reverse index try: self._mutex_reverseindex.acquire(60) for relation in self._relations: key = relation.name original_guid = self._original[key]['guid'] new_guid = self._data[key]['guid'] if original_guid != new_guid: if relation.foreign_type is None: classname = self.__class__.__name__.lower() else: classname = relation.foreign_type.__name__.lower() if original_guid is not None: reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, original_guid) reverse_index = self._volatile.get(reverse_key) if reverse_index is not None: if relation.foreign_key in reverse_index: entries = reverse_index[relation.foreign_key] if self.guid in entries: entries.remove(self.guid) reverse_index[relation.foreign_key] = entries caching_keys.append(reverse_key) self._volatile.set(reverse_key, reverse_index) if new_guid is not None: reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, new_guid) reverse_index = self._volatile.get(reverse_key) if reverse_index is not None: if relation.foreign_key in reverse_index: entries = reverse_index[relation.foreign_key] if self.guid not in entries: entries.append(self.guid) reverse_index[relation.foreign_key] = entries caching_keys.append(reverse_key) self._volatile.set(reverse_key, reverse_index) else: reverse_index[relation.foreign_key] = [self.guid] caching_keys.append(reverse_key) self._volatile.set(reverse_key, reverse_index) else: reverse_index = {relation.foreign_key: [self.guid]} caching_keys.append(reverse_key) self._volatile.set(reverse_key, reverse_index) if self._new is True: reverse_key = 'ovs_reverseindex_{0}_{1}'.format(self._classname, self.guid) reverse_index = self._volatile.get(reverse_key) if reverse_index is None: reverse_index = {} relations = RelationMapper.load_foreign_relations(self.__class__) if relations is not None: for key, _ in relations.iteritems(): reverse_index[key] = [] caching_keys.append(reverse_key) self._volatile.set(reverse_key, reverse_index) finally: self._mutex_reverseindex.release() # Second, invalidate property lists try: self._mutex_listcache.acquire(60) cache_key = '{0}_{1}'.format(DataList.cachelink, self._classname) cache_list = Toolbox.try_get(cache_key, {}) change = False for list_key in cache_list.keys(): fields = cache_list[list_key] if ('__all' in fields and self._new) or list(set(fields) & set(changed_fields)): change = True self._volatile.delete(list_key) del cache_list[list_key] if change is True: self._volatile.set(cache_key, cache_list) self._persistent.set(cache_key, cache_list) finally: self._mutex_listcache.release() if _hook is not None and hasattr(_hook, '__call__'): _hook() # Save the data try: self._mutex_version.acquire(30) this_version = self._data['_version'] try: store_version = self._persistent.get(self._key)['_version'] except KeyNotFoundException: store_version = 0 if this_version == store_version: self._data['_version'] = this_version + 1 self._persistent.set(self._key, self._data) self._volatile.delete(self._key) successful = True else: tries += 1 finally: self._mutex_version.release() if tries > 5: raise SaveRaceConditionException() except: for key in caching_keys: self._volatile.delete(key) raise self.invalidate_dynamics() self._original = copy.deepcopy(self._data) self.dirty = False self._new = False ####################### # Other CRUDs ####################### def delete(self, abandon=None, _hook=None): """ Delete the given object. It also invalidates certain lists """ if self.volatile is True: raise VolatileObjectException() # Check foreign relations relations = RelationMapper.load_foreign_relations(self.__class__) if relations is not None: for key, info in relations.iteritems(): items = getattr(self, key) if info['list'] is True: if len(items) > 0: if abandon is not None and (key in abandon or '_all' in abandon): for item in items.itersafe(): setattr(item, info['key'], None) try: item.save() except ObjectNotFoundException: pass else: multi = 'are {0} items'.format(len(items)) if len(items) > 1 else 'is 1 item' raise LinkedObjectException('There {0} left in self.{1}'.format(multi, key)) elif items is not None: # No list (so a 1-to-1 relation), so there should be an object, or None item = items # More clear naming if abandon is not None and (key in abandon or '_all' in abandon): setattr(item, info['key'], None) try: item.save() except ObjectNotFoundException: pass else: raise LinkedObjectException('There is still an item linked in self.{0}'.format(key)) # Delete the object out of the persistent store try: self._persistent.delete(self._key) except KeyNotFoundException: pass # First, update reverse index try: self._mutex_reverseindex.acquire(60) for relation in self._relations: key = relation.name original_guid = self._original[key]['guid'] if original_guid is not None: if relation.foreign_type is None: classname = self.__class__.__name__.lower() else: classname = relation.foreign_type.__name__.lower() reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, original_guid) reverse_index = self._volatile.get(reverse_key) if reverse_index is not None: if relation.foreign_key in reverse_index: entries = reverse_index[relation.foreign_key] if self.guid in entries: entries.remove(self.guid) reverse_index[relation.foreign_key] = entries self._volatile.set(reverse_key, reverse_index) self._volatile.delete('ovs_reverseindex_{0}_{1}'.format(self._classname, self.guid)) finally: self._mutex_reverseindex.release() # Second, invalidate property lists try: self._mutex_listcache.acquire(60) cache_key = '{0}_{1}'.format(DataList.cachelink, self._classname) cache_list = Toolbox.try_get(cache_key, {}) change = False for list_key in cache_list.keys(): fields = cache_list[list_key] if '__all' in fields: change = True self._volatile.delete(list_key) del cache_list[list_key] if change is True: self._volatile.set(cache_key, cache_list) self._persistent.set(cache_key, cache_list) finally: self._mutex_listcache.release() # Delete the object and its properties out of the volatile store self.invalidate_dynamics() self._volatile.delete(self._key) # Discard all pending changes def discard(self): """ Discard all pending changes, reloading the data from the persistent backend """ self.__init__(guid=self._guid, datastore_wins=self._datastore_wins) def invalidate_dynamics(self, properties=None): """ Invalidates all dynamic property caches. Use with caution, as this action can introduce a short performance hit. """ for dynamic in self._dynamics: if properties is None or dynamic.name in properties: key = '{0}_{1}'.format(self._key, dynamic.name) mutex = VolatileMutex(key) try: if dynamic.locked: mutex.acquire() self._volatile.delete(key) finally: mutex.release() def invalidate_cached_objects(self): """ Invalidates cached objects so they are reloaded when used. """ for relation in self._relations: if relation.name in self._objects: del self._objects[relation.name] def export(self): """ Exports this object's data for import in another object """ data = {} for prop in self._properties: data[prop.name] = self._data[prop.name] return data def serialize(self, depth=0): """ Serializes the internal data, getting rid of certain metadata like descriptors """ data = {'guid': self.guid} for relation in self._relations: key = relation.name if depth == 0: data['{0}_guid'.format(key)] = self._data[key]['guid'] else: instance = getattr(self, key) if instance is not None: data[key] = getattr(self, key).serialize(depth=(depth - 1)) else: data[key] = None for prop in self._properties: data[prop.name] = self._data[prop.name] for dynamic in self._dynamics: data[dynamic.name] = getattr(self, dynamic.name) return data def copy(self, other_object, include=None, exclude=None, include_relations=False): """ Copies all _properties (and optionally _relations) properties over from a given hybrid to self. One can pass in a list of properties that should be copied, or a list of properties that should not be copied. Exclude > Include """ if include is not None and not isinstance(include, list): raise TypeError('Argument include should be None or a list of strings') if exclude is not None and not isinstance(exclude, list): raise TypeError('Argument exclude should be None or a list of strings') if self.__class__.__name__ != other_object.__class__.__name__: raise TypeError('Properties can only be loaded from hybrids of the same type') all_properties = [prop.name for prop in self._properties] all_relations = [relation.name for relation in self._relations] if include: properties_to_copy = include else: properties_to_copy = all_properties if include_relations: properties_to_copy += all_relations if exclude: properties_to_copy = [p for p in properties_to_copy if p not in exclude] possible_options = all_properties + (all_relations if include_relations else []) properties_to_copy = [p for p in properties_to_copy if p in possible_options] for key in properties_to_copy: setattr(self, key, getattr(other_object, key)) def updated_on_datastore(self): """ Checks whether this object has been modified on the datastore """ if self.volatile is True: return False this_version = self._data['_version'] cached_object = self._volatile.get(self._key) if cached_object is None: try: backend_version = self._persistent.get(self._key)['_version'] except KeyNotFoundException: backend_version = -1 else: backend_version = cached_object['_version'] return this_version != backend_version ####################### # Properties ####################### @property def guid(self): """ The primary key of the object """ return self._guid ####################### # Helper methods ####################### def _backend_property(self, function, dynamic): """ Handles the internal caching of dynamic properties """ caller_name = dynamic.name cache_key = '{0}_{1}'.format(self._key, caller_name) mutex = VolatileMutex(cache_key) try: cached_data = self._volatile.get(cache_key) if cached_data is None: if dynamic.locked: mutex.acquire() cached_data = self._volatile.get(cache_key) if cached_data is None: function_info = inspect.getargspec(function) if 'dynamic' in function_info.args: cached_data = function(dynamic=dynamic) # Load data from backend else: cached_data = function() if cached_data is not None: correct, allowed_types, given_type = Toolbox.check_type(cached_data, dynamic.return_type) if not correct: raise TypeError('Dynamic property {0} allows types {1}. {2} given'.format( caller_name, str(allowed_types), given_type )) if dynamic.timeout > 0: self._volatile.set(cache_key, cached_data, dynamic.timeout) return cached_data finally: mutex.release() def __str__(self): """ The string representation of a DataObject is the serialized value """ return json.dumps(self.serialize(), indent=4, cls=DataObjectAttributeEncoder) def __hash__(self): """ Defines a hashing equivalent for a given object. The key (object type and guid) is considered to be identifying """ return hash(self._key) def __eq__(self, other): """ Checks whether two objects are the same. """ if not Descriptor.isinstance(other, self.__class__): return False return self.__hash__() == other.__hash__() def __ne__(self, other): """ Checks whether to objects are not the same. """ if not Descriptor.isinstance(other, self.__class__): return True return not self.__eq__(other) @staticmethod def enumerator(name, items): """ Generates an enumerator """ class Enumerator(dict): def __init__(self, *args, **kwargs): super(Enumerator, self).__init__(*args, **kwargs) if isinstance(items, list): enumerator = Enumerator(zip(items, items)) elif isinstance(items, dict): enumerator = Enumerator(**items) else: raise ValueError("Argument 'items' should be a list or a dict. A '{0}' was given".format(type(items))) enumerator.__name__ = name for item in enumerator: setattr(enumerator, item, enumerator[item]) return enumerator
def update_vmachine_config(vmachine, vm_object, pmachine=None): """ Update a vMachine configuration with a given vMachine configuration """ try: vdisks_synced = 0 if vmachine.name is None: MessageController.fire(MessageController.Type.EVENT, {'type': 'vmachine_created', 'metadata': {'name': vm_object['name']}}) elif vmachine.name != vm_object['name']: MessageController.fire(MessageController.Type.EVENT, {'type': 'vmachine_renamed', 'metadata': {'old_name': vmachine.name, 'new_name': vm_object['name']}}) if pmachine is not None: vmachine.pmachine = pmachine vmachine.name = vm_object['name'] vmachine.hypervisor_id = vm_object['id'] vmachine.devicename = vm_object['backing']['filename'] vmachine.save() # Updating and linking disks storagedrivers = StorageDriverList.get_storagedrivers() datastores = dict([('{0}:{1}'.format(storagedriver.storage_ip, storagedriver.mountpoint), storagedriver) for storagedriver in storagedrivers]) vdisk_guids = [] mutex = VolatileMutex('{0}_{1}'.format(vmachine.name, vmachine.devicename)) for disk in vm_object['disks']: ensure_safety = False if disk['datastore'] in vm_object['datastores']: datastore = vm_object['datastores'][disk['datastore']] if datastore in datastores: try: mutex.acquire(wait=10) vdisk = VDiskList.get_by_devicename_and_vpool(disk['filename'], datastores[datastore].vpool) if vdisk is None: # The disk couldn't be located, but is in our datastore. We might be in a recovery scenario vdisk = VDisk() vdisk.vpool = datastores[datastore].vpool vdisk.reload_client() vdisk.devicename = disk['filename'] vdisk.volume_id = vdisk.storagedriver_client.get_volume_id(str(disk['backingfilename'])) vdisk.size = vdisk.info['volume_size'] # Create the disk in a locked context, but don't execute long running-task in same context vdisk.save() ensure_safety = True finally: mutex.release() if ensure_safety: MDSServiceController.ensure_safety(vdisk) # Update the disk with information from the hypervisor if vdisk.vmachine is None: MessageController.fire(MessageController.Type.EVENT, {'type': 'vdisk_attached', 'metadata': {'vmachine_name': vmachine.name, 'vdisk_name': disk['name']}}) vdisk.vmachine = vmachine vdisk.name = disk['name'] vdisk.order = disk['order'] vdisk.save() vdisk_guids.append(vdisk.guid) vdisks_synced += 1 for vdisk in vmachine.vdisks: if vdisk.guid not in vdisk_guids: MessageController.fire(MessageController.Type.EVENT, {'type': 'vdisk_detached', 'metadata': {'vmachine_name': vmachine.name, 'vdisk_name': vdisk.name}}) vdisk.vmachine = None vdisk.save() logger.info('Updating vMachine finished (name {0}, {1} vdisks (re)linked)'.format( vmachine.name, vdisks_synced )) except Exception as ex: logger.info('Error during vMachine update: {0}'.format(str(ex))) raise
class DistributedScheduler(Scheduler): """ Distributed scheduler that can run on multiple nodes at the same time. """ TIMEOUT = 60 * 30 def __init__(self, *args, **kwargs): """ Initializes the distributed scheduler """ self._persistent = PersistentFactory.get_client() self._namespace = 'ovs_celery_beat' self._mutex = VolatileMutex('celery_beat') self._has_lock = False super(DistributedScheduler, self).__init__(*args, **kwargs) logger.debug('DS init') def setup_schedule(self): """ Setups the schedule """ logger.debug('DS setting up schedule') self._load_schedule() self.merge_inplace(DistributedScheduler._discover_schedule()) self.install_default_entries(self.schedule) for entry in self.schedule: logger.debug('* {0}'.format(entry)) logger.debug('DS setting up schedule - done') @staticmethod def _discover_schedule(): schedule = {} path = os.path.join(os.path.dirname(__file__), 'lib') for filename in os.listdir(path): if os.path.isfile(os.path.join( path, filename)) and filename.endswith( '.py') and filename != '__init__.py': name = filename.replace('.py', '') module = imp.load_source(name, os.path.join(path, filename)) for member in inspect.getmembers(module): if inspect.isclass(member[1]) \ and member[1].__module__ == name \ and 'object' in [base.__name__ for base in member[1].__bases__]: for submember in inspect.getmembers(member[1]): if hasattr(submember[1], 'schedule') and isinstance( submember[1].schedule, crontab): schedule[submember[1].name] = { 'task': submember[1].name, 'schedule': submember[1].schedule, 'args': [] } return schedule def _load_schedule(self): """ Loads the most recent schedule from the persistent store """ self.schedule = {} try: logger.debug('DS loading schedule entries') self._mutex.acquire(wait=10) try: self.schedule = cPickle.loads( str( self._persistent.get('{0}_entries'.format( self._namespace)))) except: # In case an exception occurs during loading the schedule, it is ignored and the default schedule # will be used/restored. pass finally: self._mutex.release() def sync(self): if self._has_lock is True: try: logger.debug('DS syncing schedule entries') self._mutex.acquire(wait=10) self._persistent.set('{0}_entries'.format(self._namespace), cPickle.dumps(self.schedule)) finally: self._mutex.release() else: logger.debug('DS skipping sync: lock is not ours') def tick(self): """ Runs one iteration of the scheduler. This is guarded with a distributed lock """ self._has_lock = False try: logger.debug('DS executing tick') self._mutex.acquire(wait=10) node_now = current_app._get_current_object().now() node_timestamp = time.mktime(node_now.timetuple()) node_name = System.get_my_machine_id() try: lock = self._persistent.get('{0}_lock'.format(self._namespace)) except KeyNotFoundException: lock = None if lock is None: # There is no lock yet, so the lock is acquired self._has_lock = True logger.debug('DS there was no lock in tick') else: if lock['name'] == node_name: # The current node holds the lock logger.debug('DS keeps own lock') self._has_lock = True elif node_timestamp - lock[ 'timestamp'] > DistributedScheduler.TIMEOUT: # The current lock is timed out, so the lock is stolen logger.debug('DS last lock refresh is {0}s old'.format( node_timestamp - lock['timestamp'])) logger.debug('DS stealing lock from {0}'.format( lock['name'])) self._load_schedule() self._has_lock = True else: logger.debug('DS lock is not ours') if self._has_lock is True: lock = {'name': node_name, 'timestamp': node_timestamp} logger.debug('DS refreshing lock') self._persistent.set('{0}_lock'.format(self._namespace), lock) finally: self._mutex.release() if self._has_lock is True: logger.debug('DS executing tick workload') remaining_times = [] try: for entry in self.schedule.itervalues(): next_time_to_run = self.maybe_due(entry, self.publisher) if next_time_to_run: remaining_times.append(next_time_to_run) except RuntimeError: pass logger.debug('DS executing tick workload - done') return min(remaining_times + [self.max_interval]) else: return self.max_interval
class DistributedScheduler(Scheduler): """ Distributed scheduler that can run on multiple nodes at the same time. """ TIMEOUT = 60 * 30 def __init__(self, *args, **kwargs): """ Initializes the distributed scheduler """ self._persistent = PersistentFactory.get_client() self._namespace = 'ovs_celery_beat' self._mutex = VolatileMutex('celery_beat') self._has_lock = False super(DistributedScheduler, self).__init__(*args, **kwargs) logger.debug('DS init') def setup_schedule(self): """ Setups the schedule """ logger.debug('DS setting up schedule') self._load_schedule() self.merge_inplace(self.app.conf.CELERYBEAT_SCHEDULE) self.install_default_entries(self.schedule) logger.debug('DS setting up schedule - done') def _load_schedule(self): """ Loads the most recent schedule from the persistent store """ self.schedule = {} try: logger.debug('DS loading schedule entries') self._mutex.acquire(wait=10) try: self.schedule = cPickle.loads( str(self._persistent.get('{0}_entries'.format(self._namespace)))) except: # In case an exception occurs during loading the schedule, it is ignored and the default schedule # will be used/restored. pass finally: self._mutex.release() def sync(self): if self._has_lock is True: try: logger.debug('DS syncing schedule entries') self._mutex.acquire(wait=10) self._persistent.set('{0}_entries'.format( self._namespace), cPickle.dumps(self.schedule)) finally: self._mutex.release() else: logger.debug('DS skipping sync: lock is not ours') def tick(self): """ Runs one iteration of the scheduler. This is guarded with a distributed lock """ self._has_lock = False try: logger.debug('DS executing tick') self._mutex.acquire(wait=10) node_now = current_app._get_current_object().now() node_timestamp = time.mktime(node_now.timetuple()) node_name = Configuration.get('ovs.core.uniqueid') try: lock = self._persistent.get('{0}_lock'.format(self._namespace)) except KeyNotFoundException: lock = None if lock is None: # There is no lock yet, so the lock is acquired self._has_lock = True logger.debug('DS there was no lock in tick') else: if lock['name'] == node_name: # The current node holds the lock logger.debug('DS keeps own lock') self._has_lock = True elif node_timestamp - lock['timestamp'] > DistributedScheduler.TIMEOUT: # The current lock is timed out, so the lock is stolen logger.debug('DS last lock refresh is {0}s old'.format( node_timestamp - lock['timestamp'])) logger.debug( 'DS stealing lock from {0}'.format(lock['name'])) self._load_schedule() self._has_lock = True else: logger.debug('DS lock is not ours') if self._has_lock is True: lock = {'name': node_name, 'timestamp': node_timestamp} logger.debug('DS refreshing lock') self._persistent.set('{0}_lock'.format(self._namespace), lock) finally: self._mutex.release() if self._has_lock is True: logger.debug('DS executing tick workload') remaining_times = [] try: for entry in self.schedule.itervalues(): next_time_to_run = self.maybe_due(entry, self.publisher) if next_time_to_run: remaining_times.append(next_time_to_run) except RuntimeError: pass logger.debug('DS executing tick workload - done') return min(remaining_times + [self.max_interval]) else: return self.max_interval
def _load(self): """ Tries to load the result for the given key from the volatile cache, or executes the query if not yet available. Afterwards (if a key is given), the result will be (re)cached """ self.data = self._volatile.get(self._key) if self._key is not None else None if self.data is None: # The query should be a dictionary: # {'object': Disk, # Object on which the query should be executed # 'data' : DataList.select.XYZ, # The requested result # 'query' : <query>} # The actual query # Where <query> is a query(group) dictionary: # {'type' : DataList.where_operator.ABC, # Whether the items should be AND/OR # 'items': <items>} # The items in the group # Where the <items> is any combination of one or more <filter> or <query> # A <filter> tuple example: # (<field>, DataList.operator.GHI, <value>) # For example EQUALS # The field is any property you would also find on the given object. In case of # properties, you can dot as far as you like. This means you can combine AND and OR # in any possible combination Toolbox.log_cache_hit('datalist', False) hybrid_structure = HybridRunner.get_hybrids() items = self._query['query']['items'] query_type = self._query['query']['type'] query_data = self._query['data'] query_object = self._query['object'] query_object_id = Descriptor(query_object).descriptor['identifier'] if query_object_id in hybrid_structure and query_object_id != hybrid_structure[query_object_id]['identifier']: query_object = Descriptor().load(hybrid_structure[query_object_id]).get_object() invalidations = {query_object.__name__.lower(): ['__all']} DataList._build_invalidations(invalidations, query_object, items) for class_name in invalidations: key = '{0}_{1}'.format(DataList.cachelink, class_name) mutex = VolatileMutex('listcache_{0}'.format(class_name)) try: mutex.acquire(60) cache_list = Toolbox.try_get(key, {}) current_fields = cache_list.get(self._key, []) current_fields = list(set(current_fields + ['__all'] + invalidations[class_name])) cache_list[self._key] = current_fields self._volatile.set(key, cache_list) self._persistent.set(key, cache_list) finally: mutex.release() self.from_cache = False namespace = query_object()._namespace name = query_object.__name__.lower() guids = DataList.get_pks(namespace, name) if query_data == DataList.select.COUNT: self.data = 0 else: self.data = [] for guid in guids: try: instance = query_object(guid) if query_type == DataList.where_operator.AND: include = self._exec_and(instance, items) elif query_type == DataList.where_operator.OR: include = self._exec_or(instance, items) else: raise NotImplementedError('The given operator is not yet implemented.') if include: if query_data == DataList.select.COUNT: self.data += 1 elif query_data == DataList.select.GUIDS: self.data.append(guid) else: raise NotImplementedError('The given selector type is not implemented') except ObjectNotFoundException: pass if 'post_query' in DataList.test_hooks: DataList.test_hooks['post_query'](self) if self._key is not None and len(guids) > 0 and self._can_cache: invalidated = False for class_name in invalidations: key = '{0}_{1}'.format(DataList.cachelink, class_name) cache_list = Toolbox.try_get(key, {}) if self._key not in cache_list: invalidated = True # If the key under which the list should be saved was already invalidated since the invalidations # were saved, the returned list is most likely outdated. This is OK for this result, but the list # won't get cached if invalidated is False: self._volatile.set(self._key, self.data, 300 + randint(0, 300)) # Cache between 5 and 10 minutes else: Toolbox.log_cache_hit('datalist', True) self.from_cache = True return self
def clone(machineguid, timestamp, name): """ Clone a vmachine using the disk snapshot based on a snapshot timestamp @param machineguid: guid of the machine to clone @param timestamp: timestamp of the disk snapshots to use for the clone @param name: name for the new machine """ machine = VMachine(machineguid) timestamp = str(timestamp) if timestamp not in (snap['timestamp'] for snap in machine.snapshots): raise RuntimeError( 'Invalid timestamp provided, not a valid snapshot of this vmachine.' ) vpool = None storagerouter = None if machine.pmachine is not None and machine.pmachine.hvtype == 'VMWARE': for vdisk in machine.vdisks: if vdisk.vpool is not None: vpool = vdisk.vpool break for vdisk in machine.vdisks: if vdisk.storagerouter_guid: storagerouter = StorageRouter(vdisk.storagerouter_guid) break hv = Factory.get(machine.pmachine) vm_path = hv.get_vmachine_path( name, storagerouter.machine_id if storagerouter is not None else '') # mutex in sync_with_hypervisor uses "None" for KVM hvtype mutex = VolatileMutex('{0}_{1}'.format( hv.clean_vmachine_filename(vm_path), vpool.guid if vpool is not None else 'none')) disks = {} for snapshot in machine.snapshots: if snapshot['timestamp'] == timestamp: for diskguid, snapshotguid in snapshot['snapshots'].iteritems( ): disks[diskguid] = snapshotguid try: mutex.acquire(wait=120) new_machine = VMachine() new_machine.copy(machine) new_machine.name = name new_machine.devicename = hv.clean_vmachine_filename(vm_path) new_machine.pmachine = machine.pmachine new_machine.save() finally: mutex.release new_disk_guids = [] vm_disks = [] mountpoint = None disks_by_order = sorted(machine.vdisks, key=lambda x: x.order) try: for currentDisk in disks_by_order: if machine.is_vtemplate and currentDisk.templatesnapshot: snapshotid = currentDisk.templatesnapshot else: snapshotid = disks[currentDisk.guid] prefix = '%s-clone' % currentDisk.name result = VDiskController.clone( diskguid=currentDisk.guid, snapshotid=snapshotid, devicename=prefix, pmachineguid=new_machine.pmachine_guid, machinename=new_machine.name, machineguid=new_machine.guid) new_disk_guids.append(result['diskguid']) mountpoint = StorageDriverList.get_by_storagedriver_id( currentDisk.storagedriver_id).mountpoint vm_disks.append(result) except Exception as ex: logger.error('Failed to clone disks. {0}'.format(ex)) VMachineController.delete(machineguid=new_machine.guid) raise try: result = hv.clone_vm(machine.hypervisor_id, name, vm_disks, mountpoint) except Exception as ex: logger.error('Failed to clone vm. {0}'.format(ex)) VMachineController.delete(machineguid=new_machine.guid) raise try: mutex.acquire(wait=120) new_machine.hypervisor_id = result new_machine.save() finally: mutex.release() return new_machine.guid
def update_from_voldrv(name, storagedriver_id): """ This method will update/create a vmachine based on a given vmx/xml file """ pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id) if pmachine.hvtype not in ['VMWARE', 'KVM']: return hypervisor = Factory.get(pmachine) name = hypervisor.clean_vmachine_filename(name) storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id) vpool = storagedriver.vpool machine_ids = [storagedriver.storagerouter.machine_id for storagedriver in vpool.storagedrivers] if hypervisor.should_process(name, machine_ids=machine_ids): if pmachine.hvtype == 'VMWARE': storagedriver = StorageDriverList.get_by_storagedriver_id(storagedriver_id) vpool = storagedriver.vpool else: vpool = None pmachine = PMachineList.get_by_storagedriver_id(storagedriver_id) mutex = VolatileMutex('{0}_{1}'.format(name, vpool.guid if vpool is not None else 'none')) try: mutex.acquire(wait=120) limit = 5 exists = hypervisor.file_exists(storagedriver, name) while limit > 0 and exists is False: time.sleep(1) exists = hypervisor.file_exists(storagedriver, name) limit -= 1 if exists is False: logger.info('Could not locate vmachine with name {0} on vpool {1}'.format(name, vpool.name)) vmachine = VMachineList.get_by_devicename_and_vpool(name, vpool) if vmachine is not None: VMachineController.delete_from_voldrv(name, storagedriver_id=storagedriver_id) return finally: mutex.release() try: mutex.acquire(wait=5) vmachine = VMachineList.get_by_devicename_and_vpool(name, vpool) if not vmachine: vmachine = VMachine() vmachine.vpool = vpool vmachine.pmachine = pmachine vmachine.status = 'CREATED' vmachine.devicename = name vmachine.save() finally: mutex.release() if pmachine.hvtype == 'KVM': try: VMachineController.sync_with_hypervisor(vmachine.guid, storagedriver_id=storagedriver_id) vmachine.status = 'SYNC' except: vmachine.status = 'SYNC_NOK' vmachine.save() else: logger.info('Ignored invalid file {0}'.format(name))
class DistributedScheduler(Scheduler): """ Distributed scheduler that can run on multiple nodes at the same time. """ TIMEOUT = 60 * 30 def __init__(self, *args, **kwargs): """ Initializes the distributed scheduler """ self._persistent = PersistentFactory.get_client() self._namespace = 'ovs_celery_beat' self._mutex = VolatileMutex('celery_beat') self._has_lock = False super(DistributedScheduler, self).__init__(*args, **kwargs) logger.debug('DS init') def setup_schedule(self): """ Setups the schedule """ logger.debug('DS setting up schedule') self._load_schedule() self.merge_inplace(DistributedScheduler._discover_schedule()) self.install_default_entries(self.schedule) for entry in self.schedule: logger.debug('* {0}'.format(entry)) logger.debug('DS setting up schedule - done') @staticmethod def _discover_schedule(): schedule = {} path = os.path.join(os.path.dirname(__file__), 'lib') for filename in os.listdir(path): if os.path.isfile(os.path.join(path, filename)) and filename.endswith('.py') and filename != '__init__.py': name = filename.replace('.py', '') module = imp.load_source(name, os.path.join(path, filename)) for member in inspect.getmembers(module): if inspect.isclass(member[1]) \ and member[1].__module__ == name \ and 'object' in [base.__name__ for base in member[1].__bases__]: for submember in inspect.getmembers(member[1]): if hasattr(submember[1], 'schedule') and isinstance(submember[1].schedule, crontab): schedule[submember[1].name] = {'task': submember[1].name, 'schedule': submember[1].schedule, 'args': []} return schedule def _load_schedule(self): """ Loads the most recent schedule from the persistent store """ self.schedule = {} try: logger.debug('DS loading schedule entries') self._mutex.acquire(wait=10) try: self.schedule = cPickle.loads(str(self._persistent.get('{0}_entries'.format(self._namespace)))) except: # In case an exception occurs during loading the schedule, it is ignored and the default schedule # will be used/restored. pass finally: self._mutex.release() def sync(self): if self._has_lock is True: try: logger.debug('DS syncing schedule entries') self._mutex.acquire(wait=10) self._persistent.set('{0}_entries'.format( self._namespace), cPickle.dumps(self.schedule)) finally: self._mutex.release() else: logger.debug('DS skipping sync: lock is not ours') def tick(self): """ Runs one iteration of the scheduler. This is guarded with a distributed lock """ self._has_lock = False try: logger.debug('DS executing tick') self._mutex.acquire(wait=10) node_now = current_app._get_current_object().now() node_timestamp = time.mktime(node_now.timetuple()) node_name = System.get_my_machine_id() try: lock = self._persistent.get('{0}_lock'.format(self._namespace)) except KeyNotFoundException: lock = None if lock is None: # There is no lock yet, so the lock is acquired self._has_lock = True logger.debug('DS there was no lock in tick') else: if lock['name'] == node_name: # The current node holds the lock logger.debug('DS keeps own lock') self._has_lock = True elif node_timestamp - lock['timestamp'] > DistributedScheduler.TIMEOUT: # The current lock is timed out, so the lock is stolen logger.debug('DS last lock refresh is {0}s old'.format( node_timestamp - lock['timestamp'])) logger.debug( 'DS stealing lock from {0}'.format(lock['name'])) self._load_schedule() self._has_lock = True else: logger.debug('DS lock is not ours') if self._has_lock is True: lock = {'name': node_name, 'timestamp': node_timestamp} logger.debug('DS refreshing lock') self._persistent.set('{0}_lock'.format(self._namespace), lock) finally: self._mutex.release() if self._has_lock is True: logger.debug('DS executing tick workload') remaining_times = [] try: for entry in self.schedule.itervalues(): next_time_to_run = self.maybe_due(entry, self.publisher) if next_time_to_run: remaining_times.append(next_time_to_run) except RuntimeError: pass logger.debug('DS executing tick workload - done') return min(remaining_times + [self.max_interval]) else: return self.max_interval
def clone(machineguid, timestamp, name): """ Clone a vmachine using the disk snapshot based on a snapshot timestamp @param machineguid: guid of the machine to clone @param timestamp: timestamp of the disk snapshots to use for the clone @param name: name for the new machine """ machine = VMachine(machineguid) timestamp = str(timestamp) if timestamp not in (snap['timestamp'] for snap in machine.snapshots): raise RuntimeError('Invalid timestamp provided, not a valid snapshot of this vmachine.') vpool = None storagerouter = None if machine.pmachine is not None and machine.pmachine.hvtype == 'VMWARE': for vdisk in machine.vdisks: if vdisk.vpool is not None: vpool = vdisk.vpool break for vdisk in machine.vdisks: if vdisk.storagerouter_guid: storagerouter = StorageRouter(vdisk.storagerouter_guid) break hv = Factory.get(machine.pmachine) vm_path = hv.get_vmachine_path(name, storagerouter.machine_id if storagerouter is not None else '') # mutex in sync_with_hypervisor uses "None" for KVM hvtype mutex = VolatileMutex('{0}_{1}'.format(hv.clean_vmachine_filename(vm_path), vpool.guid if vpool is not None else 'none')) disks = {} for snapshot in machine.snapshots: if snapshot['timestamp'] == timestamp: for diskguid, snapshotguid in snapshot['snapshots'].iteritems(): disks[diskguid] = snapshotguid try: mutex.acquire(wait=120) new_machine = VMachine() new_machine.copy(machine) new_machine.name = name new_machine.devicename = hv.clean_vmachine_filename(vm_path) new_machine.pmachine = machine.pmachine new_machine.save() finally: mutex.release new_disk_guids = [] vm_disks = [] mountpoint = None disks_by_order = sorted(machine.vdisks, key=lambda x: x.order) try: for currentDisk in disks_by_order: if machine.is_vtemplate and currentDisk.templatesnapshot: snapshotid = currentDisk.templatesnapshot else: snapshotid = disks[currentDisk.guid] prefix = '%s-clone' % currentDisk.name result = VDiskController.clone(diskguid=currentDisk.guid, snapshotid=snapshotid, devicename=prefix, pmachineguid=new_machine.pmachine_guid, machinename=new_machine.name, machineguid=new_machine.guid) new_disk_guids.append(result['diskguid']) mountpoint = StorageDriverList.get_by_storagedriver_id(currentDisk.storagedriver_id).mountpoint vm_disks.append(result) except Exception as ex: logger.error('Failed to clone disks. {0}'.format(ex)) VMachineController.delete(machineguid=new_machine.guid) raise try: result = hv.clone_vm(machine.hypervisor_id, name, vm_disks, mountpoint) except Exception as ex: logger.error('Failed to clone vm. {0}'.format(ex)) VMachineController.delete(machineguid=new_machine.guid) raise try: mutex.acquire(wait=120) new_machine.hypervisor_id = result new_machine.save() finally: mutex.release() return new_machine.guid
def get_relation_set(remote_class, remote_key, own_class, own_key, own_guid): """ This method will get a DataList for a relation. On a cache miss, the relation DataList will be rebuild and due to the nature of the full table scan, it will update all relations in the mean time. """ # Example: # * remote_class = vDisk # * remote_key = vmachine # * own_class = vMachine # * own_key = vdisks # Called to load the vMachine.vdisks list (resulting in a possible scan of vDisk objects) # * own_guid = this vMachine object's guid volatile = VolatileFactory.get_client() own_name = own_class.__name__.lower() datalist = DataList({}, '{0}_{1}_{2}'.format(own_name, own_guid, remote_key), load=False) reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid) # Check whether the requested information is available in cache reverse_index = volatile.get(reverse_key) if reverse_index is not None and own_key in reverse_index: Toolbox.log_cache_hit('datalist', True) datalist.data = reverse_index[own_key] datalist.from_cache = True return datalist Toolbox.log_cache_hit('datalist', False) mutex = VolatileMutex('reverseindex') remote_name = remote_class.__name__.lower() blueprint_object = remote_class() # vDisk object foreign_guids = {} remote_namespace = blueprint_object._namespace remote_keys = DataList.get_pks(remote_namespace, remote_name) handled_flows = [] for guid in remote_keys: try: instance = remote_class(guid) for relation in blueprint_object._relations: # E.g. vmachine or vpool relation if relation.foreign_type is None: classname = remote_name foreign_namespace = blueprint_object._namespace else: classname = relation.foreign_type.__name__.lower() foreign_namespace = relation.foreign_type()._namespace if classname not in foreign_guids: foreign_guids[classname] = DataList.get_pks( foreign_namespace, classname) flow = '{0}_{1}'.format(classname, relation.foreign_key) if flow not in handled_flows: handled_flows.append(flow) try: mutex.acquire(60) for foreign_guid in foreign_guids[classname]: reverse_key = 'ovs_reverseindex_{0}_{1}'.format( classname, foreign_guid) reverse_index = volatile.get(reverse_key) if reverse_index is None: reverse_index = {} if relation.foreign_key not in reverse_index: reverse_index[relation.foreign_key] = [] volatile.set(reverse_key, reverse_index) finally: mutex.release() key = getattr(instance, '{0}_guid'.format(relation.name)) if key is not None: try: mutex.acquire(60) reverse_index = volatile.get( 'ovs_reverseindex_{0}_{1}'.format( classname, key)) if reverse_index is None: reverse_index = {} if relation.foreign_key not in reverse_index: reverse_index[relation.foreign_key] = [] if guid not in reverse_index[relation.foreign_key]: reverse_index[relation.foreign_key].append( guid) volatile.set( 'ovs_reverseindex_{0}_{1}'.format( classname, key), reverse_index) finally: mutex.release() except ObjectNotFoundException: pass try: mutex.acquire(60) reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid) reverse_index = volatile.get(reverse_key) if reverse_index is None: reverse_index = {} if own_key not in reverse_index: reverse_index[own_key] = [] volatile.set(reverse_key, reverse_index) datalist.data = reverse_index[own_key] datalist.from_cache = False finally: mutex.release() return datalist
def _load(self): """ Tries to load the result for the given key from the volatile cache, or executes the query if not yet available. Afterwards (if a key is given), the result will be (re)cached """ self.data = self._volatile.get( self._key) if self._key is not None else None if self.data is None: # The query should be a dictionary: # {'object': Disk, # Object on which the query should be executed # 'data' : DataList.select.XYZ, # The requested result # 'query' : <query>} # The actual query # Where <query> is a query(group) dictionary: # {'type' : DataList.where_operator.ABC, # Whether the items should be AND/OR # 'items': <items>} # The items in the group # Where the <items> is any combination of one or more <filter> or <query> # A <filter> tuple example: # (<field>, DataList.operator.GHI, <value>) # For example EQUALS # The field is any property you would also find on the given object. In case of # properties, you can dot as far as you like. This means you can combine AND and OR # in any possible combination Toolbox.log_cache_hit('datalist', False) hybrid_structure = HybridRunner.get_hybrids() items = self._query['query']['items'] query_type = self._query['query']['type'] query_data = self._query['data'] query_object = self._query['object'] query_object_id = Descriptor(query_object).descriptor['identifier'] if query_object_id in hybrid_structure and query_object_id != hybrid_structure[ query_object_id]['identifier']: query_object = Descriptor().load( hybrid_structure[query_object_id]).get_object() invalidations = {query_object.__name__.lower(): ['__all']} DataList._build_invalidations(invalidations, query_object, items) for class_name in invalidations: key = '{0}_{1}'.format(DataList.cachelink, class_name) mutex = VolatileMutex('listcache_{0}'.format(class_name)) try: mutex.acquire(60) cache_list = Toolbox.try_get(key, {}) current_fields = cache_list.get(self._key, []) current_fields = list( set(current_fields + ['__all'] + invalidations[class_name])) cache_list[self._key] = current_fields self._volatile.set(key, cache_list) self._persistent.set(key, cache_list) finally: mutex.release() self.from_cache = False namespace = query_object()._namespace name = query_object.__name__.lower() guids = DataList.get_pks(namespace, name) if query_data == DataList.select.COUNT: self.data = 0 else: self.data = [] for guid in guids: try: instance = query_object(guid) if query_type == DataList.where_operator.AND: include = self._exec_and(instance, items) elif query_type == DataList.where_operator.OR: include = self._exec_or(instance, items) else: raise NotImplementedError( 'The given operator is not yet implemented.') if include: if query_data == DataList.select.COUNT: self.data += 1 elif query_data == DataList.select.GUIDS: self.data.append(guid) else: raise NotImplementedError( 'The given selector type is not implemented') except ObjectNotFoundException: pass if self._post_query_hook is not None: self._post_query_hook(self) if self._key is not None and len(guids) > 0 and self._can_cache: invalidated = False for class_name in invalidations: key = '{0}_{1}'.format(DataList.cachelink, class_name) cache_list = Toolbox.try_get(key, {}) if self._key not in cache_list: invalidated = True # If the key under which the list should be saved was already invalidated since the invalidations # were saved, the returned list is most likely outdated. This is OK for this result, but the list # won't get cached if invalidated is False: self._volatile.set( self._key, self.data, 300 + randint(0, 300)) # Cache between 5 and 10 minutes else: Toolbox.log_cache_hit('datalist', True) self.from_cache = True return self
def get_relation_set(remote_class, remote_key, own_class, own_key, own_guid): """ This method will get a DataList for a relation. On a cache miss, the relation DataList will be rebuild and due to the nature of the full table scan, it will update all relations in the mean time. """ # Example: # * remote_class = vDisk # * remote_key = vmachine # * own_class = vMachine # * own_key = vdisks # Called to load the vMachine.vdisks list (resulting in a possible scan of vDisk objects) # * own_guid = this vMachine object's guid volatile = VolatileFactory.get_client() own_name = own_class.__name__.lower() datalist = DataList({}, '{0}_{1}_{2}'.format(own_name, own_guid, remote_key), load=False) reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid) # Check whether the requested information is available in cache reverse_index = volatile.get(reverse_key) if reverse_index is not None and own_key in reverse_index: Toolbox.log_cache_hit('datalist', True) datalist.data = reverse_index[own_key] datalist.from_cache = True return datalist Toolbox.log_cache_hit('datalist', False) mutex = VolatileMutex('reverseindex') remote_name = remote_class.__name__.lower() blueprint_object = remote_class() # vDisk object foreign_guids = {} remote_namespace = blueprint_object._namespace for relation in blueprint_object._relations: # E.g. vmachine or vpool relation if relation.foreign_type is None: classname = remote_name foreign_namespace = blueprint_object._namespace else: classname = relation.foreign_type.__name__.lower() foreign_namespace = relation.foreign_type()._namespace if classname not in foreign_guids: foreign_guids[classname] = DataList.get_pks(foreign_namespace, classname) try: mutex.acquire(60) for foreign_guid in foreign_guids[classname]: reverse_key = 'ovs_reverseindex_{0}_{1}'.format(classname, foreign_guid) reverse_index = volatile.get(reverse_key) if reverse_index is None: reverse_index = {} if relation.foreign_key not in reverse_index: reverse_index[relation.foreign_key] = [] volatile.set(reverse_key, reverse_index) finally: mutex.release() remote_keys = DataList.get_pks(remote_namespace, remote_name) for guid in remote_keys: try: instance = remote_class(guid) for relation in blueprint_object._relations: # E.g. vmachine or vpool relation if relation.foreign_type is None: classname = remote_name else: classname = relation.foreign_type.__name__.lower() key = getattr(instance, '{0}_guid'.format(relation.name)) if key is not None: try: mutex.acquire(60) reverse_index = volatile.get('ovs_reverseindex_{0}_{1}'.format(classname, key)) if reverse_index is None: reverse_index = {} if relation.foreign_key not in reverse_index: reverse_index[relation.foreign_key] = [] if guid not in reverse_index[relation.foreign_key]: if instance.updated_on_datastore(): raise ConcurrencyException() reverse_index[relation.foreign_key].append(guid) volatile.set('ovs_reverseindex_{0}_{1}'.format(classname, key), reverse_index) finally: mutex.release() except ObjectNotFoundException: pass except ConcurrencyException: pass try: mutex.acquire(60) reverse_key = 'ovs_reverseindex_{0}_{1}'.format(own_name, own_guid) reverse_index = volatile.get(reverse_key) if reverse_index is None: reverse_index = {} if own_key not in reverse_index: reverse_index[own_key] = [] volatile.set(reverse_key, reverse_index) datalist.data = reverse_index[own_key] datalist.from_cache = False finally: mutex.release() return datalist
def update_vmachine_config(vmachine, vm_object, pmachine=None): """ Update a vMachine configuration with a given vMachine configuration :param vmachine: Virtual Machine to update :param vm_object: New virtual machine info :param pmachine: Physical machine of the virtual machine """ try: vdisks_synced = 0 if vmachine.name is None: MessageController.fire( MessageController.Type.EVENT, { 'type': 'vmachine_created', 'metadata': { 'name': vm_object['name'] } }) elif vmachine.name != vm_object['name']: MessageController.fire( MessageController.Type.EVENT, { 'type': 'vmachine_renamed', 'metadata': { 'old_name': vmachine.name, 'new_name': vm_object['name'] } }) if pmachine is not None: vmachine.pmachine = pmachine vmachine.name = vm_object['name'] vmachine.hypervisor_id = vm_object['id'] vmachine.devicename = vm_object['backing']['filename'] vmachine.save() # Updating and linking disks storagedrivers = StorageDriverList.get_storagedrivers() datastores = dict([('{0}:{1}'.format(storagedriver.storage_ip, storagedriver.mountpoint), storagedriver) for storagedriver in storagedrivers]) vdisk_guids = [] mutex = VolatileMutex('{0}_{1}'.format(vmachine.name, vmachine.devicename)) for disk in vm_object['disks']: ensure_safety = False if disk['datastore'] in vm_object['datastores']: datastore = vm_object['datastores'][disk['datastore']] if datastore in datastores: try: mutex.acquire(wait=10) vdisk = VDiskList.get_by_devicename_and_vpool( disk['filename'], datastores[datastore].vpool) if vdisk is None: # The disk couldn't be located, but is in our datastore. We might be in a recovery scenario vdisk = VDisk() vdisk.vpool = datastores[datastore].vpool vdisk.reload_client() vdisk.devicename = disk['filename'] vdisk.volume_id = vdisk.storagedriver_client.get_volume_id( str(disk['backingfilename'])) vdisk.size = vdisk.info['volume_size'] # Create the disk in a locked context, but don't execute long running-task in same context vdisk.save() ensure_safety = True finally: mutex.release() if ensure_safety: MDSServiceController.ensure_safety(vdisk) VDiskController.dtl_checkup(vdisk_guid=vdisk.guid) # Update the disk with information from the hypervisor if vdisk.vmachine is None: MessageController.fire( MessageController.Type.EVENT, { 'type': 'vdisk_attached', 'metadata': { 'vmachine_name': vmachine.name, 'vdisk_name': disk['name'] } }) vdisk.vmachine = vmachine vdisk.name = disk['name'] vdisk.order = disk['order'] vdisk.save() vdisk_guids.append(vdisk.guid) vdisks_synced += 1 for vdisk in vmachine.vdisks: if vdisk.guid not in vdisk_guids: MessageController.fire( MessageController.Type.EVENT, { 'type': 'vdisk_detached', 'metadata': { 'vmachine_name': vmachine.name, 'vdisk_name': vdisk.name } }) vdisk.vmachine = None vdisk.save() logger.info( 'Updating vMachine finished (name {0}, {1} vdisks (re)linked)'. format(vmachine.name, vdisks_synced)) except Exception as ex: logger.info('Error during vMachine update: {0}'.format(str(ex))) raise
def test_volatiemutex(self): """ Validates the volatile mutex """ mutex = VolatileMutex('test') mutex.acquire() mutex.acquire() # Should not raise errors mutex.release() mutex.release() # Should not raise errors mutex._volatile.add(mutex.key(), 1, 10) with self.assertRaises(RuntimeError): mutex.acquire(wait=1) mutex._volatile.delete(mutex.key()) mutex.acquire() time.sleep(0.5) mutex.release()