def create_vdisks_for_mds_service(amount, start_id, mds_service=None, storagedriver=None): """ Generates vdisks and appends them to a given mds_service """ if (mds_service is None and storagedriver is None) or (mds_service is not None and storagedriver is not None): raise RuntimeError( 'Either `mds_service` or `storagedriver` should be passed') vdisks = {} storagedriver_id = None vpool = None mds_services = [] if mds_service is not None: mds_services.append(mds_service) for sd in mds_service.vpool.storagedrivers: if sd.storagerouter_guid == mds_service.service.storagerouter_guid: storagedriver_id = sd.storagedriver_id vpool = sd.vpool if storagedriver_id is None: raise RuntimeError( 'The given MDSService is located on a node without StorageDriver' ) else: storagedriver_id = storagedriver.storagedriver_id vpool = storagedriver.vpool srclient = StorageRouterClient(vpool.guid, None) for i in xrange(start_id, start_id + amount): devicename = 'vdisk_{0}'.format(i) mds_backend_config = DalHelper.generate_mds_metadata_backend_config( mds_services) volume_id = srclient.create_volume(devicename, mds_backend_config, 0, str(storagedriver_id)) if len(mds_services) == 1: MDSClient.set_catchup(mds_services[0], volume_id, 50) vdisk = VDisk() vdisk.name = str(i) vdisk.devicename = devicename vdisk.volume_id = volume_id vdisk.vpool = vpool vdisk.size = 0 vdisk.save() vdisk.reload_client('storagedriver') if mds_service is not None: junction = MDSServiceVDisk() junction.vdisk = vdisk junction.mds_service = mds_service junction.is_master = True junction.save() vdisks[i] = vdisk return vdisks
def setUpClass(cls): """ Sets up the unittest, mocking a certain set of 3rd party libraries and extensions. This makes sure the unittests can be executed without those libraries installed """ # Load dummy stores PersistentFactory.store = DummyPersistentStore() VolatileFactory.store = DummyVolatileStore() # Replace mocked classes sys.modules[ 'ovs.extensions.storageserver.storagedriver'] = StorageDriverModule # Import required modules/classes after mocking is done from ovs.dal.hybrids.vdisk import VDisk from ovs.dal.hybrids.service import Service from ovs.dal.hybrids.vpool import VPool from ovs.dal.hybrids.storagerouter import StorageRouter from ovs.dal.hybrids.pmachine import PMachine from ovs.dal.hybrids.servicetype import ServiceType from ovs.dal.hybrids.storagedriver import StorageDriver from ovs.dal.hybrids.backendtype import BackendType from ovs.dal.hybrids.j_mdsservice import MDSService from ovs.dal.hybrids.j_mdsservicevdisk import MDSServiceVDisk from ovs.extensions.generic.volatilemutex import VolatileMutex from ovs.lib.mdsservice import MDSServiceController # Globalize mocked classes global VDisk global VPool global Service global StorageRouter global StorageDriver global BackendType global PMachine global MDSService global ServiceType global MDSServiceVDisk global VolatileMutex global MDSServiceController _ = VDisk(), VPool(), Service(), MDSService(), MDSServiceVDisk(), ServiceType(), \ StorageRouter(), StorageDriver(), BackendType(), PMachine(), \ VolatileMutex('dummy'), MDSServiceController # Configuration def _get(key): c = PersistentFactory.get_client() if c.exists(key): return c.get(key) return None Configuration.get = staticmethod(_get) # Cleaning storage VolatileFactory.store.clean() PersistentFactory.store.clean()
def setUpClass(cls): """ Sets up the unittest, mocking a certain set of 3rd party libraries and extensions. This makes sure the unittests can be executed without those libraries installed """ # Load dummy stores PersistentFactory.store = DummyPersistentStore() VolatileFactory.store = DummyVolatileStore() # Replace mocked classes sys.modules[ 'ovs.extensions.storageserver.storagedriver'] = StorageDriverModule sys.modules['ovs.extensions.hypervisor.hypervisors.kvm'] = KVMModule # Import required modules/classes after mocking is done from ovs.dal.hybrids.backendtype import BackendType from ovs.dal.hybrids.vdisk import VDisk from ovs.dal.hybrids.j_mdsservice import MDSService from ovs.dal.hybrids.j_mdsservicevdisk import MDSServiceVDisk from ovs.lib.vdisk import VDiskController from ovs.dal.hybrids.pmachine import PMachine from ovs.dal.hybrids.vmachine import VMachine from ovs.dal.hybrids.vpool import VPool from ovs.dal.hybrids.storagedriver import StorageDriver from ovs.dal.hybrids.storagerouter import StorageRouter from ovs.dal.hybrids.failuredomain import FailureDomain from ovs.dal.hybrids.service import Service from ovs.dal.hybrids.servicetype import ServiceType from ovs.dal.lists.vdisklist import VDiskList from ovs.lib.mdsservice import MDSServiceController # Globalize mocked classes global VDisk global VDiskController global PMachine global VMachine global BackendType global VPool global StorageDriver global StorageRouter global FailureDomain global MDSService global MDSServiceVDisk global Service global ServiceType global VDiskList global MDSServiceController _ = VDisk(), PMachine(), VMachine(), VDiskController, VPool(), BackendType(), StorageDriver(), StorageRouter(), \ FailureDomain(), MDSService(), MDSServiceVDisk(), Service(), ServiceType(), VDiskList, MDSServiceController # Cleaning storage VolatileFactory.store.clean() PersistentFactory.store.clean()
def sync_vdisk_to_reality(vdisk): """ Syncs a vdisk to reality (except hypervisor) :param vdisk: vDisk to synchronize :type vdisk: VDisk :return: None """ vdisk.reload_client('storagedriver') vdisk.invalidate_dynamics(['info']) config = vdisk.info['metadata_backend_config'] config_dict = {} for item in config: if item['ip'] not in config_dict: config_dict[item['ip']] = [] config_dict[item['ip']].append(item['port']) mds_dict = {} for junction in vdisk.mds_services: service = junction.mds_service.service storagerouter = service.storagerouter if config[0]['ip'] == storagerouter.ip and config[0][ 'port'] == service.ports[0]: junction.is_master = True junction.save() if storagerouter.ip not in mds_dict: mds_dict[storagerouter.ip] = [] mds_dict[storagerouter.ip].append(service.ports[0]) elif storagerouter.ip in config_dict and service.ports[ 0] in config_dict[storagerouter.ip]: junction.is_master = False junction.save() if storagerouter.ip not in mds_dict: mds_dict[storagerouter.ip] = [] mds_dict[storagerouter.ip].append(service.ports[0]) else: junction.delete() for ip, ports in config_dict.iteritems(): for port in ports: if ip not in mds_dict or port not in mds_dict[ip]: service = ServiceList.get_by_ip_ports(ip, [port]) if service is not None: mds_service_vdisk = MDSServiceVDisk() mds_service_vdisk.vdisk = vdisk mds_service_vdisk.mds_service = service.mds_service mds_service_vdisk.is_master = config[0][ 'ip'] == service.storagerouter.ip and config[0][ 'port'] == service.ports[0] mds_service_vdisk.save()
def _create_vdisks_for_mds_service(self, amount, start_id, mds_service=None, vpool=None): """ Generates vdisks and appends them to a given mds_service """ vdisks = {} for i in xrange(start_id, start_id + amount): disk = VDisk() disk.name = str(i) disk.devicename = 'disk_{0}'.format(i) disk.volume_id = 'disk_{0}'.format(i) disk.vpool = mds_service.vpool if mds_service is not None else vpool disk.size = 0 disk.save() disk.reload_client() if mds_service is not None: storagedriver_id = None for sd in mds_service.vpool.storagedrivers: if sd.storagerouter_guid == mds_service.service.storagerouter_guid: storagedriver_id = sd.storagedriver_id junction = MDSServiceVDisk() junction.vdisk = disk junction.mds_service = mds_service junction.is_master = True junction.save() config = type( 'MDSNodeConfig', (), { 'address': self._generate_nc_function( True, mds_service), 'port': self._generate_nc_function(False, mds_service) })() mds_backend_config = type( 'MDSMetaDataBackendConfig', (), {'node_configs': self._generate_bc_function([config])})() StorageDriverClient.metadata_backend_config['disk_{0}'.format( i)] = mds_backend_config StorageDriverClient.catch_up['disk_{0}'.format(i)] = 50 StorageDriverClient.vrouter_id['disk_{0}'.format( i)] = storagedriver_id vdisks[i] = disk return vdisks
def _sync_vdisk_to_reality(cls, vdisk): """ Syncs the MDS junction services for a vDisk to the services configured in the StorageDriver :param vdisk: vDisk to synchronize :type vdisk: ovs.dal.hybrids.vdisk.VDisk :return: None :rtype: NoneType """ cls._logger.info('vDisk {0} - {1}: Syncing to reality'.format(vdisk.guid, vdisk.name)) sd_master_ip = None # IP of the master service according to StorageDriver sd_master_port = None # Port of the master service according to StorageDriver sd_mds_config = collections.OrderedDict() # MDS services according to StorageDriver model_mds_config = collections.OrderedDict() # MDS services according to model vdisk.reload_client('storagedriver') vdisk.invalidate_dynamics(['info', 'storagerouter_guid']) # Verify the StorageDriver services cls._logger.debug('vDisk {0} - {1}: Current MDS Config: {2}'.format(vdisk.guid, vdisk.name, vdisk.info['metadata_backend_config'])) for index, mds_entry in enumerate(vdisk.info['metadata_backend_config']): ip = mds_entry['ip'] port = mds_entry['port'] if index == 0: # First entry is the master MDS service sd_master_ip = ip sd_master_port = port if ip not in sd_mds_config: sd_mds_config[ip] = [] sd_mds_config[ip].append(port) # Verify the model junction services (Relations between the MDS Services and the vDisks) for junction in list(vdisk.mds_services): model_ip = junction.mds_service.service.storagerouter.ip model_port = junction.mds_service.service.ports[0] cls._logger.debug('vDisk {0} - {1}: Validating junction service {2}:{3}'.format(vdisk.guid, vdisk.name, model_ip, model_port)) # Remove duplicate junction services if model_ip in model_mds_config and model_port in model_mds_config[model_ip]: cls._logger.warning('vDisk {0} - {1}: Deleting junction service {2}:{3} : Duplicate'.format(vdisk.guid, vdisk.name, model_ip, model_port)) junction.delete() continue # Remove junction services not known by StorageDriver elif model_ip not in sd_mds_config or model_port not in sd_mds_config[model_ip]: cls._logger.warning('vDisk {0} - {1}: Deleting junction service {2}:{3} : Unknown by StorageDriver'.format(vdisk.guid, vdisk.name, model_ip, model_port)) junction.delete() continue junction.is_master = model_ip == sd_master_ip and model_port == sd_master_port junction.save() if model_ip not in model_mds_config: model_mds_config[model_ip] = [] model_mds_config[model_ip].append(model_port) cls._logger.debug('vDisk {0} - {1}: MDS services according to model: {2}'.format(vdisk.guid, vdisk.name, ', '.join(['{0}:{1}'.format(ip, port) for ip, ports in model_mds_config.iteritems() for port in ports]))) cls._logger.debug('vDisk {0} - {1}: MDS services according to StorageDriver: {2}'.format(vdisk.guid, vdisk.name, ', '.join(['{0}:{1}'.format(ip, port) for ip, ports in sd_mds_config.iteritems() for port in ports]))) for ip, ports in sd_mds_config.iteritems(): for port in ports: if ip not in model_mds_config or port not in model_mds_config[ip]: cls._logger.debug('vDisk {0} - {1}: Modeling junction service {2}:{3}'.format(vdisk.guid, vdisk.name, ip, port)) service = ServiceList.get_by_ip_ports(ip, [port]) if service is None and vdisk.storagerouter_guid is not None: cls._logger.critical('vDisk {0} - {1}: Failed to find an MDS Service for {2}:{3}. Creating a new MDS Service'.format(vdisk.guid, vdisk.name, ip, port)) storagerouter = StorageRouter(vdisk.storagerouter_guid) try: service = cls.prepare_mds_service(storagerouter=storagerouter, vpool=vdisk.vpool).service except Exception: cls._logger.exception('vDisk {0} - {1}: Creating MDS Service failed'.format(vdisk.guid, vdisk.name)) if service is not None: mds_service_vdisk = MDSServiceVDisk() mds_service_vdisk.vdisk = vdisk mds_service_vdisk.mds_service = service.mds_service mds_service_vdisk.is_master = sd_master_ip == service.storagerouter.ip and sd_master_port == service.ports[0] mds_service_vdisk.save() cls._logger.debug('vDisk {0} - {1}: Modeled junction service {2}:{3}'.format(vdisk.guid, vdisk.name, ip, port)) cls._logger.info('vDisk {0} - {1}: Synced to reality'.format(vdisk.guid, vdisk.name))