def _create_vdisks_for_mds_service(self, amount, start_id, mds_service=None, vpool=None): """ Generates vdisks and appends them to a given mds_service """ vdisks = {} for i in xrange(start_id, start_id + amount): disk = VDisk() disk.name = str(i) disk.devicename = 'disk_{0}'.format(i) disk.volume_id = 'disk_{0}'.format(i) disk.vpool = mds_service.vpool if mds_service is not None else vpool disk.size = 0 disk.save() disk.reload_client() if mds_service is not None: storagedriver_id = None for sd in mds_service.vpool.storagedrivers: if sd.storagerouter_guid == mds_service.service.storagerouter_guid: storagedriver_id = sd.storagedriver_id junction = MDSServiceVDisk() junction.vdisk = disk junction.mds_service = mds_service junction.is_master = True junction.save() config = type('MDSNodeConfig', (), {'address': self._generate_nc_function(True, mds_service), 'port': self._generate_nc_function(False, mds_service)})() mds_backend_config = type('MDSMetaDataBackendConfig', (), {'node_configs': self._generate_bc_function([config])})() StorageDriverClient.metadata_backend_config['disk_{0}'.format(i)] = mds_backend_config StorageDriverClient.catch_up['disk_{0}'.format(i)] = 50 StorageDriverClient.vrouter_id['disk_{0}'.format(i)] = storagedriver_id vdisks[i] = disk return vdisks
def create_vdisks_for_mds_service(amount, start_id, mds_service=None, storagedriver=None): """ Generates vdisks and appends them to a given mds_service """ if (mds_service is None and storagedriver is None) or (mds_service is not None and storagedriver is not None): raise RuntimeError( 'Either `mds_service` or `storagedriver` should be passed') vdisks = {} storagedriver_id = None vpool = None mds_services = [] if mds_service is not None: mds_services.append(mds_service) for sd in mds_service.vpool.storagedrivers: if sd.storagerouter_guid == mds_service.service.storagerouter_guid: storagedriver_id = sd.storagedriver_id vpool = sd.vpool if storagedriver_id is None: raise RuntimeError( 'The given MDSService is located on a node without StorageDriver' ) else: storagedriver_id = storagedriver.storagedriver_id vpool = storagedriver.vpool srclient = StorageRouterClient(vpool.guid, None) for i in xrange(start_id, start_id + amount): devicename = 'vdisk_{0}'.format(i) mds_backend_config = DalHelper.generate_mds_metadata_backend_config( mds_services) volume_id = srclient.create_volume(devicename, mds_backend_config, 0, str(storagedriver_id)) if len(mds_services) == 1: MDSClient.set_catchup(mds_services[0], volume_id, 50) vdisk = VDisk() vdisk.name = str(i) vdisk.devicename = devicename vdisk.volume_id = volume_id vdisk.vpool = vpool vdisk.size = 0 vdisk.save() vdisk.reload_client('storagedriver') if mds_service is not None: junction = MDSServiceVDisk() junction.vdisk = vdisk junction.mds_service = mds_service junction.is_master = True junction.save() vdisks[i] = vdisk return vdisks
def sync_vdisk_to_reality(vdisk): """ Syncs a vdisk to reality (except hypervisor) :param vdisk: vDisk to synchronize :type vdisk: VDisk :return: None """ vdisk.reload_client('storagedriver') vdisk.invalidate_dynamics(['info']) config = vdisk.info['metadata_backend_config'] config_dict = {} for item in config: if item['ip'] not in config_dict: config_dict[item['ip']] = [] config_dict[item['ip']].append(item['port']) mds_dict = {} for junction in vdisk.mds_services: service = junction.mds_service.service storagerouter = service.storagerouter if config[0]['ip'] == storagerouter.ip and config[0][ 'port'] == service.ports[0]: junction.is_master = True junction.save() if storagerouter.ip not in mds_dict: mds_dict[storagerouter.ip] = [] mds_dict[storagerouter.ip].append(service.ports[0]) elif storagerouter.ip in config_dict and service.ports[ 0] in config_dict[storagerouter.ip]: junction.is_master = False junction.save() if storagerouter.ip not in mds_dict: mds_dict[storagerouter.ip] = [] mds_dict[storagerouter.ip].append(service.ports[0]) else: junction.delete() for ip, ports in config_dict.iteritems(): for port in ports: if ip not in mds_dict or port not in mds_dict[ip]: service = ServiceList.get_by_ip_ports(ip, [port]) if service is not None: mds_service_vdisk = MDSServiceVDisk() mds_service_vdisk.vdisk = vdisk mds_service_vdisk.mds_service = service.mds_service mds_service_vdisk.is_master = config[0][ 'ip'] == service.storagerouter.ip and config[0][ 'port'] == service.ports[0] mds_service_vdisk.save()
def sync_vdisk_to_reality(vdisk): """ Syncs a vdisk to reality (except hypervisor) :param vdisk: vDisk to synchronize :type vdisk: VDisk :return: None """ vdisk.reload_client("storagedriver") vdisk.invalidate_dynamics(["info"]) config = vdisk.info["metadata_backend_config"] config_dict = {} for item in config: if item["ip"] not in config_dict: config_dict[item["ip"]] = [] config_dict[item["ip"]].append(item["port"]) mds_dict = {} for junction in vdisk.mds_services: service = junction.mds_service.service storagerouter = service.storagerouter if config[0]["ip"] == storagerouter.ip and config[0]["port"] == service.ports[0]: junction.is_master = True junction.save() if storagerouter.ip not in mds_dict: mds_dict[storagerouter.ip] = [] mds_dict[storagerouter.ip].append(service.ports[0]) elif storagerouter.ip in config_dict and service.ports[0] in config_dict[storagerouter.ip]: junction.is_master = False junction.save() if storagerouter.ip not in mds_dict: mds_dict[storagerouter.ip] = [] mds_dict[storagerouter.ip].append(service.ports[0]) else: junction.delete() for ip, ports in config_dict.iteritems(): for port in ports: if ip not in mds_dict or port not in mds_dict[ip]: service = ServiceList.get_by_ip_ports(ip, [port]) if service is not None: mds_service_vdisk = MDSServiceVDisk() mds_service_vdisk.vdisk = vdisk mds_service_vdisk.mds_service = service.mds_service mds_service_vdisk.is_master = ( config[0]["ip"] == service.storagerouter.ip and config[0]["port"] == service.ports[0] ) mds_service_vdisk.save()
def _create_vdisks_for_mds_service(self, amount, start_id, mds_service=None, vpool=None): """ Generates vdisks and appends them to a given mds_service """ vdisks = {} for i in xrange(start_id, start_id + amount): disk = VDisk() disk.name = str(i) disk.devicename = 'disk_{0}'.format(i) disk.volume_id = 'disk_{0}'.format(i) disk.vpool = mds_service.vpool if mds_service is not None else vpool disk.size = 0 disk.save() disk.reload_client() if mds_service is not None: storagedriver_id = None for sd in mds_service.vpool.storagedrivers: if sd.storagerouter_guid == mds_service.service.storagerouter_guid: storagedriver_id = sd.storagedriver_id junction = MDSServiceVDisk() junction.vdisk = disk junction.mds_service = mds_service junction.is_master = True junction.save() config = type( 'MDSNodeConfig', (), { 'address': self._generate_nc_function( True, mds_service), 'port': self._generate_nc_function(False, mds_service) })() mds_backend_config = type( 'MDSMetaDataBackendConfig', (), {'node_configs': self._generate_bc_function([config])})() StorageDriverClient.metadata_backend_config['disk_{0}'.format( i)] = mds_backend_config StorageDriverClient.catch_up['disk_{0}'.format(i)] = 50 StorageDriverClient.vrouter_id['disk_{0}'.format( i)] = storagedriver_id vdisks[i] = disk return vdisks
def create_vdisks_for_mds_service(amount, start_id, mds_service=None, storagedriver=None): """ Generates vdisks and appends them to a given mds_service """ if (mds_service is None and storagedriver is None) or (mds_service is not None and storagedriver is not None): raise RuntimeError("Either `mds_service` or `storagedriver` should be passed") vdisks = {} storagedriver_id = None vpool = None mds_services = [] if mds_service is not None: mds_services.append(mds_service) for sd in mds_service.vpool.storagedrivers: if sd.storagerouter_guid == mds_service.service.storagerouter_guid: storagedriver_id = sd.storagedriver_id vpool = sd.vpool if storagedriver_id is None: raise RuntimeError("The given MDSService is located on a node without StorageDriver") else: storagedriver_id = storagedriver.storagedriver_id vpool = storagedriver.vpool srclient = StorageRouterClient(vpool.guid, None) for i in xrange(start_id, start_id + amount): devicename = "vdisk_{0}".format(i) mds_backend_config = Helper._generate_mdsmetadatabackendconfig(mds_services) volume_id = srclient.create_volume(devicename, mds_backend_config, 0, str(storagedriver_id)) if len(mds_services) == 1: MDSClient.set_catchup(mds_services[0], volume_id, 50) vdisk = VDisk() vdisk.name = str(i) vdisk.devicename = devicename vdisk.volume_id = volume_id vdisk.vpool = vpool vdisk.size = 0 vdisk.save() vdisk.reload_client("storagedriver") if mds_service is not None: junction = MDSServiceVDisk() junction.vdisk = vdisk junction.mds_service = mds_service junction.is_master = True junction.save() vdisks[i] = vdisk return vdisks
def sync_vdisk_to_reality(vdisk): """ Syncs a vdisk to reality (except hypervisor) """ vdisk.reload_client() vdisk.invalidate_dynamics(['info']) config = vdisk.info['metadata_backend_config'] config_dict = {} for item in config: if item['ip'] not in config_dict: config_dict[item['ip']] = [] config_dict[item['ip']].append(item['port']) mds_dict = {} for junction in vdisk.mds_services: service = junction.mds_service.service storagerouter = service.storagerouter if config[0]['ip'] == storagerouter.ip and config[0]['port'] == service.ports[0]: junction.is_master = True junction.save() if storagerouter.ip not in mds_dict: mds_dict[storagerouter.ip] = [] mds_dict[storagerouter.ip].append(service.ports[0]) elif storagerouter.ip in config_dict and service.ports[0] in config_dict[storagerouter.ip]: junction.is_master = False junction.save() if storagerouter.ip not in mds_dict: mds_dict[storagerouter.ip] = [] mds_dict[storagerouter.ip].append(service.ports[0]) else: junction.delete() for ip, ports in config_dict.iteritems(): for port in ports: if ip not in mds_dict or port not in mds_dict[ip]: service = ServiceList.get_by_ip_ports(ip, [port]) if service is not None: mds_service_vdisk = MDSServiceVDisk() mds_service_vdisk.vdisk = vdisk mds_service_vdisk.mds_service = service.mds_service mds_service_vdisk.is_master = config[0]['ip'] == service.storagerouter.ip and config[0]['port'] == service.ports[0] mds_service_vdisk.save()
def _sync_vdisk_to_reality(cls, vdisk): """ Syncs the MDS junction services for a vDisk to the services configured in the StorageDriver :param vdisk: vDisk to synchronize :type vdisk: ovs.dal.hybrids.vdisk.VDisk :return: None :rtype: NoneType """ cls._logger.info('vDisk {0} - {1}: Syncing to reality'.format(vdisk.guid, vdisk.name)) sd_master_ip = None # IP of the master service according to StorageDriver sd_master_port = None # Port of the master service according to StorageDriver sd_mds_config = collections.OrderedDict() # MDS services according to StorageDriver model_mds_config = collections.OrderedDict() # MDS services according to model vdisk.reload_client('storagedriver') vdisk.invalidate_dynamics(['info', 'storagerouter_guid']) # Verify the StorageDriver services cls._logger.debug('vDisk {0} - {1}: Current MDS Config: {2}'.format(vdisk.guid, vdisk.name, vdisk.info['metadata_backend_config'])) for index, mds_entry in enumerate(vdisk.info['metadata_backend_config']): ip = mds_entry['ip'] port = mds_entry['port'] if index == 0: # First entry is the master MDS service sd_master_ip = ip sd_master_port = port if ip not in sd_mds_config: sd_mds_config[ip] = [] sd_mds_config[ip].append(port) # Verify the model junction services (Relations between the MDS Services and the vDisks) for junction in list(vdisk.mds_services): model_ip = junction.mds_service.service.storagerouter.ip model_port = junction.mds_service.service.ports[0] cls._logger.debug('vDisk {0} - {1}: Validating junction service {2}:{3}'.format(vdisk.guid, vdisk.name, model_ip, model_port)) # Remove duplicate junction services if model_ip in model_mds_config and model_port in model_mds_config[model_ip]: cls._logger.warning('vDisk {0} - {1}: Deleting junction service {2}:{3} : Duplicate'.format(vdisk.guid, vdisk.name, model_ip, model_port)) junction.delete() continue # Remove junction services not known by StorageDriver elif model_ip not in sd_mds_config or model_port not in sd_mds_config[model_ip]: cls._logger.warning('vDisk {0} - {1}: Deleting junction service {2}:{3} : Unknown by StorageDriver'.format(vdisk.guid, vdisk.name, model_ip, model_port)) junction.delete() continue junction.is_master = model_ip == sd_master_ip and model_port == sd_master_port junction.save() if model_ip not in model_mds_config: model_mds_config[model_ip] = [] model_mds_config[model_ip].append(model_port) cls._logger.debug('vDisk {0} - {1}: MDS services according to model: {2}'.format(vdisk.guid, vdisk.name, ', '.join(['{0}:{1}'.format(ip, port) for ip, ports in model_mds_config.iteritems() for port in ports]))) cls._logger.debug('vDisk {0} - {1}: MDS services according to StorageDriver: {2}'.format(vdisk.guid, vdisk.name, ', '.join(['{0}:{1}'.format(ip, port) for ip, ports in sd_mds_config.iteritems() for port in ports]))) for ip, ports in sd_mds_config.iteritems(): for port in ports: if ip not in model_mds_config or port not in model_mds_config[ip]: cls._logger.debug('vDisk {0} - {1}: Modeling junction service {2}:{3}'.format(vdisk.guid, vdisk.name, ip, port)) service = ServiceList.get_by_ip_ports(ip, [port]) if service is None and vdisk.storagerouter_guid is not None: cls._logger.critical('vDisk {0} - {1}: Failed to find an MDS Service for {2}:{3}. Creating a new MDS Service'.format(vdisk.guid, vdisk.name, ip, port)) storagerouter = StorageRouter(vdisk.storagerouter_guid) try: service = cls.prepare_mds_service(storagerouter=storagerouter, vpool=vdisk.vpool).service except Exception: cls._logger.exception('vDisk {0} - {1}: Creating MDS Service failed'.format(vdisk.guid, vdisk.name)) if service is not None: mds_service_vdisk = MDSServiceVDisk() mds_service_vdisk.vdisk = vdisk mds_service_vdisk.mds_service = service.mds_service mds_service_vdisk.is_master = sd_master_ip == service.storagerouter.ip and sd_master_port == service.ports[0] mds_service_vdisk.save() cls._logger.debug('vDisk {0} - {1}: Modeled junction service {2}:{3}'.format(vdisk.guid, vdisk.name, ip, port)) cls._logger.info('vDisk {0} - {1}: Synced to reality'.format(vdisk.guid, vdisk.name))