Пример #1
0
 def _create_vdisks_for_mds_service(self, amount, start_id, mds_service=None, vpool=None):
     """
     Generates vdisks and appends them to a given mds_service
     """
     vdisks = {}
     for i in xrange(start_id, start_id + amount):
         disk = VDisk()
         disk.name = str(i)
         disk.devicename = 'disk_{0}'.format(i)
         disk.volume_id = 'disk_{0}'.format(i)
         disk.vpool = mds_service.vpool if mds_service is not None else vpool
         disk.size = 0
         disk.save()
         disk.reload_client()
         if mds_service is not None:
             storagedriver_id = None
             for sd in mds_service.vpool.storagedrivers:
                 if sd.storagerouter_guid == mds_service.service.storagerouter_guid:
                     storagedriver_id = sd.storagedriver_id
             junction = MDSServiceVDisk()
             junction.vdisk = disk
             junction.mds_service = mds_service
             junction.is_master = True
             junction.save()
             config = type('MDSNodeConfig', (),
                           {'address': self._generate_nc_function(True, mds_service),
                            'port': self._generate_nc_function(False, mds_service)})()
             mds_backend_config = type('MDSMetaDataBackendConfig', (),
                                       {'node_configs': self._generate_bc_function([config])})()
             StorageDriverClient.metadata_backend_config['disk_{0}'.format(i)] = mds_backend_config
             StorageDriverClient.catch_up['disk_{0}'.format(i)] = 50
             StorageDriverClient.vrouter_id['disk_{0}'.format(i)] = storagedriver_id
         vdisks[i] = disk
     return vdisks
Пример #2
0
 def create_vdisks_for_mds_service(amount,
                                   start_id,
                                   mds_service=None,
                                   storagedriver=None):
     """
     Generates vdisks and appends them to a given mds_service
     """
     if (mds_service is None
             and storagedriver is None) or (mds_service is not None
                                            and storagedriver is not None):
         raise RuntimeError(
             'Either `mds_service` or `storagedriver` should be passed')
     vdisks = {}
     storagedriver_id = None
     vpool = None
     mds_services = []
     if mds_service is not None:
         mds_services.append(mds_service)
         for sd in mds_service.vpool.storagedrivers:
             if sd.storagerouter_guid == mds_service.service.storagerouter_guid:
                 storagedriver_id = sd.storagedriver_id
                 vpool = sd.vpool
         if storagedriver_id is None:
             raise RuntimeError(
                 'The given MDSService is located on a node without StorageDriver'
             )
     else:
         storagedriver_id = storagedriver.storagedriver_id
         vpool = storagedriver.vpool
     srclient = StorageRouterClient(vpool.guid, None)
     for i in xrange(start_id, start_id + amount):
         devicename = 'vdisk_{0}'.format(i)
         mds_backend_config = DalHelper.generate_mds_metadata_backend_config(
             mds_services)
         volume_id = srclient.create_volume(devicename, mds_backend_config,
                                            0, str(storagedriver_id))
         if len(mds_services) == 1:
             MDSClient.set_catchup(mds_services[0], volume_id, 50)
         vdisk = VDisk()
         vdisk.name = str(i)
         vdisk.devicename = devicename
         vdisk.volume_id = volume_id
         vdisk.vpool = vpool
         vdisk.size = 0
         vdisk.save()
         vdisk.reload_client('storagedriver')
         if mds_service is not None:
             junction = MDSServiceVDisk()
             junction.vdisk = vdisk
             junction.mds_service = mds_service
             junction.is_master = True
             junction.save()
         vdisks[i] = vdisk
     return vdisks
Пример #3
0
    def sync_vdisk_to_reality(vdisk):
        """
        Syncs a vdisk to reality (except hypervisor)
        :param vdisk: vDisk to synchronize
        :type vdisk: VDisk

        :return: None
        """
        vdisk.reload_client('storagedriver')
        vdisk.invalidate_dynamics(['info'])
        config = vdisk.info['metadata_backend_config']
        config_dict = {}
        for item in config:
            if item['ip'] not in config_dict:
                config_dict[item['ip']] = []
            config_dict[item['ip']].append(item['port'])
        mds_dict = {}
        for junction in vdisk.mds_services:
            service = junction.mds_service.service
            storagerouter = service.storagerouter
            if config[0]['ip'] == storagerouter.ip and config[0][
                    'port'] == service.ports[0]:
                junction.is_master = True
                junction.save()
                if storagerouter.ip not in mds_dict:
                    mds_dict[storagerouter.ip] = []
                mds_dict[storagerouter.ip].append(service.ports[0])
            elif storagerouter.ip in config_dict and service.ports[
                    0] in config_dict[storagerouter.ip]:
                junction.is_master = False
                junction.save()
                if storagerouter.ip not in mds_dict:
                    mds_dict[storagerouter.ip] = []
                mds_dict[storagerouter.ip].append(service.ports[0])
            else:
                junction.delete()
        for ip, ports in config_dict.iteritems():
            for port in ports:
                if ip not in mds_dict or port not in mds_dict[ip]:
                    service = ServiceList.get_by_ip_ports(ip, [port])
                    if service is not None:
                        mds_service_vdisk = MDSServiceVDisk()
                        mds_service_vdisk.vdisk = vdisk
                        mds_service_vdisk.mds_service = service.mds_service
                        mds_service_vdisk.is_master = config[0][
                            'ip'] == service.storagerouter.ip and config[0][
                                'port'] == service.ports[0]
                        mds_service_vdisk.save()
Пример #4
0
    def sync_vdisk_to_reality(vdisk):
        """
        Syncs a vdisk to reality (except hypervisor)
        :param vdisk: vDisk to synchronize
        :type vdisk: VDisk

        :return: None
        """
        vdisk.reload_client("storagedriver")
        vdisk.invalidate_dynamics(["info"])
        config = vdisk.info["metadata_backend_config"]
        config_dict = {}
        for item in config:
            if item["ip"] not in config_dict:
                config_dict[item["ip"]] = []
            config_dict[item["ip"]].append(item["port"])
        mds_dict = {}
        for junction in vdisk.mds_services:
            service = junction.mds_service.service
            storagerouter = service.storagerouter
            if config[0]["ip"] == storagerouter.ip and config[0]["port"] == service.ports[0]:
                junction.is_master = True
                junction.save()
                if storagerouter.ip not in mds_dict:
                    mds_dict[storagerouter.ip] = []
                mds_dict[storagerouter.ip].append(service.ports[0])
            elif storagerouter.ip in config_dict and service.ports[0] in config_dict[storagerouter.ip]:
                junction.is_master = False
                junction.save()
                if storagerouter.ip not in mds_dict:
                    mds_dict[storagerouter.ip] = []
                mds_dict[storagerouter.ip].append(service.ports[0])
            else:
                junction.delete()
        for ip, ports in config_dict.iteritems():
            for port in ports:
                if ip not in mds_dict or port not in mds_dict[ip]:
                    service = ServiceList.get_by_ip_ports(ip, [port])
                    if service is not None:
                        mds_service_vdisk = MDSServiceVDisk()
                        mds_service_vdisk.vdisk = vdisk
                        mds_service_vdisk.mds_service = service.mds_service
                        mds_service_vdisk.is_master = (
                            config[0]["ip"] == service.storagerouter.ip and config[0]["port"] == service.ports[0]
                        )
                        mds_service_vdisk.save()
Пример #5
0
 def _create_vdisks_for_mds_service(self,
                                    amount,
                                    start_id,
                                    mds_service=None,
                                    vpool=None):
     """
     Generates vdisks and appends them to a given mds_service
     """
     vdisks = {}
     for i in xrange(start_id, start_id + amount):
         disk = VDisk()
         disk.name = str(i)
         disk.devicename = 'disk_{0}'.format(i)
         disk.volume_id = 'disk_{0}'.format(i)
         disk.vpool = mds_service.vpool if mds_service is not None else vpool
         disk.size = 0
         disk.save()
         disk.reload_client()
         if mds_service is not None:
             storagedriver_id = None
             for sd in mds_service.vpool.storagedrivers:
                 if sd.storagerouter_guid == mds_service.service.storagerouter_guid:
                     storagedriver_id = sd.storagedriver_id
             junction = MDSServiceVDisk()
             junction.vdisk = disk
             junction.mds_service = mds_service
             junction.is_master = True
             junction.save()
             config = type(
                 'MDSNodeConfig', (), {
                     'address': self._generate_nc_function(
                         True, mds_service),
                     'port': self._generate_nc_function(False, mds_service)
                 })()
             mds_backend_config = type(
                 'MDSMetaDataBackendConfig', (),
                 {'node_configs': self._generate_bc_function([config])})()
             StorageDriverClient.metadata_backend_config['disk_{0}'.format(
                 i)] = mds_backend_config
             StorageDriverClient.catch_up['disk_{0}'.format(i)] = 50
             StorageDriverClient.vrouter_id['disk_{0}'.format(
                 i)] = storagedriver_id
         vdisks[i] = disk
     return vdisks
Пример #6
0
 def create_vdisks_for_mds_service(amount, start_id, mds_service=None, storagedriver=None):
     """
     Generates vdisks and appends them to a given mds_service
     """
     if (mds_service is None and storagedriver is None) or (mds_service is not None and storagedriver is not None):
         raise RuntimeError("Either `mds_service` or `storagedriver` should be passed")
     vdisks = {}
     storagedriver_id = None
     vpool = None
     mds_services = []
     if mds_service is not None:
         mds_services.append(mds_service)
         for sd in mds_service.vpool.storagedrivers:
             if sd.storagerouter_guid == mds_service.service.storagerouter_guid:
                 storagedriver_id = sd.storagedriver_id
                 vpool = sd.vpool
         if storagedriver_id is None:
             raise RuntimeError("The given MDSService is located on a node without StorageDriver")
     else:
         storagedriver_id = storagedriver.storagedriver_id
         vpool = storagedriver.vpool
     srclient = StorageRouterClient(vpool.guid, None)
     for i in xrange(start_id, start_id + amount):
         devicename = "vdisk_{0}".format(i)
         mds_backend_config = Helper._generate_mdsmetadatabackendconfig(mds_services)
         volume_id = srclient.create_volume(devicename, mds_backend_config, 0, str(storagedriver_id))
         if len(mds_services) == 1:
             MDSClient.set_catchup(mds_services[0], volume_id, 50)
         vdisk = VDisk()
         vdisk.name = str(i)
         vdisk.devicename = devicename
         vdisk.volume_id = volume_id
         vdisk.vpool = vpool
         vdisk.size = 0
         vdisk.save()
         vdisk.reload_client("storagedriver")
         if mds_service is not None:
             junction = MDSServiceVDisk()
             junction.vdisk = vdisk
             junction.mds_service = mds_service
             junction.is_master = True
             junction.save()
         vdisks[i] = vdisk
     return vdisks
Пример #7
0
    def sync_vdisk_to_reality(vdisk):
        """
        Syncs a vdisk to reality (except hypervisor)
        """

        vdisk.reload_client()
        vdisk.invalidate_dynamics(['info'])
        config = vdisk.info['metadata_backend_config']
        config_dict = {}
        for item in config:
            if item['ip'] not in config_dict:
                config_dict[item['ip']] = []
            config_dict[item['ip']].append(item['port'])
        mds_dict = {}
        for junction in vdisk.mds_services:
            service = junction.mds_service.service
            storagerouter = service.storagerouter
            if config[0]['ip'] == storagerouter.ip and config[0]['port'] == service.ports[0]:
                junction.is_master = True
                junction.save()
                if storagerouter.ip not in mds_dict:
                    mds_dict[storagerouter.ip] = []
                mds_dict[storagerouter.ip].append(service.ports[0])
            elif storagerouter.ip in config_dict and service.ports[0] in config_dict[storagerouter.ip]:
                junction.is_master = False
                junction.save()
                if storagerouter.ip not in mds_dict:
                    mds_dict[storagerouter.ip] = []
                mds_dict[storagerouter.ip].append(service.ports[0])
            else:
                junction.delete()
        for ip, ports in config_dict.iteritems():
            for port in ports:
                if ip not in mds_dict or port not in mds_dict[ip]:
                    service = ServiceList.get_by_ip_ports(ip, [port])
                    if service is not None:
                        mds_service_vdisk = MDSServiceVDisk()
                        mds_service_vdisk.vdisk = vdisk
                        mds_service_vdisk.mds_service = service.mds_service
                        mds_service_vdisk.is_master = config[0]['ip'] == service.storagerouter.ip and config[0]['port'] == service.ports[0]
                        mds_service_vdisk.save()
Пример #8
0
    def _sync_vdisk_to_reality(cls, vdisk):
        """
        Syncs the MDS junction services for a vDisk to the services configured in the StorageDriver
        :param vdisk: vDisk to synchronize
        :type vdisk: ovs.dal.hybrids.vdisk.VDisk
        :return: None
        :rtype: NoneType
        """
        cls._logger.info('vDisk {0} - {1}: Syncing to reality'.format(vdisk.guid, vdisk.name))

        sd_master_ip = None  # IP of the master service according to StorageDriver
        sd_master_port = None  # Port of the master service according to StorageDriver
        sd_mds_config = collections.OrderedDict()  # MDS services according to StorageDriver
        model_mds_config = collections.OrderedDict()  # MDS services according to model

        vdisk.reload_client('storagedriver')
        vdisk.invalidate_dynamics(['info', 'storagerouter_guid'])

        # Verify the StorageDriver services
        cls._logger.debug('vDisk {0} - {1}: Current MDS Config: {2}'.format(vdisk.guid, vdisk.name, vdisk.info['metadata_backend_config']))
        for index, mds_entry in enumerate(vdisk.info['metadata_backend_config']):
            ip = mds_entry['ip']
            port = mds_entry['port']
            if index == 0:  # First entry is the master MDS service
                sd_master_ip = ip
                sd_master_port = port
            if ip not in sd_mds_config:
                sd_mds_config[ip] = []
            sd_mds_config[ip].append(port)

        # Verify the model junction services (Relations between the MDS Services and the vDisks)
        for junction in list(vdisk.mds_services):
            model_ip = junction.mds_service.service.storagerouter.ip
            model_port = junction.mds_service.service.ports[0]
            cls._logger.debug('vDisk {0} - {1}: Validating junction service {2}:{3}'.format(vdisk.guid, vdisk.name, model_ip, model_port))

            # Remove duplicate junction services
            if model_ip in model_mds_config and model_port in model_mds_config[model_ip]:
                cls._logger.warning('vDisk {0} - {1}: Deleting junction service {2}:{3} : Duplicate'.format(vdisk.guid, vdisk.name, model_ip, model_port))
                junction.delete()
                continue

            # Remove junction services not known by StorageDriver
            elif model_ip not in sd_mds_config or model_port not in sd_mds_config[model_ip]:
                cls._logger.warning('vDisk {0} - {1}: Deleting junction service {2}:{3} : Unknown by StorageDriver'.format(vdisk.guid, vdisk.name, model_ip, model_port))
                junction.delete()
                continue

            junction.is_master = model_ip == sd_master_ip and model_port == sd_master_port
            junction.save()
            if model_ip not in model_mds_config:
                model_mds_config[model_ip] = []
            model_mds_config[model_ip].append(model_port)

        cls._logger.debug('vDisk {0} - {1}: MDS services according to model: {2}'.format(vdisk.guid, vdisk.name, ', '.join(['{0}:{1}'.format(ip, port) for ip, ports in model_mds_config.iteritems() for port in ports])))
        cls._logger.debug('vDisk {0} - {1}: MDS services according to StorageDriver: {2}'.format(vdisk.guid, vdisk.name, ', '.join(['{0}:{1}'.format(ip, port) for ip, ports in sd_mds_config.iteritems() for port in ports])))
        for ip, ports in sd_mds_config.iteritems():
            for port in ports:
                if ip not in model_mds_config or port not in model_mds_config[ip]:
                    cls._logger.debug('vDisk {0} - {1}: Modeling junction service {2}:{3}'.format(vdisk.guid, vdisk.name, ip, port))
                    service = ServiceList.get_by_ip_ports(ip, [port])
                    if service is None and vdisk.storagerouter_guid is not None:
                        cls._logger.critical('vDisk {0} - {1}: Failed to find an MDS Service for {2}:{3}. Creating a new MDS Service'.format(vdisk.guid, vdisk.name, ip, port))
                        storagerouter = StorageRouter(vdisk.storagerouter_guid)
                        try:
                            service = cls.prepare_mds_service(storagerouter=storagerouter, vpool=vdisk.vpool).service
                        except Exception:
                            cls._logger.exception('vDisk {0} - {1}: Creating MDS Service failed'.format(vdisk.guid, vdisk.name))

                    if service is not None:
                        mds_service_vdisk = MDSServiceVDisk()
                        mds_service_vdisk.vdisk = vdisk
                        mds_service_vdisk.mds_service = service.mds_service
                        mds_service_vdisk.is_master = sd_master_ip == service.storagerouter.ip and sd_master_port == service.ports[0]
                        mds_service_vdisk.save()
                        cls._logger.debug('vDisk {0} - {1}: Modeled junction service {2}:{3}'.format(vdisk.guid, vdisk.name, ip, port))
        cls._logger.info('vDisk {0} - {1}: Synced to reality'.format(vdisk.guid, vdisk.name))