Ejemplo n.º 1
0
    def test_single_node(self):
        """
        Execute some DTL checkups on a single node installation
        """
        # Create 1 vdisk in single node without domains
        structure = DalHelper.build_dal_structure(
            {'vpools': [1],
             'vdisks': [(1, 1, 1, 1)],  # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
             'mds_services': [(1, 1)],  # (<id>, <storagedriver_id>)
             'storagerouters': [1],
             'storagedrivers': [(1, 1, 1)]}  # (<id>, <vpool_id>, <sr_id>)
        )
        vpool = structure['vpools'][1]
        vdisk = structure['vdisks'][1]
        storagerouter = structure['storagerouters'][1]
        # || StorageRouter || vDisk | Regular Domain || Recovery Domain ||     DTL Target    ||
        #  |       1        |   1   |                 |                  |                    |
        self._roll_out_dtl_services(vpool=vpool, storagerouters=structure['storagerouters'])
        self._run_and_validate_dtl_checkup(vdisk=vdisk,
                                           validations=[{'key': 'config', 'value': None}])

        # Create some domains, but do not attach them yet
        # || StorageRouter || vDisk | Regular Domain || Recovery Domain ||     DTL Target    ||
        #  |       1        |   1   |                 |                  |                    |
        domains = {}
        for domain_id in range(1, 3):
            domain = Domain()
            domain.name = 'domain_{0}'.format(domain_id)
            domain.save()
            domains[domain_id] = domain

        self._run_and_validate_dtl_checkup(vdisk=vdisk,
                                           validations=[{'key': 'config', 'value': None}])

        # Attach a regular Domain to the single Storage Router
        # || StorageRouter || vDisk | Regular Domain || Recovery Domain ||     DTL Target    ||
        #  |      sr 1      |   1   |     domain 1    |                  |                    |
        sr_domain = StorageRouterDomain()
        sr_domain.backup = False
        sr_domain.domain = domains[1]
        sr_domain.storagerouter = storagerouter
        sr_domain.save()

        self._run_and_validate_dtl_checkup(vdisk=vdisk,
                                           validations=[{'key': 'config', 'value': None}])

        # Attach a recovery Domain to the single Storage Router
        # || StorageRouter || vDisk | Regular Domain || Recovery Domain ||     DTL Target    ||
        #  |      sr 1      |   1   |                 |     domain 1     |                    |
        for junction in storagerouter.domains:
            junction.delete()
        sr_domain = StorageRouterDomain()
        sr_domain.backup = True
        sr_domain.domain = domains[1]
        sr_domain.storagerouter = storagerouter
        sr_domain.save()
        self._run_and_validate_dtl_checkup(vdisk=vdisk,
                                           validations=[{'key': 'config', 'value': None}])
Ejemplo n.º 2
0
    def test_single_node(self):
        """
        Execute some DTL checkups on a single node installation
        """
        # Create 1 vdisk in single node without domains
        structure = Helper.build_service_structure(
            {'vpools': [1],
             'vdisks': [(1, 1, 1, 1)],  # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
             'mds_services': [(1, 1)],  # (<id>, <storagedriver_id>)
             'storagerouters': [1],
             'storagedrivers': [(1, 1, 1)]}  # (<id>, <vpool_id>, <sr_id>)
        )
        vpool = structure['vpools'][1]
        vdisk = structure['vdisks'][1]
        storagerouter = structure['storagerouters'][1]
        # || StorageRouter || vDisk | Regular Domain || Recovery Domain ||     DTL Target    ||
        #  |       1        |   1   |                 |                  |                    |
        self._roll_out_dtl_services(vpool=vpool, storagerouters=structure['storagerouters'])
        self._run_and_validate_dtl_checkup(vdisk=vdisk,
                                           validations=[{'key': 'config', 'value': None}])

        # Create some domains, but do not attach them yet
        # || StorageRouter || vDisk | Regular Domain || Recovery Domain ||     DTL Target    ||
        #  |       1        |   1   |                 |                  |                    |
        domains = {}
        for domain_id in range(1, 3):
            domain = Domain()
            domain.name = 'domain_{0}'.format(domain_id)
            domain.save()
            domains[domain_id] = domain

        self._run_and_validate_dtl_checkup(vdisk=vdisk,
                                           validations=[{'key': 'config', 'value': None}])

        # Attach a regular Domain to the single Storage Router
        # || StorageRouter || vDisk | Regular Domain || Recovery Domain ||     DTL Target    ||
        #  |      sr 1      |   1   |     domain 1    |                  |                    |
        sr_domain = StorageRouterDomain()
        sr_domain.backup = False
        sr_domain.domain = domains[1]
        sr_domain.storagerouter = storagerouter
        sr_domain.save()

        self._run_and_validate_dtl_checkup(vdisk=vdisk,
                                           validations=[{'key': 'config', 'value': None}])

        # Attach a recovery Domain to the single Storage Router
        # || StorageRouter || vDisk | Regular Domain || Recovery Domain ||     DTL Target    ||
        #  |      sr 1      |   1   |                 |     domain 1     |                    |
        for junction in storagerouter.domains:
            junction.delete()
        sr_domain = StorageRouterDomain()
        sr_domain.backup = True
        sr_domain.domain = domains[1]
        sr_domain.storagerouter = storagerouter
        sr_domain.save()
        self._run_and_validate_dtl_checkup(vdisk=vdisk,
                                           validations=[{'key': 'config', 'value': None}])
Ejemplo n.º 3
0
    def get_domain_by_guid(domain_guid):
        """
        Fetch disk partitions by disk guid

        :param domain_guid: guid of a domain
        :type domain_guid: str
        :return: domain object
        :rtype: ovs.dal.hybrids.domain.Domain
        """
        return Domain(domain_guid)
Ejemplo n.º 4
0
    def set_domains(self, storagerouter, domain_guids, recovery_domain_guids):
        """
        Configures the given domains to the StorageRouter.
        :param storagerouter: The StorageRouter to update
        :type storagerouter: ovs.dal.hybrids.storagerouter.StorageRouter
        :param domain_guids: A list of Domain guids
        :type domain_guids: list
        :param recovery_domain_guids: A list of Domain guids to set as recovery Domain
        :type recovery_domain_guids: list
        :return: None
        :rtype: None
        """
        change = False
        for junction in storagerouter.domains:
            if junction.backup is False:
                if junction.domain_guid not in domain_guids:
                    junction.delete()
                    change = True
                else:
                    domain_guids.remove(junction.domain_guid)
            else:
                if junction.domain_guid not in recovery_domain_guids:
                    junction.delete()
                    change = True
                else:
                    recovery_domain_guids.remove(junction.domain_guid)
        for domain_guid in domain_guids + recovery_domain_guids:
            junction = StorageRouterDomain()
            junction.domain = Domain(domain_guid)
            junction.backup = domain_guid in recovery_domain_guids
            junction.storagerouter = storagerouter
            junction.save()
            change = True

        # Schedule a task to run after 60 seconds, re-schedule task if another identical task gets triggered
        if change is True:
            cache = VolatileFactory.get_client()
            task_ids = cache.get(StorageRouterViewSet.DOMAIN_CHANGE_KEY)
            if task_ids:
                for task_id in task_ids:
                    revoke(task_id)
            task_ids = [
                MDSServiceController.mds_checkup.s().apply_async(
                    countdown=60).id,
                VDiskController.dtl_checkup.s().apply_async(countdown=60).id,
                StorageDriverController.cluster_registry_checkup.s().
                apply_async(countdown=60).id
            ]
            cache.set(StorageRouterViewSet.DOMAIN_CHANGE_KEY, task_ids,
                      600)  # Store the task ids
            storagerouter.invalidate_dynamics(
                ['regular_domains', 'recovery_domains'])
Ejemplo n.º 5
0
    def set_domains(self, storagerouter, domain_guids, recovery_domain_guids):
        """
        Configures the given domains to the StorageRouter.
        :param storagerouter: The StorageRouter to update
        :type storagerouter: StorageRouter
        :param domain_guids: A list of Domain guids
        :type domain_guids: list
        :param recovery_domain_guids: A list of Domain guids to set as recovery Domain
        :type recovery_domain_guids: list
        :return: None
        :rtype: None
        """
        change = False
        for junction in storagerouter.domains:
            if junction.backup is False:
                if junction.domain_guid not in domain_guids:
                    junction.delete()
                    change = True
                else:
                    domain_guids.remove(junction.domain_guid)
            else:
                if junction.domain_guid not in recovery_domain_guids:
                    junction.delete()
                    change = True
                else:
                    recovery_domain_guids.remove(junction.domain_guid)
        for domain_guid in domain_guids + recovery_domain_guids:
            junction = StorageRouterDomain()
            junction.domain = Domain(domain_guid)
            junction.backup = domain_guid in recovery_domain_guids
            junction.storagerouter = storagerouter
            junction.save()
            change = True

        # Schedule a task to run after 60 seconds, re-schedule task if another identical task gets triggered
        if change is True:
            cache = VolatileFactory.get_client()
            task_id_domain = cache.get(StorageRouterViewSet.DOMAIN_CHANGE_KEY)
            task_id_backup = cache.get(StorageRouterViewSet.RECOVERY_DOMAIN_CHANGE_KEY)
            if task_id_domain:
                revoke(task_id_domain)  # If key exists, task was already scheduled. If task is already running, the revoke message will be ignored
            if task_id_backup:
                revoke(task_id_backup)
            async_mds_result = MDSServiceController.mds_checkup.s().apply_async(countdown=60)
            async_dtl_result = VDiskController.dtl_checkup.s().apply_async(countdown=60)
            cache.set(StorageRouterViewSet.DOMAIN_CHANGE_KEY, async_mds_result.id, 600)  # Store the task id
            cache.set(StorageRouterViewSet.RECOVERY_DOMAIN_CHANGE_KEY, async_dtl_result.id, 600)  # Store the task id
            storagerouter.invalidate_dynamics(['regular_domains', 'recovery_domains'])
Ejemplo n.º 6
0
 def create(self, request, contents=None):
     """
     Creates a new Domain
     :param request: The raw request:
     :type request: Request
     :param contents: Requested contents (serializer hint)
     :type contents: str
     """
     contents = None if contents is None else contents.split(',')
     serializer = FullSerializer(Domain,
                                 contents=contents,
                                 instance=Domain(),
                                 data=request.DATA)
     domain = serializer.deserialize()
     domain.save()
     return domain
Ejemplo n.º 7
0
 def create(self, request, contents=None):
     """
     Creates a new Domain
     :param request: The raw request:
     :type request: Request
     :param contents: Requested contents (serializer hint)
     :type contents: str
     """
     contents = None if contents is None else contents.split(',')
     serializer = FullSerializer(Domain, contents=contents, instance=Domain(), data=request.DATA)
     domain = serializer.deserialize()
     current_domains = DomainList.get_by_name(domain.name)
     if len(current_domains) > 0:
         raise HttpNotAcceptableException(error_description='A Domain with the given name already exists',
                                          error='duplicate')
     domain.save()
     return domain
Ejemplo n.º 8
0
 def set_domains(self, backend, domain_guids):
     """
     Configures the given domains to the StorageRouter.
     :param backend: The Backend to update
     :type backend: Backend
     :param domain_guids: A list of Domain guids
     :type domain_guids: list
     :return: None
     :rtype: None
     """
     for junction in backend.domains:
         if junction.domain_guid not in domain_guids:
             junction.delete()
         else:
             domain_guids.remove(junction.domain_guid)
     for domain_guid in domain_guids:
         junction = BackendDomain()
         junction.domain = Domain(domain_guid)
         junction.backend = backend
         junction.save()
     backend.invalidate_dynamics(['regular_domains'])
Ejemplo n.º 9
0
    def build_dal_structure(structure, previous_structure=None):
        """
        Builds a model structure
        Example:
            structure = DalHelper.build_service_structure(
                {'vpools': [1],
                 'domains': [],
                 'storagerouters': [1],
                 'storagedrivers': [(1, 1, 1)],  # (<id>, <vpool_id>, <storagerouter_id>)
                 'mds_services': [(1, 1)],  # (<id>, <storagedriver_id>)
                 'storagerouter_domains': []}  # (<id>, <storagerouter_id>, <domain_id>)
            )
        """
        Configuration.set(key=Configuration.EDITION_KEY,
                          value=PackageFactory.EDITION_ENTERPRISE)

        if previous_structure is None:
            previous_structure = {}
        vdisks = previous_structure.get('vdisks', {})
        vpools = previous_structure.get('vpools', {})
        domains = previous_structure.get('domains', {})
        services = previous_structure.get('services', {})
        mds_services = previous_structure.get('mds_services', {})
        storagerouters = previous_structure.get('storagerouters', {})
        storagedrivers = previous_structure.get('storagedrivers', {})
        storagerouter_domains = previous_structure.get('storagerouter_domains',
                                                       {})

        service_types = {}
        for service_type_name in ServiceType.SERVICE_TYPES.values():
            service_type = ServiceTypeList.get_by_name(service_type_name)
            if service_type is None:
                service_type = ServiceType()
                service_type.name = service_type_name
                service_type.save()
            service_types[service_type_name] = service_type
        srclients = {}
        for domain_id in structure.get('domains', []):
            if domain_id not in domains:
                domain = Domain()
                domain.name = 'domain_{0}'.format(domain_id)
                domain.save()
                domains[domain_id] = domain
        for vpool_id in structure.get('vpools', []):
            if vpool_id not in vpools:
                vpool = VPool()
                vpool.name = str(vpool_id)
                vpool.status = 'RUNNING'
                vpool.metadata = {'backend': {}, 'caching_info': {}}
                vpool.metadata_store_bits = 5
                vpool.save()
                vpools[vpool_id] = vpool
            else:
                vpool = vpools[vpool_id]
            srclients[vpool_id] = StorageRouterClient(vpool.guid, None)
            Configuration.set(
                '/ovs/vpools/{0}/mds_config|mds_tlogs'.format(vpool.guid), 100)
            Configuration.set(
                '/ovs/vpools/{0}/mds_config|mds_safety'.format(vpool.guid), 2)
            Configuration.set(
                '/ovs/vpools/{0}/mds_config|mds_maxload'.format(vpool.guid),
                75)
            Configuration.set(
                '/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(
                    vpool.guid),
                json.dumps({}, indent=4),
                raw=True)
        for sr_id in structure.get('storagerouters', []):
            if sr_id not in storagerouters:
                storagerouter = StorageRouter()
                storagerouter.name = str(sr_id)
                storagerouter.ip = '10.0.0.{0}'.format(sr_id)
                storagerouter.rdma_capable = False
                storagerouter.node_type = 'MASTER'
                storagerouter.machine_id = str(sr_id)
                storagerouter.save()
                storagerouters[sr_id] = storagerouter
                disk = Disk()
                disk.storagerouter = storagerouter
                disk.state = 'OK'
                disk.name = '/dev/uda'
                disk.size = 1 * 1024**4
                disk.is_ssd = True
                disk.aliases = ['/dev/uda']
                disk.save()
                partition = DiskPartition()
                partition.offset = 0
                partition.size = disk.size
                partition.aliases = ['/dev/uda-1']
                partition.state = 'OK'
                partition.mountpoint = '/tmp/unittest/sr_{0}/disk_1/partition_1'.format(
                    sr_id)
                partition.disk = disk
                partition.roles = [
                    DiskPartition.ROLES.DB, DiskPartition.ROLES.SCRUB
                ]
                partition.save()
            else:
                storagerouter = storagerouters[sr_id]

            # noinspection PyProtectedMember
            System._machine_id[storagerouter.ip] = str(sr_id)
            mds_start = 10000 + 100 * (sr_id - 1)
            mds_end = 10000 + 100 * sr_id - 1
            arakoon_start = 20000 + 100 * (sr_id - 1)
            storagedriver_start = 30000 + 100 * (sr_id - 1)
            storagedriver_end = 30000 + 100 * sr_id - 1
            Configuration.initialize_host(
                host_id=sr_id,
                port_info={
                    'mds': [mds_start, mds_end],
                    'arakoon': arakoon_start,
                    'storagedriver': [storagedriver_start, storagedriver_end]
                })

        for sd_id, vpool_id, sr_id in structure.get('storagedrivers', ()):
            if sd_id not in storagedrivers:
                storagedriver = StorageDriver()
                storagedriver.vpool = vpools[vpool_id]
                storagedriver.storagerouter = storagerouters[sr_id]
                storagedriver.name = str(sd_id)
                storagedriver.mountpoint = '/'
                storagedriver.cluster_ip = storagerouters[sr_id].ip
                storagedriver.storage_ip = '10.0.1.{0}'.format(sr_id)
                storagedriver.storagedriver_id = str(sd_id)
                storagedriver.ports = {
                    'management': 1,
                    'xmlrpc': 2,
                    'dtl': 3,
                    'edge': 4
                }
                storagedriver.save()
                storagedrivers[sd_id] = storagedriver
                DalHelper.set_vpool_storage_driver_configuration(
                    vpool=vpools[vpool_id], storagedriver=storagedriver)
        for mds_id, sd_id in structure.get('mds_services', ()):
            if mds_id not in mds_services:
                sd = storagedrivers[sd_id]
                s_id = '{0}-{1}'.format(sd.storagerouter.name, mds_id)
                service = Service()
                service.name = s_id
                service.storagerouter = sd.storagerouter
                service.ports = [mds_id]
                service.type = service_types['MetadataServer']
                service.save()
                services[s_id] = service
                mds_service = MDSService()
                mds_service.service = service
                mds_service.number = 0
                mds_service.capacity = 10
                mds_service.vpool = sd.vpool
                mds_service.save()
                mds_services[mds_id] = mds_service
                StorageDriverController.add_storagedriverpartition(
                    sd, {
                        'size': None,
                        'role': DiskPartition.ROLES.DB,
                        'sub_role': StorageDriverPartition.SUBROLE.MDS,
                        'partition': sd.storagerouter.disks[0].partitions[0],
                        'mds_service': mds_service
                    })
        for vdisk_id, storage_driver_id, vpool_id, mds_id in structure.get(
                'vdisks', ()):
            if vdisk_id not in vdisks:
                vpool = vpools[vpool_id]
                devicename = 'vdisk_{0}'.format(vdisk_id)
                mds_backend_config = DalHelper.generate_mds_metadata_backend_config(
                    [] if mds_id is None else [mds_services[mds_id]])
                volume_id = srclients[vpool_id].create_volume(
                    devicename, mds_backend_config, 0, str(storage_driver_id))
                vdisk = VDisk()
                vdisk.name = str(vdisk_id)
                vdisk.devicename = devicename
                vdisk.volume_id = volume_id
                vdisk.vpool = vpool
                vdisk.size = 0
                vdisk.save()
                vdisk.reload_client('storagedriver')
                vdisks[vdisk_id] = vdisk
        for srd_id, sr_id, domain_id, backup in structure.get(
                'storagerouter_domains', ()):
            if srd_id not in storagerouter_domains:
                sr_domain = StorageRouterDomain()
                sr_domain.backup = backup
                sr_domain.domain = domains[domain_id]
                sr_domain.storagerouter = storagerouters[sr_id]
                sr_domain.save()
                storagerouter_domains[srd_id] = sr_domain
        return {
            'vdisks': vdisks,
            'vpools': vpools,
            'domains': domains,
            'services': services,
            'mds_services': mds_services,
            'service_types': service_types,
            'storagerouters': storagerouters,
            'storagedrivers': storagedrivers,
            'storagerouter_domains': storagerouter_domains
        }
Ejemplo n.º 10
0
    def build_service_structure(structure, previous_structure=None):
        """
        Builds an MDS service structure
        Example:
            structure = Helper.build_service_structure(
                {'vpools': [1],
                 'domains': [],
                 'storagerouters': [1],
                 'storagedrivers': [(1, 1, 1)],  # (<id>, <vpool_id>, <storagerouter_id>)
                 'mds_services': [(1, 1)],  # (<id>, <storagedriver_id>)
                 'storagerouter_domains': []}  # (<id>, <storagerouter_id>, <domain_id>)
            )
        """
        if previous_structure is None:
            previous_structure = {}
        vdisks = previous_structure.get('vdisks', {})
        vpools = previous_structure.get('vpools', {})
        domains = previous_structure.get('domains', {})
        services = previous_structure.get('services', {})
        mds_services = previous_structure.get('mds_services', {})
        storagerouters = previous_structure.get('storagerouters', {})
        storagedrivers = previous_structure.get('storagedrivers', {})
        storagerouter_domains = previous_structure.get('storagerouter_domains',
                                                       {})

        service_type = ServiceTypeList.get_by_name('MetadataServer')
        if service_type is None:
            service_type = ServiceType()
            service_type.name = 'MetadataServer'
            service_type.save()
        srclients = {}
        for domain_id in structure.get('domains', []):
            if domain_id not in domains:
                domain = Domain()
                domain.name = 'domain_{0}'.format(domain_id)
                domain.save()
                domains[domain_id] = domain
        for vpool_id in structure.get('vpools', []):
            if vpool_id not in vpools:
                vpool = VPool()
                vpool.name = str(vpool_id)
                vpool.status = 'RUNNING'
                vpool.save()
                vpools[vpool_id] = vpool
            else:
                vpool = vpools[vpool_id]
            srclients[vpool_id] = StorageRouterClient(vpool.guid, None)
        for sr_id in structure.get('storagerouters', []):
            if sr_id not in storagerouters:
                storagerouter = StorageRouter()
                storagerouter.name = str(sr_id)
                storagerouter.ip = '10.0.0.{0}'.format(sr_id)
                storagerouter.rdma_capable = False
                storagerouter.node_type = 'MASTER'
                storagerouter.machine_id = str(sr_id)
                storagerouter.save()
                storagerouters[sr_id] = storagerouter
                disk = Disk()
                disk.storagerouter = storagerouter
                disk.state = 'OK'
                disk.name = '/dev/uda'
                disk.size = 1 * 1024**4
                disk.is_ssd = True
                disk.aliases = ['/dev/uda']
                disk.save()
                partition = DiskPartition()
                partition.offset = 0
                partition.size = disk.size
                partition.aliases = ['/dev/uda-1']
                partition.state = 'OK'
                partition.mountpoint = '/tmp/unittest/sr_{0}/disk_1/partition_1'.format(
                    sr_id)
                partition.disk = disk
                partition.roles = [
                    DiskPartition.ROLES.DB, DiskPartition.ROLES.SCRUB
                ]
                partition.save()
        for sd_id, vpool_id, sr_id in structure.get('storagedrivers', ()):
            if sd_id not in storagedrivers:
                storagedriver = StorageDriver()
                storagedriver.vpool = vpools[vpool_id]
                storagedriver.storagerouter = storagerouters[sr_id]
                storagedriver.name = str(sd_id)
                storagedriver.mountpoint = '/'
                storagedriver.cluster_ip = storagerouters[sr_id].ip
                storagedriver.storage_ip = '10.0.1.{0}'.format(sr_id)
                storagedriver.storagedriver_id = str(sd_id)
                storagedriver.ports = {
                    'management': 1,
                    'xmlrpc': 2,
                    'dtl': 3,
                    'edge': 4
                }
                storagedriver.save()
                storagedrivers[sd_id] = storagedriver
                Helper._set_vpool_storage_driver_configuration(
                    vpool=vpools[vpool_id], storagedriver=storagedriver)
        for mds_id, sd_id in structure.get('mds_services', ()):
            if mds_id not in mds_services:
                sd = storagedrivers[sd_id]
                s_id = '{0}-{1}'.format(sd.storagerouter.name, mds_id)
                service = Service()
                service.name = s_id
                service.storagerouter = sd.storagerouter
                service.ports = [mds_id]
                service.type = service_type
                service.save()
                services[s_id] = service
                mds_service = MDSService()
                mds_service.service = service
                mds_service.number = 0
                mds_service.capacity = 10
                mds_service.vpool = sd.vpool
                mds_service.save()
                mds_services[mds_id] = mds_service
                StorageDriverController.add_storagedriverpartition(
                    sd, {
                        'size': None,
                        'role': DiskPartition.ROLES.DB,
                        'sub_role': StorageDriverPartition.SUBROLE.MDS,
                        'partition': sd.storagerouter.disks[0].partitions[0],
                        'mds_service': mds_service
                    })
        for vdisk_id, storage_driver_id, vpool_id, mds_id in structure.get(
                'vdisks', ()):
            if vdisk_id not in vdisks:
                vpool = vpools[vpool_id]
                devicename = 'vdisk_{0}'.format(vdisk_id)
                mds_backend_config = Helper._generate_mdsmetadatabackendconfig(
                    [] if mds_id is None else [mds_services[mds_id]])
                volume_id = srclients[vpool_id].create_volume(
                    devicename, mds_backend_config, 0, str(storage_driver_id))
                vdisk = VDisk()
                vdisk.name = str(vdisk_id)
                vdisk.devicename = devicename
                vdisk.volume_id = volume_id
                vdisk.vpool = vpool
                vdisk.size = 0
                vdisk.save()
                vdisk.reload_client('storagedriver')
                vdisks[vdisk_id] = vdisk
        for srd_id, sr_id, domain_id, backup in structure.get(
                'storagerouter_domains', ()):
            if srd_id not in storagerouter_domains:
                sr_domain = StorageRouterDomain()
                sr_domain.backup = backup
                sr_domain.domain = domains[domain_id]
                sr_domain.storagerouter = storagerouters[sr_id]
                sr_domain.save()
                storagerouter_domains[srd_id] = sr_domain
        return {
            'vdisks': vdisks,
            'vpools': vpools,
            'domains': domains,
            'services': services,
            'service_type': service_type,
            'mds_services': mds_services,
            'storagerouters': storagerouters,
            'storagedrivers': storagedrivers,
            'storagerouter_domains': storagerouter_domains
        }
Ejemplo n.º 11
0
    def build_service_structure(structure, previous_structure=None):
        """
        Builds an MDS service structure
        Example:
            structure = Helper.build_service_structure(
                {'vpools': [1],
                 'domains': [],
                 'storagerouters': [1],
                 'storagedrivers': [(1, 1, 1)],  # (<id>, <vpool_id>, <storagerouter_id>)
                 'mds_services': [(1, 1)],  # (<id>, <storagedriver_id>)
                 'storagerouter_domains': []}  # (<id>, <storagerouter_id>, <domain_id>)
            )
        """
        if previous_structure is None:
            previous_structure = {}
        vdisks = previous_structure.get("vdisks", {})
        vpools = previous_structure.get("vpools", {})
        domains = previous_structure.get("domains", {})
        services = previous_structure.get("services", {})
        mds_services = previous_structure.get("mds_services", {})
        storagerouters = previous_structure.get("storagerouters", {})
        storagedrivers = previous_structure.get("storagedrivers", {})
        storagerouter_domains = previous_structure.get("storagerouter_domains", {})

        service_type = ServiceTypeList.get_by_name("MetadataServer")
        if service_type is None:
            service_type = ServiceType()
            service_type.name = "MetadataServer"
            service_type.save()
        srclients = {}
        for domain_id in structure.get("domains", []):
            if domain_id not in domains:
                domain = Domain()
                domain.name = "domain_{0}".format(domain_id)
                domain.save()
                domains[domain_id] = domain
        for vpool_id in structure.get("vpools", []):
            if vpool_id not in vpools:
                vpool = VPool()
                vpool.name = str(vpool_id)
                vpool.status = "RUNNING"
                vpool.save()
                vpools[vpool_id] = vpool
            else:
                vpool = vpools[vpool_id]
            srclients[vpool_id] = StorageRouterClient(vpool.guid, None)
        for sr_id in structure.get("storagerouters", []):
            if sr_id not in storagerouters:
                storagerouter = StorageRouter()
                storagerouter.name = str(sr_id)
                storagerouter.ip = "10.0.0.{0}".format(sr_id)
                storagerouter.rdma_capable = False
                storagerouter.node_type = "MASTER"
                storagerouter.machine_id = str(sr_id)
                storagerouter.save()
                storagerouters[sr_id] = storagerouter
                disk = Disk()
                disk.storagerouter = storagerouter
                disk.state = "OK"
                disk.name = "/dev/uda"
                disk.size = 1 * 1024 ** 4
                disk.is_ssd = True
                disk.aliases = ["/dev/uda"]
                disk.save()
                partition = DiskPartition()
                partition.offset = 0
                partition.size = disk.size
                partition.aliases = ["/dev/uda-1"]
                partition.state = "OK"
                partition.mountpoint = "/tmp/unittest/sr_{0}/disk_1/partition_1".format(sr_id)
                partition.disk = disk
                partition.roles = [DiskPartition.ROLES.DB, DiskPartition.ROLES.SCRUB]
                partition.save()
        for sd_id, vpool_id, sr_id in structure.get("storagedrivers", ()):
            if sd_id not in storagedrivers:
                storagedriver = StorageDriver()
                storagedriver.vpool = vpools[vpool_id]
                storagedriver.storagerouter = storagerouters[sr_id]
                storagedriver.name = str(sd_id)
                storagedriver.mountpoint = "/"
                storagedriver.cluster_ip = storagerouters[sr_id].ip
                storagedriver.storage_ip = "10.0.1.{0}".format(sr_id)
                storagedriver.storagedriver_id = str(sd_id)
                storagedriver.ports = {"management": 1, "xmlrpc": 2, "dtl": 3, "edge": 4}
                storagedriver.save()
                storagedrivers[sd_id] = storagedriver
                Helper._set_vpool_storage_driver_configuration(vpool=vpools[vpool_id], storagedriver=storagedriver)
        for mds_id, sd_id in structure.get("mds_services", ()):
            if mds_id not in mds_services:
                sd = storagedrivers[sd_id]
                s_id = "{0}-{1}".format(sd.storagerouter.name, mds_id)
                service = Service()
                service.name = s_id
                service.storagerouter = sd.storagerouter
                service.ports = [mds_id]
                service.type = service_type
                service.save()
                services[s_id] = service
                mds_service = MDSService()
                mds_service.service = service
                mds_service.number = 0
                mds_service.capacity = 10
                mds_service.vpool = sd.vpool
                mds_service.save()
                mds_services[mds_id] = mds_service
                StorageDriverController.add_storagedriverpartition(
                    sd,
                    {
                        "size": None,
                        "role": DiskPartition.ROLES.DB,
                        "sub_role": StorageDriverPartition.SUBROLE.MDS,
                        "partition": sd.storagerouter.disks[0].partitions[0],
                        "mds_service": mds_service,
                    },
                )
        for vdisk_id, storage_driver_id, vpool_id, mds_id in structure.get("vdisks", ()):
            if vdisk_id not in vdisks:
                vpool = vpools[vpool_id]
                devicename = "vdisk_{0}".format(vdisk_id)
                mds_backend_config = Helper._generate_mdsmetadatabackendconfig(
                    [] if mds_id is None else [mds_services[mds_id]]
                )
                volume_id = srclients[vpool_id].create_volume(devicename, mds_backend_config, 0, str(storage_driver_id))
                vdisk = VDisk()
                vdisk.name = str(vdisk_id)
                vdisk.devicename = devicename
                vdisk.volume_id = volume_id
                vdisk.vpool = vpool
                vdisk.size = 0
                vdisk.save()
                vdisk.reload_client("storagedriver")
                vdisks[vdisk_id] = vdisk
        for srd_id, sr_id, domain_id, backup in structure.get("storagerouter_domains", ()):
            if srd_id not in storagerouter_domains:
                sr_domain = StorageRouterDomain()
                sr_domain.backup = backup
                sr_domain.domain = domains[domain_id]
                sr_domain.storagerouter = storagerouters[sr_id]
                sr_domain.save()
                storagerouter_domains[srd_id] = sr_domain
        return {
            "vdisks": vdisks,
            "vpools": vpools,
            "domains": domains,
            "services": services,
            "service_type": service_type,
            "mds_services": mds_services,
            "storagerouters": storagerouters,
            "storagedrivers": storagedrivers,
            "storagerouter_domains": storagerouter_domains,
        }