コード例 #1
0
ファイル: disk.py プロジェクト: yongshengma/framework
 def _model_disk(cls, generic_disk_model, storagerouter):
     # type: (GenericDisk, StorageRouter) -> Disk
     """
     Models a disk
     :param generic_disk_model: The generic modeled disk (returned by Disktools)
     :type generic_disk_model: GenericDisk
     :param storagerouter: Storagerouter to which this disk belongs
     :type storagerouter: StorageRouter
     :return: The newly modeled disk
     :rtype: Disk
     """
     DiskController._logger.info('Disk {0} - Creating disk - {1}'.format(generic_disk_model.name, generic_disk_model.__dict__))
     disk = Disk()
     disk.storagerouter = storagerouter
     disk.name = generic_disk_model.name
     DiskController._update_disk(disk, generic_disk_model)
     for partition in generic_disk_model.partitions:  # type: GenericPartition
         cls._model_partition(partition, disk)
     return disk
コード例 #2
0
    def test_happypath(self):
        """
        Validates the happy path; Hourly snapshots are taken with a few manual consistent
        every now an then. The delete policy is executed every day
        """
        # Setup
        # There are 2 machines; one with two disks, one with one disk and a stand-alone additional disk
        failure_domain = FailureDomain()
        failure_domain.name = 'Test'
        failure_domain.save()
        backend_type = BackendType()
        backend_type.name = 'BackendType'
        backend_type.code = 'BT'
        backend_type.save()
        vpool = VPool()
        vpool.name = 'vpool'
        vpool.status = 'RUNNING'
        vpool.backend_type = backend_type
        vpool.save()
        pmachine = PMachine()
        pmachine.name = 'PMachine'
        pmachine.username = '******'
        pmachine.ip = '127.0.0.1'
        pmachine.hvtype = 'VMWARE'
        pmachine.save()
        storage_router = StorageRouter()
        storage_router.name = 'storage_router'
        storage_router.ip = '127.0.0.1'
        storage_router.pmachine = pmachine
        storage_router.machine_id = System.get_my_machine_id()
        storage_router.rdma_capable = False
        storage_router.primary_failure_domain = failure_domain
        storage_router.save()
        disk = Disk()
        disk.name = 'physical_disk_1'
        disk.path = '/dev/non-existent'
        disk.size = 500 * 1024**3
        disk.state = 'OK'
        disk.is_ssd = True
        disk.storagerouter = storage_router
        disk.save()
        disk_partition = DiskPartition()
        disk_partition.id = 'disk_partition_id'
        disk_partition.disk = disk
        disk_partition.path = '/dev/disk/non-existent'
        disk_partition.size = 400 * 1024**3
        disk_partition.state = 'OK'
        disk_partition.offset = 1024
        disk_partition.roles = [DiskPartition.ROLES.SCRUB]
        disk_partition.mountpoint = '/var/tmp'
        disk_partition.save()
        vmachine_1 = VMachine()
        vmachine_1.name = 'vmachine_1'
        vmachine_1.devicename = 'dummy'
        vmachine_1.pmachine = pmachine
        vmachine_1.save()
        vdisk_1_1 = VDisk()
        vdisk_1_1.name = 'vdisk_1_1'
        vdisk_1_1.volume_id = 'vdisk_1_1'
        vdisk_1_1.vmachine = vmachine_1
        vdisk_1_1.vpool = vpool
        vdisk_1_1.devicename = 'dummy'
        vdisk_1_1.size = 0
        vdisk_1_1.save()
        vdisk_1_1.reload_client()
        vdisk_1_2 = VDisk()
        vdisk_1_2.name = 'vdisk_1_2'
        vdisk_1_2.volume_id = 'vdisk_1_2'
        vdisk_1_2.vmachine = vmachine_1
        vdisk_1_2.vpool = vpool
        vdisk_1_2.devicename = 'dummy'
        vdisk_1_2.size = 0
        vdisk_1_2.save()
        vdisk_1_2.reload_client()
        vmachine_2 = VMachine()
        vmachine_2.name = 'vmachine_2'
        vmachine_2.devicename = 'dummy'
        vmachine_2.pmachine = pmachine
        vmachine_2.save()
        vdisk_2_1 = VDisk()
        vdisk_2_1.name = 'vdisk_2_1'
        vdisk_2_1.volume_id = 'vdisk_2_1'
        vdisk_2_1.vmachine = vmachine_2
        vdisk_2_1.vpool = vpool
        vdisk_2_1.devicename = 'dummy'
        vdisk_2_1.size = 0
        vdisk_2_1.save()
        vdisk_2_1.reload_client()
        vdisk_3 = VDisk()
        vdisk_3.name = 'vdisk_3'
        vdisk_3.volume_id = 'vdisk_3'
        vdisk_3.vpool = vpool
        vdisk_3.devicename = 'dummy'
        vdisk_3.size = 0
        vdisk_3.save()
        vdisk_3.reload_client()

        for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
            [
                dynamic for dynamic in disk._dynamics
                if dynamic.name == 'snapshots'
            ][0].timeout = 0

        # Run the testing scenario
        travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
        if travis is True:
            print 'Running in Travis, reducing output.'
        debug = not travis
        amount_of_days = 50
        base = datetime.datetime.now().date()
        day = datetime.timedelta(1)
        minute = 60
        hour = minute * 60

        for d in xrange(0, amount_of_days):
            base_timestamp = self._make_timestamp(base, day * d)
            print ''
            print 'Day cycle: {0}: {1}'.format(
                d,
                datetime.datetime.fromtimestamp(base_timestamp).strftime(
                    '%Y-%m-%d'))

            # At the start of the day, delete snapshot policy runs at 00:30
            print '- Deleting snapshots'
            ScheduledTaskController.delete_snapshots(timestamp=base_timestamp +
                                                     (minute * 30))

            # Validate snapshots
            print '- Validating snapshots'
            for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
                self._validate(vdisk, d, base, amount_of_days, debug)

            # During the day, snapshots are taken
            # - Create non consistent snapshot every hour, between 2:00 and 22:00
            # - Create consistent snapshot at 6:30, 12:30, 18:30
            print '- Creating snapshots'
            for h in xrange(2, 23):
                timestamp = base_timestamp + (hour * h)
                for vm in [vmachine_1, vmachine_2]:
                    VMachineController.snapshot(machineguid=vm.guid,
                                                label='ss_i_{0}:00'.format(
                                                    str(h)),
                                                is_consistent=False,
                                                timestamp=timestamp)
                    if h in [6, 12, 18]:
                        ts = (timestamp + (minute * 30))
                        VMachineController.snapshot(machineguid=vm.guid,
                                                    label='ss_c_{0}:30'.format(
                                                        str(h)),
                                                    is_consistent=True,
                                                    timestamp=ts)

                VDiskController.create_snapshot(diskguid=vdisk_3.guid,
                                                metadata={
                                                    'label':
                                                    'ss_i_{0}:00'.format(
                                                        str(h)),
                                                    'is_consistent':
                                                    False,
                                                    'timestamp':
                                                    str(timestamp),
                                                    'machineguid':
                                                    None
                                                })
                if h in [6, 12, 18]:
                    ts = (timestamp + (minute * 30))
                    VDiskController.create_snapshot(diskguid=vdisk_3.guid,
                                                    metadata={
                                                        'label':
                                                        'ss_c_{0}:30'.format(
                                                            str(h)),
                                                        'is_consistent':
                                                        True,
                                                        'timestamp':
                                                        str(ts),
                                                        'machineguid':
                                                        None
                                                    })
コード例 #3
0
ファイル: disk.py プロジェクト: th3architect/framework
    def sync_with_reality(storagerouter_guid=None):
        """
        Syncs the Disks from all StorageRouters with the reality.
        :param storagerouter_guid: Guid of the Storage Router to synchronize
        """
        storagerouters = []
        if storagerouter_guid is not None:
            storagerouters.append(StorageRouter(storagerouter_guid))
        else:
            storagerouters = StorageRouterList.get_storagerouters()
        for storagerouter in storagerouters:
            try:
                client = SSHClient(storagerouter, username='******')
            except UnableToConnectException:
                DiskController._logger.info(
                    'Could not connect to StorageRouter {0}, skipping'.format(
                        storagerouter.ip))
                continue
            configuration = {}
            # Gather mount data
            mount_mapping = {}
            mount_data = client.run('mount')
            for mount in mount_data.splitlines():
                mount = mount.strip()
                match = re.search('(/dev/(.+?)) on (/.*?) type.*', mount)
                if match is not None:
                    dev_name = match.groups()[0]
                    uuid = client.run(
                        'blkid -o value -s UUID {0}'.format(dev_name))
                    if uuid:
                        mount_mapping[uuid] = match.groups()[2]
                    else:
                        mount_mapping[match.groups()[1]] = match.groups()[2]
            # Gather raid information
            try:
                md_information = client.run('mdadm --detail /dev/md*',
                                            suppress_logging=True)
            except CalledProcessError:
                md_information = ''
            raid_members = []
            for member in re.findall('(?: +[0-9]+){4} +[^/]+/dev/([a-z0-9]+)',
                                     md_information):
                raid_members.append(member)
            # Gather disk information
            with remote(storagerouter.ip, [Context, os]) as rem:
                context = rem.Context()
                devices = [
                    device
                    for device in context.list_devices(subsystem='block')
                    if ('ID_TYPE' in device and device['ID_TYPE'] == 'disk') or
                    ('DEVNAME' in device and
                     ('loop' in device['DEVNAME'] or 'nvme' in
                      device['DEVNAME'] or 'md' in device['DEVNAME']))
                ]
                for device in devices:
                    is_partition = device['DEVTYPE'] == 'partition'
                    device_path = device['DEVNAME']
                    device_name = device_path.split('/')[-1]
                    partition_id = None
                    partition_name = None
                    extended_partition_info = None
                    if is_partition is True:
                        partition_name = device[
                            'ID_FS_UUID'] if 'ID_FS_UUID' in device else device_name
                        if 'ID_PART_ENTRY_NUMBER' in device:
                            extended_partition_info = True
                            partition_id = device['ID_PART_ENTRY_NUMBER']
                            if device_name.startswith(
                                    'nvme') or device_name.startswith('loop'):
                                device_name = device_name[:0 -
                                                          int(len(partition_id)
                                                              ) - 1]
                            elif device_name.startswith('md'):
                                device_name = device_name[:device_name.
                                                          index('p')]
                            else:
                                device_name = device_name[:0 -
                                                          int(len(partition_id)
                                                              )]
                        else:
                            DiskController._logger.debug(
                                'Partition {0} has no partition metadata'.
                                format(device_path))
                            extended_partition_info = False
                            match = re.match('^(\D+?)(\d+)$', device_name)
                            if match is None:
                                DiskController._logger.debug(
                                    'Could not handle disk/partition {0}'.
                                    format(device_path))
                                continue  # Unable to handle this disk/partition
                            partition_id = match.groups()[1]
                            device_name = match.groups()[0]
                    sectors = int(
                        client.run(
                            'cat /sys/block/{0}/size'.format(device_name)))
                    sector_size = int(
                        client.run(
                            'cat /sys/block/{0}/queue/hw_sector_size'.format(
                                device_name)))
                    rotational = int(
                        client.run(
                            'cat /sys/block/{0}/queue/rotational'.format(
                                device_name)))

                    if sectors == 0:
                        continue
                    if device_name in raid_members:
                        continue
                    if device_name not in configuration:
                        configuration[device_name] = {'partitions': {}}
                    path = None
                    for path_type in ['by-id', 'by-uuid']:
                        if path is not None:
                            break
                        if 'DEVLINKS' in device:
                            for item in device['DEVLINKS'].split(' '):
                                if path_type in item:
                                    path = item
                    if path is None:
                        path = device_path
                    if is_partition is True:
                        if 'ID_PART_ENTRY_TYPE' in device and device[
                                'ID_PART_ENTRY_TYPE'] == '0x5':
                            continue  # This is an extended partition, let's skip that one
                        if extended_partition_info is True:
                            offset = int(
                                device['ID_PART_ENTRY_OFFSET']) * sector_size
                            size = int(
                                device['ID_PART_ENTRY_SIZE']) * sector_size
                        else:
                            match = re.match('^(\D+?)(\d+)$', device_path)
                            if match is None:
                                DiskController._logger.debug(
                                    'Could not handle disk/partition {0}'.
                                    format(device_path))
                                continue  # Unable to handle this disk/partition
                            partitions_info = DiskTools.get_partitions_info(
                                match.groups()[0])
                            if device_path in partitions_info:
                                partition_info = partitions_info[device_path]
                                offset = int(partition_info['start'])
                                size = int(partition_info['size'])
                            else:
                                DiskController._logger.warning(
                                    'Could not retrieve partition info for disk/partition {0}'
                                    .format(device_path))
                                continue
                        configuration[device_name]['partitions'][
                            partition_id] = {
                                'offset': offset,
                                'size': size,
                                'path': path,
                                'state': 'OK'
                            }
                        partition_data = configuration[device_name][
                            'partitions'][partition_id]
                        if partition_name in mount_mapping:
                            mountpoint = mount_mapping[partition_name]
                            partition_data['mountpoint'] = mountpoint
                            partition_data['inode'] = rem.os.stat(
                                mountpoint).st_dev
                            del mount_mapping[partition_name]
                            try:
                                client.run('touch {0}/{1}; rm {0}/{1}'.format(
                                    mountpoint, str(time.time())))
                            except CalledProcessError:
                                partition_data['state'] = 'FAILURE'
                                pass
                        if 'ID_FS_TYPE' in device:
                            partition_data['filesystem'] = device['ID_FS_TYPE']
                    else:
                        configuration[device_name].update({
                            'name':
                            device_name,
                            'path':
                            path,
                            'vendor':
                            device['ID_VENDOR']
                            if 'ID_VENDOR' in device else None,
                            'model':
                            device['ID_MODEL']
                            if 'ID_MODEL' in device else None,
                            'size':
                            sector_size * sectors,
                            'is_ssd':
                            rotational == 0,
                            'state':
                            'OK'
                        })
                    for partition_name in mount_mapping:
                        device_name = partition_name.split('/')[-1]
                        match = re.search('^(\D+?)(\d+)$', device_name)
                        if match is not None:
                            device_name = match.groups()[0]
                            partition_id = match.groups()[1]
                            if device_name not in configuration:
                                configuration[device_name] = {
                                    'partitions': {},
                                    'state': 'MISSING'
                                }
                            configuration[device_name]['partitions'][
                                partition_id] = {
                                    'mountpoint':
                                    mount_mapping[partition_name],
                                    'state': 'MISSING'
                                }
            # Sync the model
            disk_names = []
            for disk in storagerouter.disks:
                if disk.name not in configuration:
                    for partition in disk.partitions:
                        partition.delete()
                    disk.delete()
                else:
                    disk_names.append(disk.name)
                    DiskController._update_disk(disk, configuration[disk.name])
                    partitions = []
                    partition_info = configuration[disk.name]['partitions']
                    for partition in disk.partitions:
                        if partition.id not in partition_info:
                            partition.delete()
                        else:
                            partitions.append(partition.id)
                            DiskController._update_partition(
                                partition, partition_info[partition.id])
                    for partition_id in partition_info:
                        if partition_id not in partitions:
                            DiskController._create_partition(
                                partition_id, partition_info[partition_id],
                                disk)
            for disk_name in configuration:
                if disk_name not in disk_names and configuration[disk_name][
                        'state'] not in ['MISSING']:
                    disk = Disk()
                    disk.storagerouter = storagerouter
                    disk.name = disk_name
                    DiskController._update_disk(disk, configuration[disk_name])
                    partition_info = configuration[disk_name]['partitions']
                    for partition_id in partition_info:
                        if partition_info[partition_id]['state'] not in [
                                'MISSING'
                        ]:
                            DiskController._create_partition(
                                partition_id, partition_info[partition_id],
                                disk)
コード例 #4
0
ファイル: disk.py プロジェクト: openvstorage/framework
    def sync_with_reality(storagerouter_guid):
        """
        Syncs the Disks from all StorageRouters with the reality.

        CHANGES MADE TO THIS CODE SHOULD BE REFLECTED IN THE ASD-MANAGER list_disks CALL TOO!!!!!!!!!!!!!!!!!!!!

        :param storagerouter_guid: Guid of the Storage Router to synchronize
        :type storagerouter_guid: str
        :return: None
        """
        storagerouter = StorageRouter(storagerouter_guid)
        try:
            client = SSHClient(storagerouter, username='******')
        except UnableToConnectException:
            DiskController._logger.exception('Could not connect to StorageRouter {0}'.format(storagerouter.ip))
            raise

        # Retrieve all symlinks for all devices
        # Example of name_alias_mapping:
        # {'/dev/md0': ['/dev/disk/by-id/md-uuid-ad2de634:26d97253:5eda0a23:96986b76', '/dev/disk/by-id/md-name-OVS-1:0'],
        #  '/dev/sda': ['/dev/disk/by-path/pci-0000:03:00.0-sas-0x5000c295fe2ff771-lun-0'],
        #  '/dev/sda1': ['/dev/disk/by-uuid/e3e0bc62-4edc-4c6b-a6ce-1f39e8f27e41', '/dev/disk/by-path/pci-0000:03:00.0-sas-0x5000c295fe2ff771-lun-0-part1']}
        name_alias_mapping = {}
        alias_name_mapping = {}
        for path_type in client.dir_list(directory='/dev/disk'):
            if path_type in ['by-uuid', 'by-partuuid']:  # UUIDs can change after creating a filesystem on a partition
                continue
            directory = '/dev/disk/{0}'.format(path_type)
            for symlink in client.dir_list(directory=directory):
                symlink_path = '{0}/{1}'.format(directory, symlink)
                link = client.file_read_link(symlink_path)
                if link not in name_alias_mapping:
                    name_alias_mapping[link] = []
                name_alias_mapping[link].append(symlink_path)
                alias_name_mapping[symlink_path] = link

        # Parse 'lsblk' output
        # --exclude 1 for RAM devices, 2 for floppy devices, 11 for CD-ROM devices (See https://www.kernel.org/doc/Documentation/devices.txt)
        command = ['lsblk', '--pairs', '--bytes', '--noheadings', '--exclude', '1,2,11']
        output = '--output=KNAME,SIZE,MODEL,STATE,MAJ:MIN,FSTYPE,TYPE,ROTA,MOUNTPOINT,LOG-SEC{0}'
        regex = '^KNAME="(?P<name>.*)" SIZE="(?P<size>\d*)" MODEL="(?P<model>.*)" STATE="(?P<state>.*)" MAJ:MIN="(?P<dev_nr>.*)" FSTYPE="(?P<fstype>.*)" TYPE="(?P<type>.*)" ROTA="(?P<rota>[0,1])" MOUNTPOINT="(?P<mtpt>.*)" LOG-SEC="(?P<sector_size>\d*)"( SERIAL="(?P<serial>.*)")?$'
        try:
            devices = client.run(command + [output.format(',SERIAL')]).splitlines()
        except:
            devices = client.run(command + [output.format('')]).splitlines()
        device_regex = re.compile(regex)
        configuration = {}
        parsed_devices = []
        for device in devices:
            match = re.match(device_regex, device)
            if match is None:
                DiskController._logger.error('Device regex did not match for {0}. Please investigate'.format(device))
                raise Exception('Failed to parse \'lsblk\' output')

            groupdict = match.groupdict()
            name = groupdict['name'].strip()
            size = groupdict['size'].strip()
            model = groupdict['model'].strip()
            state = groupdict['state'].strip()
            dev_nr = groupdict['dev_nr'].strip()
            serial = (groupdict['serial'] or '').strip()
            fs_type = groupdict['fstype'].strip()
            dev_type = groupdict['type'].strip()
            rotational = groupdict['rota'].strip()
            mount_point = groupdict['mtpt'].strip()
            sector_size = groupdict['sector_size'].strip()

            if dev_type == 'rom':
                continue

            link = client.file_read_link('/sys/block/{0}'.format(name))
            device_state = None
            friendly_path = '/dev/{0}'.format(name)
            system_aliases = name_alias_mapping.get(friendly_path, [friendly_path])
            device_is_also_partition = False
            if link is not None:  # If this returns, it means its a device and not a partition
                device_is_also_partition = mount_point != ''  # LVM, RAID1, ... have the tendency to be a device with a partition on it, but the partition is not reported by 'lsblk'
                device_state = Disk.STATES.OK if state == 'running' or dev_nr.split(':')[0] != '8' else Disk.STATES.FAILURE
                parsed_devices.append({'name': name,
                                       'state': device_state})
                configuration[name] = {'name': name,
                                       'size': int(size),
                                       'model': model if model != '' else None,
                                       'serial': serial if serial != '' else None,
                                       'state': device_state,
                                       'is_ssd': rotational == '0',
                                       'aliases': system_aliases,
                                       'partitions': {}}
            if link is None or device_is_also_partition is True:
                current_device = None
                current_device_state = None
                if device_is_also_partition is True:
                    offset = 0
                    current_device = name
                    current_device_state = device_state
                else:
                    offset = 0
                    for device_info in reversed(parsed_devices):
                        try:
                            current_device = device_info['name']
                            current_device_state = device_info['state']
                            offset = int(client.file_read('/sys/block/{0}/{1}/start'.format(current_device, name))) * int(sector_size)
                            break
                        except Exception:
                            pass
                if current_device is None:
                    raise RuntimeError('Failed to retrieve the device information for current partition')
                mount_point = mount_point if mount_point != '' else None
                partition_state = Disk.STATES.OK if current_device_state == Disk.STATES.OK else Disk.STATES.FAILURE
                if mount_point is not None and fs_type != 'swap':
                    try:
                        filename = '{0}/{1}'.format(mount_point, str(time.time()))
                        client.run(['touch', filename])
                        client.run(['rm', filename])
                    except Exception:
                        partition_state = Disk.STATES.FAILURE

                configuration[current_device]['partitions'][offset] = {'size': int(size),
                                                                       'state': partition_state,
                                                                       'offset': offset,
                                                                       'aliases': system_aliases,
                                                                       'filesystem': fs_type if fs_type != '' else None,
                                                                       'mountpoint': mount_point}

        # Sync the model
        for disk in storagerouter.disks:
            disk_info = None
            for alias in disk.aliases:
                if alias in alias_name_mapping:
                    name = alias_name_mapping[alias].replace('/dev/', '')
                    if name in configuration:
                        disk_info = configuration.pop(name)
                        break

            if disk_info is None and disk.name in configuration and (disk.name.startswith('fio') or
                                                                     disk.name.startswith('loop') or
                                                                     disk.name.startswith('nvme')):  # Partitioned loop, nvme devices no longer show up in alias_name_mapping
                disk_info = configuration.pop(disk.name)

            # Remove disk / partitions if not reported by 'lsblk'
            if disk_info is None:
                DiskController._logger.info('Disk {0} - No longer found'.format(disk.name))
                delete = True
                for partition in disk.partitions:
                    if len(partition.roles) > 0:
                        delete = False
                        DiskController._logger.warning('Disk {0} - Partition with offset {1} - Has roles, will not delete'.format(disk.name, partition.offset))
                        break
                if delete is True:
                    for partition in disk.partitions:
                        partition.delete()
                    disk.delete()
                    DiskController._logger.info('Disk {0} - Deleted'.format(disk.name))
                else:
                    for partition in disk.partitions:
                        DiskController._update_partition(partition, {'state': 'MISSING'})
                        DiskController._logger.warning('Disk {0} - Partition with offset {1} - Updated status to MISSING'.format(disk.name, partition.offset))
                    DiskController._update_disk(disk, {'state': 'MISSING'})
                    DiskController._logger.warning('Disk {0} - Updated status to MISSING'.format(disk.name))

            else:  # Update existing disks and their partitions
                DiskController._logger.info('Disk {0} - Found, updating'.format(disk.name))
                DiskController._update_disk(disk, disk_info)
                partition_info = disk_info['partitions']
                for partition in disk.partitions:
                    if partition.offset not in partition_info:
                        DiskController._logger.info('Disk {0} - Partition with offset {1} - No longer found'.format(disk.name, partition.offset))
                        if len(partition.roles) > 0:
                            DiskController._logger.warning('Disk {0} - Partition with offset {1} - Update status to MISSING'.format(disk.name, partition.offset))
                            DiskController._update_partition(partition, {'state': 'MISSING'})
                        else:
                            DiskController._logger.info('Disk {0} - Partition with offset {1} - Deleting'.format(disk.name, partition.offset))
                            partition.delete()
                    else:
                        DiskController._update_partition(partition, partition_info.pop(partition.offset))
                for partition_offset in partition_info:
                    DiskController._logger.info('Disk {0} - Creating partition - {1}'.format(disk.name, partition_info[partition_offset]))
                    DiskController._create_partition(partition_info[partition_offset], disk)
        # Create all disks and their partitions not yet modeled
        for disk_name in configuration:
            DiskController._logger.info('Disk {0} - Creating disk - {1}'.format(disk_name, configuration[disk_name]))
            disk = Disk()
            disk.storagerouter = storagerouter
            disk.name = disk_name
            DiskController._update_disk(disk, configuration[disk_name])
            partition_info = configuration[disk_name]['partitions']
            for partition_offset in partition_info:
                DiskController._create_partition(partition_info[partition_offset], disk)
コード例 #5
0
ファイル: disk.py プロジェクト: winglq/framework
    def sync_with_reality(storagerouter_guid):
        """
        Syncs the Disks from all StorageRouters with the reality.

        CHANGES MADE TO THIS CODE SHOULD BE REFLECTED IN THE ASD-MANAGER list_disks CALL TOO!!!!!!!!!!!!!!!!!!!!

        :param storagerouter_guid: Guid of the Storage Router to synchronize
        :type storagerouter_guid: str
        :return: None
        """
        storagerouter = StorageRouter(storagerouter_guid)
        try:
            client = SSHClient(storagerouter, username='******')
        except UnableToConnectException:
            DiskController._logger.exception(
                'Could not connect to StorageRouter {0}'.format(
                    storagerouter.ip))
            raise

        # Retrieve all symlinks for all devices
        # Example of name_alias_mapping:
        # {'/dev/md0': ['/dev/disk/by-id/md-uuid-ad2de634:26d97253:5eda0a23:96986b76', '/dev/disk/by-id/md-name-OVS-1:0'],
        #  '/dev/sda': ['/dev/disk/by-path/pci-0000:03:00.0-sas-0x5000c295fe2ff771-lun-0'],
        #  '/dev/sda1': ['/dev/disk/by-uuid/e3e0bc62-4edc-4c6b-a6ce-1f39e8f27e41', '/dev/disk/by-path/pci-0000:03:00.0-sas-0x5000c295fe2ff771-lun-0-part1']}
        name_alias_mapping = {}
        alias_name_mapping = {}
        for path_type in client.dir_list(directory='/dev/disk'):
            if path_type in [
                    'by-uuid', 'by-partuuid'
            ]:  # UUIDs can change after creating a filesystem on a partition
                continue
            directory = '/dev/disk/{0}'.format(path_type)
            for symlink in client.dir_list(directory=directory):
                symlink_path = '{0}/{1}'.format(directory, symlink)
                link = client.file_read_link(symlink_path)
                if link not in name_alias_mapping:
                    name_alias_mapping[link] = []
                name_alias_mapping[link].append(symlink_path)
                alias_name_mapping[symlink_path] = link

        # Parse 'lsblk' output
        # --exclude 1 for RAM devices, 2 for floppy devices, 11 for CD-ROM devices (See https://www.kernel.org/doc/Documentation/devices.txt)
        devices = client.run([
            'lsblk', '--pairs', '--bytes', '--noheadings', '--exclude',
            '1,2,11',
            '--output=KNAME,SIZE,MODEL,STATE,MAJ:MIN,FSTYPE,TYPE,ROTA,MOUNTPOINT,LOG-SEC'
        ]).splitlines()
        device_regex = re.compile(
            '^KNAME="(?P<name>.*)" SIZE="(?P<size>\d*)" MODEL="(?P<model>.*)" STATE="(?P<state>.*)" MAJ:MIN="(?P<dev_nr>.*)" FSTYPE="(?P<fstype>.*)" TYPE="(?P<type>.*)" ROTA="(?P<rota>[0,1])" MOUNTPOINT="(?P<mtpt>.*)" LOG-SEC="(?P<sector_size>\d*)"$'
        )
        configuration = {}
        parsed_devices = []
        for device in devices:
            match = re.match(device_regex, device)
            if match is None:
                DiskController._logger.error(
                    'Device regex did not match for {0}. Please investigate'.
                    format(device))
                raise Exception('Failed to parse \'lsblk\' output')

            groupdict = match.groupdict()
            name = groupdict['name'].strip()
            size = groupdict['size'].strip()
            model = groupdict['model'].strip()
            state = groupdict['state'].strip()
            dev_nr = groupdict['dev_nr'].strip()
            fs_type = groupdict['fstype'].strip()
            dev_type = groupdict['type'].strip()
            rotational = groupdict['rota'].strip()
            mount_point = groupdict['mtpt'].strip()
            sector_size = groupdict['sector_size'].strip()

            if dev_type == 'rom':
                continue

            link = client.file_read_link('/sys/block/{0}'.format(name))
            device_state = None
            friendly_path = '/dev/{0}'.format(name)
            system_aliases = name_alias_mapping.get(friendly_path,
                                                    [friendly_path])
            device_is_also_partition = False
            if link is not None:  # If this returns, it means its a device and not a partition
                device_is_also_partition = mount_point != ''  # LVM, RAID1, ... have the tendency to be a device with a partition on it, but the partition is not reported by 'lsblk'
                device_state = Disk.STATES.OK if state == 'running' or dev_nr.split(
                    ':')[0] != '8' else Disk.STATES.FAILURE
                parsed_devices.append({'name': name, 'state': device_state})
                configuration[name] = {
                    'name': name,
                    'size': int(size),
                    'model': model if model != '' else None,
                    'state': device_state,
                    'is_ssd': rotational == '0',
                    'aliases': system_aliases,
                    'partitions': {}
                }
            if link is None or device_is_also_partition is True:
                current_device = None
                current_device_state = None
                if device_is_also_partition is True:
                    offset = 0
                    current_device = name
                    current_device_state = device_state
                else:
                    offset = 0
                    for device_info in reversed(parsed_devices):
                        try:
                            current_device = device_info['name']
                            current_device_state = device_info['state']
                            offset = int(
                                client.file_read(
                                    '/sys/block/{0}/{1}/start'.format(
                                        current_device,
                                        name))) * int(sector_size)
                            break
                        except Exception:
                            pass
                if current_device is None:
                    raise RuntimeError(
                        'Failed to retrieve the device information for current partition'
                    )
                mount_point = mount_point if mount_point != '' else None
                partition_state = Disk.STATES.OK if current_device_state == Disk.STATES.OK else Disk.STATES.FAILURE
                if mount_point is not None and fs_type != 'swap':
                    try:
                        filename = '{0}/{1}'.format(mount_point,
                                                    str(time.time()))
                        client.run(['touch', filename])
                        client.run(['rm', filename])
                    except Exception:
                        partition_state = Disk.STATES.FAILURE

                configuration[current_device]['partitions'][offset] = {
                    'size': int(size),
                    'state': partition_state,
                    'offset': offset,
                    'aliases': system_aliases,
                    'filesystem': fs_type if fs_type != '' else None,
                    'mountpoint': mount_point
                }

        # Sync the model
        for disk in storagerouter.disks:
            disk_info = None
            for alias in disk.aliases:
                if alias in alias_name_mapping:
                    name = alias_name_mapping[alias].replace('/dev/', '')
                    if name in configuration:
                        disk_info = configuration.pop(name)
                        break

            if disk_info is None and disk.name in configuration and (
                    disk.name.startswith('fio') or disk.name.startswith('loop')
                    or disk.name.startswith('nvme')
            ):  # Partitioned loop, nvme devices no longer show up in alias_name_mapping
                disk_info = configuration.pop(disk.name)

            # Remove disk / partitions if not reported by 'lsblk'
            if disk_info is None:
                DiskController._logger.info(
                    'Disk {0} - No longer found'.format(disk.name))
                delete = True
                for partition in disk.partitions:
                    if len(partition.roles) > 0:
                        delete = False
                        DiskController._logger.warning(
                            'Disk {0} - Partition with offset {1} - Has roles, will not delete'
                            .format(disk.name, partition.offset))
                        break
                if delete is True:
                    for partition in disk.partitions:
                        partition.delete()
                    disk.delete()
                    DiskController._logger.info('Disk {0} - Deleted'.format(
                        disk.name))
                else:
                    for partition in disk.partitions:
                        DiskController._update_partition(
                            partition, {'state': 'MISSING'})
                        DiskController._logger.warning(
                            'Disk {0} - Partition with offset {1} - Updated status to MISSING'
                            .format(disk.name, partition.offset))
                    DiskController._update_disk(disk, {'state': 'MISSING'})
                    DiskController._logger.warning(
                        'Disk {0} - Updated status to MISSING'.format(
                            disk.name))

            else:  # Update existing disks and their partitions
                DiskController._logger.info(
                    'Disk {0} - Found, updating'.format(disk.name))
                DiskController._update_disk(disk, disk_info)
                partition_info = disk_info['partitions']
                for partition in disk.partitions:
                    if partition.offset not in partition_info:
                        DiskController._logger.info(
                            'Disk {0} - Partition with offset {1} - No longer found'
                            .format(disk.name, partition.offset))
                        if len(partition.roles) > 0:
                            DiskController._logger.warning(
                                'Disk {0} - Partition with offset {1} - Update status to MISSING'
                                .format(disk.name, partition.offset))
                            DiskController._update_partition(
                                partition, {'state': 'MISSING'})
                        else:
                            DiskController._logger.info(
                                'Disk {0} - Partition with offset {1} - Deleting'
                                .format(disk.name, partition.offset))
                            partition.delete()
                    else:
                        DiskController._update_partition(
                            partition, partition_info.pop(partition.offset))
                for partition_offset in partition_info:
                    DiskController._logger.info(
                        'Disk {0} - Creating partition - {1}'.format(
                            disk.name, partition_info[partition_offset]))
                    DiskController._create_partition(
                        partition_info[partition_offset], disk)
        # Create all disks and their partitions not yet modeled
        for disk_name in configuration:
            DiskController._logger.info(
                'Disk {0} - Creating disk - {1}'.format(
                    disk_name, configuration[disk_name]))
            disk = Disk()
            disk.storagerouter = storagerouter
            disk.name = disk_name
            DiskController._update_disk(disk, configuration[disk_name])
            partition_info = configuration[disk_name]['partitions']
            for partition_offset in partition_info:
                DiskController._create_partition(
                    partition_info[partition_offset], disk)
コード例 #6
0
    def test_happypath(self):
        """
        Validates the happy path; Hourly snapshots are taken with a few manual consistent
        every now an then. The delete policy is executed every day
        """
        # Setup
        # There are 2 machines; one with two disks, one with one disk and an additional disk
        failure_domain = FailureDomain()
        failure_domain.name = "Test"
        failure_domain.save()
        backend_type = BackendType()
        backend_type.name = "BackendType"
        backend_type.code = "BT"
        backend_type.save()
        vpool = VPool()
        vpool.name = "vpool"
        vpool.backend_type = backend_type
        vpool.save()
        pmachine = PMachine()
        pmachine.name = "PMachine"
        pmachine.username = "******"
        pmachine.ip = "127.0.0.1"
        pmachine.hvtype = "VMWARE"
        pmachine.save()
        storage_router = StorageRouter()
        storage_router.name = "storage_router"
        storage_router.ip = "127.0.0.1"
        storage_router.pmachine = pmachine
        storage_router.machine_id = System.get_my_machine_id()
        storage_router.rdma_capable = False
        storage_router.primary_failure_domain = failure_domain
        storage_router.save()
        disk = Disk()
        disk.name = "physical_disk_1"
        disk.path = "/dev/non-existent"
        disk.size = 500 * 1024 ** 3
        disk.state = "OK"
        disk.is_ssd = True
        disk.storagerouter = storage_router
        disk.save()
        disk_partition = DiskPartition()
        disk_partition.id = "disk_partition_id"
        disk_partition.disk = disk
        disk_partition.path = "/dev/disk/non-existent"
        disk_partition.size = 400 * 1024 ** 3
        disk_partition.state = "OK"
        disk_partition.offset = 1024
        disk_partition.roles = [DiskPartition.ROLES.SCRUB]
        disk_partition.mountpoint = "/var/tmp"
        disk_partition.save()
        vmachine_1 = VMachine()
        vmachine_1.name = "vmachine_1"
        vmachine_1.devicename = "dummy"
        vmachine_1.pmachine = pmachine
        vmachine_1.save()
        vdisk_1_1 = VDisk()
        vdisk_1_1.name = "vdisk_1_1"
        vdisk_1_1.volume_id = "vdisk_1_1"
        vdisk_1_1.vmachine = vmachine_1
        vdisk_1_1.vpool = vpool
        vdisk_1_1.devicename = "dummy"
        vdisk_1_1.size = 0
        vdisk_1_1.save()
        vdisk_1_1.reload_client()
        vdisk_1_2 = VDisk()
        vdisk_1_2.name = "vdisk_1_2"
        vdisk_1_2.volume_id = "vdisk_1_2"
        vdisk_1_2.vmachine = vmachine_1
        vdisk_1_2.vpool = vpool
        vdisk_1_2.devicename = "dummy"
        vdisk_1_2.size = 0
        vdisk_1_2.save()
        vdisk_1_2.reload_client()
        vmachine_2 = VMachine()
        vmachine_2.name = "vmachine_2"
        vmachine_2.devicename = "dummy"
        vmachine_2.pmachine = pmachine
        vmachine_2.save()
        vdisk_2_1 = VDisk()
        vdisk_2_1.name = "vdisk_2_1"
        vdisk_2_1.volume_id = "vdisk_2_1"
        vdisk_2_1.vmachine = vmachine_2
        vdisk_2_1.vpool = vpool
        vdisk_2_1.devicename = "dummy"
        vdisk_2_1.size = 0
        vdisk_2_1.save()
        vdisk_2_1.reload_client()
        vdisk_3 = VDisk()
        vdisk_3.name = "vdisk_3"
        vdisk_3.volume_id = "vdisk_3"
        vdisk_3.vpool = vpool
        vdisk_3.devicename = "dummy"
        vdisk_3.size = 0
        vdisk_3.save()
        vdisk_3.reload_client()

        for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
            [dynamic for dynamic in disk._dynamics if dynamic.name == "snapshots"][0].timeout = 0

        # Run the testing scenario
        debug = True
        amount_of_days = 50
        base = datetime.now().date()
        day = timedelta(1)
        minute = 60
        hour = minute * 60

        for d in xrange(0, amount_of_days):
            base_timestamp = DeleteSnapshots._make_timestamp(base, day * d)
            print ""
            print "Day cycle: {0}: {1}".format(d, datetime.fromtimestamp(base_timestamp).strftime("%Y-%m-%d"))

            # At the start of the day, delete snapshot policy runs at 00:30
            print "- Deleting snapshots"
            ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))

            # Validate snapshots
            print "- Validating snapshots"
            for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
                self._validate(vdisk, d, base, amount_of_days, debug)

            # During the day, snapshots are taken
            # - Create non consistent snapshot every hour, between 2:00 and 22:00
            # - Create consistent snapshot at 6:30, 12:30, 18:30
            print "- Creating snapshots"
            for h in xrange(2, 23):
                timestamp = base_timestamp + (hour * h)
                for vm in [vmachine_1, vmachine_2]:
                    VMachineController.snapshot(
                        machineguid=vm.guid,
                        label="ss_i_{0}:00".format(str(h)),
                        is_consistent=False,
                        timestamp=timestamp,
                    )
                    if h in [6, 12, 18]:
                        ts = timestamp + (minute * 30)
                        VMachineController.snapshot(
                            machineguid=vm.guid, label="ss_c_{0}:30".format(str(h)), is_consistent=True, timestamp=ts
                        )

                VDiskController.create_snapshot(
                    diskguid=vdisk_3.guid,
                    metadata={
                        "label": "ss_i_{0}:00".format(str(h)),
                        "is_consistent": False,
                        "timestamp": str(timestamp),
                        "machineguid": None,
                    },
                )
                if h in [6, 12, 18]:
                    ts = timestamp + (minute * 30)
                    VDiskController.create_snapshot(
                        diskguid=vdisk_3.guid,
                        metadata={
                            "label": "ss_c_{0}:30".format(str(h)),
                            "is_consistent": True,
                            "timestamp": str(ts),
                            "machineguid": None,
                        },
                    )
コード例 #7
0
    def test_happypath(self):
        """
        Validates the happy path; Hourly snapshots are taken with a few manual consistent
        every now an then. The delete policy is executed every day
        """
        # Setup
        # There are 2 machines; one with two disks, one with one disk and a stand-alone additional disk
        failure_domain = FailureDomain()
        failure_domain.name = 'Test'
        failure_domain.save()
        backend_type = BackendType()
        backend_type.name = 'BackendType'
        backend_type.code = 'BT'
        backend_type.save()
        vpool = VPool()
        vpool.name = 'vpool'
        vpool.status = 'RUNNING'
        vpool.backend_type = backend_type
        vpool.save()
        pmachine = PMachine()
        pmachine.name = 'PMachine'
        pmachine.username = '******'
        pmachine.ip = '127.0.0.1'
        pmachine.hvtype = 'VMWARE'
        pmachine.save()
        storage_router = StorageRouter()
        storage_router.name = 'storage_router'
        storage_router.ip = '127.0.0.1'
        storage_router.pmachine = pmachine
        storage_router.machine_id = System.get_my_machine_id()
        storage_router.rdma_capable = False
        storage_router.primary_failure_domain = failure_domain
        storage_router.save()
        disk = Disk()
        disk.name = 'physical_disk_1'
        disk.path = '/dev/non-existent'
        disk.size = 500 * 1024 ** 3
        disk.state = 'OK'
        disk.is_ssd = True
        disk.storagerouter = storage_router
        disk.save()
        disk_partition = DiskPartition()
        disk_partition.id = 'disk_partition_id'
        disk_partition.disk = disk
        disk_partition.path = '/dev/disk/non-existent'
        disk_partition.size = 400 * 1024 ** 3
        disk_partition.state = 'OK'
        disk_partition.offset = 1024
        disk_partition.roles = [DiskPartition.ROLES.SCRUB]
        disk_partition.mountpoint = '/var/tmp'
        disk_partition.save()
        vmachine_1 = VMachine()
        vmachine_1.name = 'vmachine_1'
        vmachine_1.devicename = 'dummy'
        vmachine_1.pmachine = pmachine
        vmachine_1.save()
        vdisk_1_1 = VDisk()
        vdisk_1_1.name = 'vdisk_1_1'
        vdisk_1_1.volume_id = 'vdisk_1_1'
        vdisk_1_1.vmachine = vmachine_1
        vdisk_1_1.vpool = vpool
        vdisk_1_1.devicename = 'dummy'
        vdisk_1_1.size = 0
        vdisk_1_1.save()
        vdisk_1_1.reload_client()
        vdisk_1_2 = VDisk()
        vdisk_1_2.name = 'vdisk_1_2'
        vdisk_1_2.volume_id = 'vdisk_1_2'
        vdisk_1_2.vmachine = vmachine_1
        vdisk_1_2.vpool = vpool
        vdisk_1_2.devicename = 'dummy'
        vdisk_1_2.size = 0
        vdisk_1_2.save()
        vdisk_1_2.reload_client()
        vmachine_2 = VMachine()
        vmachine_2.name = 'vmachine_2'
        vmachine_2.devicename = 'dummy'
        vmachine_2.pmachine = pmachine
        vmachine_2.save()
        vdisk_2_1 = VDisk()
        vdisk_2_1.name = 'vdisk_2_1'
        vdisk_2_1.volume_id = 'vdisk_2_1'
        vdisk_2_1.vmachine = vmachine_2
        vdisk_2_1.vpool = vpool
        vdisk_2_1.devicename = 'dummy'
        vdisk_2_1.size = 0
        vdisk_2_1.save()
        vdisk_2_1.reload_client()
        vdisk_3 = VDisk()
        vdisk_3.name = 'vdisk_3'
        vdisk_3.volume_id = 'vdisk_3'
        vdisk_3.vpool = vpool
        vdisk_3.devicename = 'dummy'
        vdisk_3.size = 0
        vdisk_3.save()
        vdisk_3.reload_client()

        for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
            [dynamic for dynamic in disk._dynamics if dynamic.name == 'snapshots'][0].timeout = 0

        # Run the testing scenario
        travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
        if travis is True:
            print 'Running in Travis, reducing output.'
        debug = not travis
        amount_of_days = 50
        base = datetime.datetime.now().date()
        day = datetime.timedelta(1)
        minute = 60
        hour = minute * 60

        for d in xrange(0, amount_of_days):
            base_timestamp = self._make_timestamp(base, day * d)
            print ''
            print 'Day cycle: {0}: {1}'.format(d, datetime.datetime.fromtimestamp(base_timestamp).strftime('%Y-%m-%d'))

            # At the start of the day, delete snapshot policy runs at 00:30
            print '- Deleting snapshots'
            ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))

            # Validate snapshots
            print '- Validating snapshots'
            for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
                self._validate(vdisk, d, base, amount_of_days, debug)

            # During the day, snapshots are taken
            # - Create non consistent snapshot every hour, between 2:00 and 22:00
            # - Create consistent snapshot at 6:30, 12:30, 18:30
            print '- Creating snapshots'
            for h in xrange(2, 23):
                timestamp = base_timestamp + (hour * h)
                for vm in [vmachine_1, vmachine_2]:
                    VMachineController.snapshot(machineguid=vm.guid,
                                                label='ss_i_{0}:00'.format(str(h)),
                                                is_consistent=False,
                                                timestamp=timestamp)
                    if h in [6, 12, 18]:
                        ts = (timestamp + (minute * 30))
                        VMachineController.snapshot(machineguid=vm.guid,
                                                    label='ss_c_{0}:30'.format(str(h)),
                                                    is_consistent=True,
                                                    timestamp=ts)

                VDiskController.create_snapshot(diskguid=vdisk_3.guid,
                                                metadata={'label': 'ss_i_{0}:00'.format(str(h)),
                                                          'is_consistent': False,
                                                          'timestamp': str(timestamp),
                                                          'machineguid': None})
                if h in [6, 12, 18]:
                    ts = (timestamp + (minute * 30))
                    VDiskController.create_snapshot(diskguid=vdisk_3.guid,
                                                    metadata={'label': 'ss_c_{0}:30'.format(str(h)),
                                                              'is_consistent': True,
                                                              'timestamp': str(ts),
                                                              'machineguid': None})
コード例 #8
0
    def test_happypath(self):
        """
        Validates the happy path; Hourly snapshots are taken with a few manual consistent
        every now and then. The delete policy is executed every day
        """
        vpool = VPool()
        vpool.name = 'vpool'
        vpool.status = 'RUNNING'
        vpool.save()
        storage_router = StorageRouter()
        storage_router.name = 'storage_router'
        storage_router.ip = '127.0.0.1'
        storage_router.machine_id = System.get_my_machine_id()
        storage_router.rdma_capable = False
        storage_router.save()
        disk = Disk()
        disk.name = 'physical_disk_1'
        disk.aliases = ['/dev/non-existent']
        disk.size = 500 * 1024**3
        disk.state = 'OK'
        disk.is_ssd = True
        disk.storagerouter = storage_router
        disk.save()
        disk_partition = DiskPartition()
        disk_partition.disk = disk
        disk_partition.aliases = ['/dev/disk/non-existent']
        disk_partition.size = 400 * 1024**3
        disk_partition.state = 'OK'
        disk_partition.offset = 1024
        disk_partition.roles = [DiskPartition.ROLES.SCRUB]
        disk_partition.mountpoint = '/var/tmp'
        disk_partition.save()
        vdisk_1 = VDisk()
        vdisk_1.name = 'vdisk_1'
        vdisk_1.volume_id = 'vdisk_1'
        vdisk_1.vpool = vpool
        vdisk_1.devicename = 'dummy'
        vdisk_1.size = 0
        vdisk_1.save()
        vdisk_1.reload_client('storagedriver')

        [
            dynamic for dynamic in vdisk_1._dynamics
            if dynamic.name == 'snapshots'
        ][0].timeout = 0

        # Run the testing scenario
        travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
        if travis is True:
            self._print_message('Running in Travis, reducing output.')
        debug = not travis
        amount_of_days = 50
        base = datetime.datetime.now().date()
        day = datetime.timedelta(1)
        minute = 60
        hour = minute * 60

        for d in xrange(0, amount_of_days):
            base_timestamp = self._make_timestamp(base, day * d)
            self._print_message('')
            self._print_message('Day cycle: {0}: {1}'.format(
                d,
                datetime.datetime.fromtimestamp(base_timestamp).strftime(
                    '%Y-%m-%d')))

            # At the start of the day, delete snapshot policy runs at 00:30
            self._print_message('- Deleting snapshots')
            ScheduledTaskController.delete_snapshots(timestamp=base_timestamp +
                                                     (minute * 30))

            # Validate snapshots
            self._print_message('- Validating snapshots')
            self._validate(vdisk_1, d, base, amount_of_days, debug)

            # During the day, snapshots are taken
            # - Create non consistent snapshot every hour, between 2:00 and 22:00
            # - Create consistent snapshot at 6:30, 12:30, 18:30
            self._print_message('- Creating snapshots')
            for h in xrange(2, 23):
                timestamp = base_timestamp + (hour * h)
                VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
                                                metadata={
                                                    'label':
                                                    'ss_i_{0}:00'.format(
                                                        str(h)),
                                                    'is_consistent':
                                                    False,
                                                    'timestamp':
                                                    str(timestamp),
                                                    'machineguid':
                                                    None
                                                })
                if h in [6, 12, 18]:
                    ts = (timestamp + (minute * 30))
                    VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
                                                    metadata={
                                                        'label':
                                                        'ss_c_{0}:30'.format(
                                                            str(h)),
                                                        'is_consistent':
                                                        True,
                                                        'timestamp':
                                                        str(ts),
                                                        'machineguid':
                                                        None
                                                    })
コード例 #9
0
    def test_clone_snapshot(self):
        """
        Validates that a snapshot that has clones will not be deleted while other snapshots will be deleted
        """
        # Setup
        # There are 2 disks, second one cloned from a snapshot of the first
        vpool = VPool()
        vpool.name = 'vpool'
        vpool.status = 'RUNNING'
        vpool.save()
        storage_router = StorageRouter()
        storage_router.name = 'storage_router'
        storage_router.ip = '127.0.0.1'
        storage_router.machine_id = System.get_my_machine_id()
        storage_router.rdma_capable = False
        storage_router.save()
        disk = Disk()
        disk.name = 'physical_disk_1'
        disk.aliases = ['/dev/non-existent']
        disk.size = 500 * 1024 ** 3
        disk.state = 'OK'
        disk.is_ssd = True
        disk.storagerouter = storage_router
        disk.save()
        disk_partition = DiskPartition()
        disk_partition.disk = disk
        disk_partition.aliases = ['/dev/disk/non-existent']
        disk_partition.size = 400 * 1024 ** 3
        disk_partition.state = 'OK'
        disk_partition.offset = 1024
        disk_partition.roles = [DiskPartition.ROLES.SCRUB]
        disk_partition.mountpoint = '/var/tmp'
        disk_partition.save()
        storage_driver = StorageDriver()
        storage_driver.vpool = vpool
        storage_driver.storagerouter = storage_router
        storage_driver.name = 'storage_driver_1'
        storage_driver.mountpoint = '/'
        storage_driver.cluster_ip = storage_router.ip
        storage_driver.storage_ip = '127.0.0.1'
        storage_driver.storagedriver_id = 'storage_driver_1'
        storage_driver.ports = {'management': 1,
                                'xmlrpc': 2,
                                'dtl': 3,
                                'edge': 4}
        storage_driver.save()
        service_type = ServiceType()
        service_type.name = 'MetadataServer'
        service_type.save()
        service = Service()
        service.name = 'service_1'
        service.storagerouter = storage_driver.storagerouter
        service.ports = [1]
        service.type = service_type
        service.save()
        mds_service = MDSService()
        mds_service.service = service
        mds_service.number = 0
        mds_service.capacity = 10
        mds_service.vpool = storage_driver.vpool
        mds_service.save()
        vdisk_1_1 = VDisk()
        vdisk_1_1.name = 'vdisk_1_1'
        vdisk_1_1.volume_id = 'vdisk_1_1'
        vdisk_1_1.vpool = vpool
        vdisk_1_1.devicename = 'dummy'
        vdisk_1_1.size = 0
        vdisk_1_1.save()
        vdisk_1_1.reload_client('storagedriver')

        [dynamic for dynamic in vdisk_1_1._dynamics if dynamic.name == 'snapshots'][0].timeout = 0

        travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
        if travis is True:
            print 'Running in Travis, reducing output.'

        base = datetime.datetime.now().date()
        day = datetime.timedelta(1)
        base_timestamp = self._make_timestamp(base, day)
        minute = 60
        hour = minute * 60
        for h in [6, 12, 18]:
            timestamp = base_timestamp + (hour * h)
            VDiskController.create_snapshot(vdisk_guid=vdisk_1_1.guid,
                                            metadata={'label': 'snapshot_{0}:30'.format(str(h)),
                                                      'is_consistent': True,
                                                      'timestamp': str(timestamp),
                                                      'machineguid': None})

        base_snapshot_guid = vdisk_1_1.snapshots[0]['guid']  # Oldest
        clone_vdisk = VDisk()
        clone_vdisk.name = 'clone_vdisk'
        clone_vdisk.volume_id = 'clone_vdisk'
        clone_vdisk.vpool = vpool
        clone_vdisk.devicename = 'dummy'
        clone_vdisk.parentsnapshot = base_snapshot_guid
        clone_vdisk.size = 0
        clone_vdisk.save()
        clone_vdisk.reload_client('storagedriver')

        for h in [6, 12, 18]:
            timestamp = base_timestamp + (hour * h)
            VDiskController.create_snapshot(vdisk_guid=clone_vdisk.guid,
                                            metadata={'label': 'snapshot_{0}:30'.format(str(h)),
                                                      'is_consistent': True,
                                                      'timestamp': str(timestamp),
                                                      'machineguid': None})

        base_timestamp = self._make_timestamp(base, day * 2)
        ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
        self.assertIn(base_snapshot_guid, [snap['guid'] for snap in vdisk_1_1.snapshots], 'Snapshot was deleted while there are still clones of it')
コード例 #10
0
    def test_happypath(self):
        """
        Validates the happy path; Hourly snapshots are taken with a few manual consistent
        every now and then. The delete policy is executed every day
        """
        vpool = VPool()
        vpool.name = 'vpool'
        vpool.status = 'RUNNING'
        vpool.save()
        storage_router = StorageRouter()
        storage_router.name = 'storage_router'
        storage_router.ip = '127.0.0.1'
        storage_router.machine_id = System.get_my_machine_id()
        storage_router.rdma_capable = False
        storage_router.save()
        disk = Disk()
        disk.name = 'physical_disk_1'
        disk.aliases = ['/dev/non-existent']
        disk.size = 500 * 1024 ** 3
        disk.state = 'OK'
        disk.is_ssd = True
        disk.storagerouter = storage_router
        disk.save()
        disk_partition = DiskPartition()
        disk_partition.disk = disk
        disk_partition.aliases = ['/dev/disk/non-existent']
        disk_partition.size = 400 * 1024 ** 3
        disk_partition.state = 'OK'
        disk_partition.offset = 1024
        disk_partition.roles = [DiskPartition.ROLES.SCRUB]
        disk_partition.mountpoint = '/var/tmp'
        disk_partition.save()
        vdisk_1 = VDisk()
        vdisk_1.name = 'vdisk_1'
        vdisk_1.volume_id = 'vdisk_1'
        vdisk_1.vpool = vpool
        vdisk_1.devicename = 'dummy'
        vdisk_1.size = 0
        vdisk_1.save()
        vdisk_1.reload_client('storagedriver')

        [dynamic for dynamic in vdisk_1._dynamics if dynamic.name == 'snapshots'][0].timeout = 0

        # Run the testing scenario
        travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
        if travis is True:
            self._print_message('Running in Travis, reducing output.')
        debug = not travis
        amount_of_days = 50
        base = datetime.datetime.now().date()
        day = datetime.timedelta(1)
        minute = 60
        hour = minute * 60

        for d in xrange(0, amount_of_days):
            base_timestamp = self._make_timestamp(base, day * d)
            self._print_message('')
            self._print_message('Day cycle: {0}: {1}'.format(d, datetime.datetime.fromtimestamp(base_timestamp).strftime('%Y-%m-%d')))

            # At the start of the day, delete snapshot policy runs at 00:30
            self._print_message('- Deleting snapshots')
            ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))

            # Validate snapshots
            self._print_message('- Validating snapshots')
            self._validate(vdisk_1, d, base, amount_of_days, debug)

            # During the day, snapshots are taken
            # - Create non consistent snapshot every hour, between 2:00 and 22:00
            # - Create consistent snapshot at 6:30, 12:30, 18:30
            self._print_message('- Creating snapshots')
            for h in xrange(2, 23):
                timestamp = base_timestamp + (hour * h)
                VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
                                                metadata={'label': 'ss_i_{0}:00'.format(str(h)),
                                                          'is_consistent': False,
                                                          'timestamp': str(timestamp),
                                                          'machineguid': None})
                if h in [6, 12, 18]:
                    ts = (timestamp + (minute * 30))
                    VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
                                                    metadata={'label': 'ss_c_{0}:30'.format(str(h)),
                                                              'is_consistent': True,
                                                              'timestamp': str(ts),
                                                              'machineguid': None})
コード例 #11
0
ファイル: disk.py プロジェクト: teotikalki/openvstorage
 def sync_with_reality(storagerouter_guid=None):
     """
     Syncs the Disks from all StorageRouters with the reality.
     """
     storagerouters = []
     if storagerouter_guid is not None:
         storagerouters.append(StorageRouter(storagerouter_guid))
     else:
         storagerouters = StorageRouterList.get_storagerouters()
     for storagerouter in storagerouters:
         try:
             client = SSHClient(storagerouter, username='******')
         except UnableToConnectException:
             logger.info(
                 'Could not connect to StorageRouter {0}, skipping'.format(
                     storagerouter.ip))
             continue
         configuration = {}
         # Gather mount data
         mount_mapping = {}
         mount_data = client.run('mount')
         for mount in mount_data.splitlines():
             mount = mount.strip()
             match = re.search('/dev/(.+?) on (/.*?) type.*', mount)
             if match is not None:
                 mount_mapping[match.groups()[0]] = match.groups()[1]
         # Gather disk information
         with Remote(storagerouter.ip, [Context, os]) as remote:
             context = remote.Context()
             devices = [
                 device
                 for device in context.list_devices(subsystem='block')
                 if 'ID_TYPE' in device and device['ID_TYPE'] == 'disk'
             ]
             for device in devices:
                 is_partition = device['DEVTYPE'] == 'partition'
                 device_path = device['DEVNAME']
                 device_name = device_path.split('/')[-1]
                 partition_id = None
                 partition_name = None
                 extended_parition_info = None
                 if is_partition is True:
                     if 'ID_PART_ENTRY_NUMBER' in device:
                         extended_parition_info = True
                         partition_id = device['ID_PART_ENTRY_NUMBER']
                         partition_name = device_name
                         device_name = device_name[:0 -
                                                   int(len(partition_id))]
                     else:
                         logger.debug(
                             'Partition {0} has no partition metadata'.
                             format(device_path))
                         extended_parition_info = False
                         match = re.match('^(\D+?)(\d+)$', device_name)
                         if match is None:
                             logger.debug(
                                 'Could not handle disk/partition {0}'.
                                 format(device_path))
                             continue  # Unable to handle this disk/partition
                         partition_name = device_name
                         partition_id = match.groups()[1]
                         device_name = match.groups()[0]
                 if device_name not in configuration:
                     configuration[device_name] = {'partitions': {}}
                 path = None
                 for path_type in ['by-id', 'by-uuid']:
                     if path is not None:
                         break
                     for item in device['DEVLINKS'].split(' '):
                         if path_type in item:
                             path = item
                 if path is None:
                     path = device_path
                 sectors = int(
                     client.run(
                         'cat /sys/block/{0}/size'.format(device_name)))
                 sector_size = int(
                     client.run(
                         'cat /sys/block/{0}/queue/hw_sector_size'.format(
                             device_name)))
                 rotational = int(
                     client.run(
                         'cat /sys/block/{0}/queue/rotational'.format(
                             device_name)))
                 if is_partition is True:
                     if 'ID_PART_ENTRY_TYPE' in device and device[
                             'ID_PART_ENTRY_TYPE'] == '0x5':
                         continue  # This is an extended partition, let's skip that one
                     if extended_parition_info is True:
                         offset = int(
                             device['ID_PART_ENTRY_OFFSET']) * sector_size
                         size = int(
                             device['ID_PART_ENTRY_SIZE']) * sector_size
                     else:
                         match = re.match('^(\D+?)(\d+)$', device_path)
                         if match is None:
                             logger.debug(
                                 'Could not handle disk/partition {0}'.
                                 format(device_path))
                             continue  # Unable to handle this disk/partition
                         fdisk_info = client.run(
                             'fdisk -l {0} | grep {1}'.format(
                                 match.groups()[0], device_path)).strip()
                         fdisk_data = filter(None, fdisk_info.split(' '))
                         offset = int(fdisk_data[1]) * sector_size
                         size = (int(fdisk_data[2]) -
                                 int(fdisk_data[1])) * sector_size
                     configuration[device_name]['partitions'][
                         partition_id] = {
                             'offset': offset,
                             'size': size,
                             'path': path,
                             'state': 'OK'
                         }
                     partition_data = configuration[device_name][
                         'partitions'][partition_id]
                     if partition_name in mount_mapping:
                         mountpoint = mount_mapping[partition_name]
                         partition_data['mountpoint'] = mountpoint
                         partition_data['inode'] = remote.os.stat(
                             mountpoint).st_dev
                         del mount_mapping[partition_name]
                         try:
                             client.run('touch {0}/{1}; rm {0}/{1}'.format(
                                 mountpoint, str(time.time())))
                         except CalledProcessError:
                             partition_data['state'] = 'ERROR'
                             pass
                     if 'ID_FS_TYPE' in device:
                         partition_data['filesystem'] = device['ID_FS_TYPE']
                 else:
                     configuration[device_name].update({
                         'name':
                         device_name,
                         'path':
                         path,
                         'vendor':
                         device['ID_VENDOR']
                         if 'ID_VENDOR' in device else None,
                         'model':
                         device['ID_MODEL']
                         if 'ID_MODEL' in device else None,
                         'size':
                         sector_size * sectors,
                         'is_ssd':
                         rotational == 0,
                         'state':
                         'OK'
                     })
                 for partition_name in mount_mapping:
                     device_name = partition_name.split('/')[-1]
                     match = re.search('^(\D+?)(\d+)$', device_name)
                     if match is not None:
                         device_name = match.groups()[0]
                         partition_id = match.groups()[1]
                         if device_name not in configuration:
                             configuration[device_name] = {
                                 'partitions': {},
                                 'state': 'MISSING'
                             }
                         configuration[device_name]['partitions'][
                             partition_id] = {
                                 'mountpoint':
                                 mount_mapping[partition_name],
                                 'state': 'MISSING'
                             }
         # Sync the model
         disk_names = []
         for disk in storagerouter.disks:
             if disk.name not in configuration:
                 for partition in disk.partitions:
                     partition.delete()
                 disk.delete()
             else:
                 disk_names.append(disk.name)
                 DiskController._update_disk(disk, configuration[disk.name])
                 partitions = []
                 partition_info = configuration[disk.name]['partitions']
                 for partition in disk.partitions:
                     if partition.id not in partition_info:
                         partition.delete()
                     else:
                         partitions.append(partition.id)
                         DiskController._update_partition(
                             partition, partition_info[partition.id])
                 for partition_id in partition_info:
                     if partition_id not in partitions:
                         DiskController._create_partition(
                             partition_id, partition_info[partition_id],
                             disk)
         for disk_name in configuration:
             if disk_name not in disk_names and configuration[disk_name][
                     'state'] not in ['MISSING']:
                 disk = Disk()
                 disk.storagerouter = storagerouter
                 disk.name = disk_name
                 DiskController._update_disk(disk, configuration[disk_name])
                 partition_info = configuration[disk_name]['partitions']
                 for partition_id in partition_info:
                     if partition_info[partition_id]['state'] not in [
                             'MISSING'
                     ]:
                         DiskController._create_partition(
                             partition_id, partition_info[partition_id],
                             disk)
コード例 #12
0
    def build_service_structure(structure, previous_structure=None):
        """
        Builds an MDS service structure
        Example:
            structure = Helper.build_service_structure(
                {'vpools': [1],
                 'domains': [],
                 'storagerouters': [1],
                 'storagedrivers': [(1, 1, 1)],  # (<id>, <vpool_id>, <storagerouter_id>)
                 'mds_services': [(1, 1)],  # (<id>, <storagedriver_id>)
                 'storagerouter_domains': []}  # (<id>, <storagerouter_id>, <domain_id>)
            )
        """
        if previous_structure is None:
            previous_structure = {}
        vdisks = previous_structure.get('vdisks', {})
        vpools = previous_structure.get('vpools', {})
        domains = previous_structure.get('domains', {})
        services = previous_structure.get('services', {})
        mds_services = previous_structure.get('mds_services', {})
        storagerouters = previous_structure.get('storagerouters', {})
        storagedrivers = previous_structure.get('storagedrivers', {})
        storagerouter_domains = previous_structure.get('storagerouter_domains',
                                                       {})

        service_type = ServiceTypeList.get_by_name('MetadataServer')
        if service_type is None:
            service_type = ServiceType()
            service_type.name = 'MetadataServer'
            service_type.save()
        srclients = {}
        for domain_id in structure.get('domains', []):
            if domain_id not in domains:
                domain = Domain()
                domain.name = 'domain_{0}'.format(domain_id)
                domain.save()
                domains[domain_id] = domain
        for vpool_id in structure.get('vpools', []):
            if vpool_id not in vpools:
                vpool = VPool()
                vpool.name = str(vpool_id)
                vpool.status = 'RUNNING'
                vpool.save()
                vpools[vpool_id] = vpool
            else:
                vpool = vpools[vpool_id]
            srclients[vpool_id] = StorageRouterClient(vpool.guid, None)
        for sr_id in structure.get('storagerouters', []):
            if sr_id not in storagerouters:
                storagerouter = StorageRouter()
                storagerouter.name = str(sr_id)
                storagerouter.ip = '10.0.0.{0}'.format(sr_id)
                storagerouter.rdma_capable = False
                storagerouter.node_type = 'MASTER'
                storagerouter.machine_id = str(sr_id)
                storagerouter.save()
                storagerouters[sr_id] = storagerouter
                disk = Disk()
                disk.storagerouter = storagerouter
                disk.state = 'OK'
                disk.name = '/dev/uda'
                disk.size = 1 * 1024**4
                disk.is_ssd = True
                disk.aliases = ['/dev/uda']
                disk.save()
                partition = DiskPartition()
                partition.offset = 0
                partition.size = disk.size
                partition.aliases = ['/dev/uda-1']
                partition.state = 'OK'
                partition.mountpoint = '/tmp/unittest/sr_{0}/disk_1/partition_1'.format(
                    sr_id)
                partition.disk = disk
                partition.roles = [
                    DiskPartition.ROLES.DB, DiskPartition.ROLES.SCRUB
                ]
                partition.save()
        for sd_id, vpool_id, sr_id in structure.get('storagedrivers', ()):
            if sd_id not in storagedrivers:
                storagedriver = StorageDriver()
                storagedriver.vpool = vpools[vpool_id]
                storagedriver.storagerouter = storagerouters[sr_id]
                storagedriver.name = str(sd_id)
                storagedriver.mountpoint = '/'
                storagedriver.cluster_ip = storagerouters[sr_id].ip
                storagedriver.storage_ip = '10.0.1.{0}'.format(sr_id)
                storagedriver.storagedriver_id = str(sd_id)
                storagedriver.ports = {
                    'management': 1,
                    'xmlrpc': 2,
                    'dtl': 3,
                    'edge': 4
                }
                storagedriver.save()
                storagedrivers[sd_id] = storagedriver
                Helper._set_vpool_storage_driver_configuration(
                    vpool=vpools[vpool_id], storagedriver=storagedriver)
        for mds_id, sd_id in structure.get('mds_services', ()):
            if mds_id not in mds_services:
                sd = storagedrivers[sd_id]
                s_id = '{0}-{1}'.format(sd.storagerouter.name, mds_id)
                service = Service()
                service.name = s_id
                service.storagerouter = sd.storagerouter
                service.ports = [mds_id]
                service.type = service_type
                service.save()
                services[s_id] = service
                mds_service = MDSService()
                mds_service.service = service
                mds_service.number = 0
                mds_service.capacity = 10
                mds_service.vpool = sd.vpool
                mds_service.save()
                mds_services[mds_id] = mds_service
                StorageDriverController.add_storagedriverpartition(
                    sd, {
                        'size': None,
                        'role': DiskPartition.ROLES.DB,
                        'sub_role': StorageDriverPartition.SUBROLE.MDS,
                        'partition': sd.storagerouter.disks[0].partitions[0],
                        'mds_service': mds_service
                    })
        for vdisk_id, storage_driver_id, vpool_id, mds_id in structure.get(
                'vdisks', ()):
            if vdisk_id not in vdisks:
                vpool = vpools[vpool_id]
                devicename = 'vdisk_{0}'.format(vdisk_id)
                mds_backend_config = Helper._generate_mdsmetadatabackendconfig(
                    [] if mds_id is None else [mds_services[mds_id]])
                volume_id = srclients[vpool_id].create_volume(
                    devicename, mds_backend_config, 0, str(storage_driver_id))
                vdisk = VDisk()
                vdisk.name = str(vdisk_id)
                vdisk.devicename = devicename
                vdisk.volume_id = volume_id
                vdisk.vpool = vpool
                vdisk.size = 0
                vdisk.save()
                vdisk.reload_client('storagedriver')
                vdisks[vdisk_id] = vdisk
        for srd_id, sr_id, domain_id, backup in structure.get(
                'storagerouter_domains', ()):
            if srd_id not in storagerouter_domains:
                sr_domain = StorageRouterDomain()
                sr_domain.backup = backup
                sr_domain.domain = domains[domain_id]
                sr_domain.storagerouter = storagerouters[sr_id]
                sr_domain.save()
                storagerouter_domains[srd_id] = sr_domain
        return {
            'vdisks': vdisks,
            'vpools': vpools,
            'domains': domains,
            'services': services,
            'service_type': service_type,
            'mds_services': mds_services,
            'storagerouters': storagerouters,
            'storagedrivers': storagedrivers,
            'storagerouter_domains': storagerouter_domains
        }
コード例 #13
0
ファイル: helpers.py プロジェクト: grimpy/openvstorage
    def build_service_structure(structure, previous_structure=None):
        """
        Builds an MDS service structure
        Example:
            structure = Helper.build_service_structure(
                {'vpools': [1],
                 'domains': [],
                 'storagerouters': [1],
                 'storagedrivers': [(1, 1, 1)],  # (<id>, <vpool_id>, <storagerouter_id>)
                 'mds_services': [(1, 1)],  # (<id>, <storagedriver_id>)
                 'storagerouter_domains': []}  # (<id>, <storagerouter_id>, <domain_id>)
            )
        """
        if previous_structure is None:
            previous_structure = {}
        vdisks = previous_structure.get("vdisks", {})
        vpools = previous_structure.get("vpools", {})
        domains = previous_structure.get("domains", {})
        services = previous_structure.get("services", {})
        mds_services = previous_structure.get("mds_services", {})
        storagerouters = previous_structure.get("storagerouters", {})
        storagedrivers = previous_structure.get("storagedrivers", {})
        storagerouter_domains = previous_structure.get("storagerouter_domains", {})

        service_type = ServiceTypeList.get_by_name("MetadataServer")
        if service_type is None:
            service_type = ServiceType()
            service_type.name = "MetadataServer"
            service_type.save()
        srclients = {}
        for domain_id in structure.get("domains", []):
            if domain_id not in domains:
                domain = Domain()
                domain.name = "domain_{0}".format(domain_id)
                domain.save()
                domains[domain_id] = domain
        for vpool_id in structure.get("vpools", []):
            if vpool_id not in vpools:
                vpool = VPool()
                vpool.name = str(vpool_id)
                vpool.status = "RUNNING"
                vpool.save()
                vpools[vpool_id] = vpool
            else:
                vpool = vpools[vpool_id]
            srclients[vpool_id] = StorageRouterClient(vpool.guid, None)
        for sr_id in structure.get("storagerouters", []):
            if sr_id not in storagerouters:
                storagerouter = StorageRouter()
                storagerouter.name = str(sr_id)
                storagerouter.ip = "10.0.0.{0}".format(sr_id)
                storagerouter.rdma_capable = False
                storagerouter.node_type = "MASTER"
                storagerouter.machine_id = str(sr_id)
                storagerouter.save()
                storagerouters[sr_id] = storagerouter
                disk = Disk()
                disk.storagerouter = storagerouter
                disk.state = "OK"
                disk.name = "/dev/uda"
                disk.size = 1 * 1024 ** 4
                disk.is_ssd = True
                disk.aliases = ["/dev/uda"]
                disk.save()
                partition = DiskPartition()
                partition.offset = 0
                partition.size = disk.size
                partition.aliases = ["/dev/uda-1"]
                partition.state = "OK"
                partition.mountpoint = "/tmp/unittest/sr_{0}/disk_1/partition_1".format(sr_id)
                partition.disk = disk
                partition.roles = [DiskPartition.ROLES.DB, DiskPartition.ROLES.SCRUB]
                partition.save()
        for sd_id, vpool_id, sr_id in structure.get("storagedrivers", ()):
            if sd_id not in storagedrivers:
                storagedriver = StorageDriver()
                storagedriver.vpool = vpools[vpool_id]
                storagedriver.storagerouter = storagerouters[sr_id]
                storagedriver.name = str(sd_id)
                storagedriver.mountpoint = "/"
                storagedriver.cluster_ip = storagerouters[sr_id].ip
                storagedriver.storage_ip = "10.0.1.{0}".format(sr_id)
                storagedriver.storagedriver_id = str(sd_id)
                storagedriver.ports = {"management": 1, "xmlrpc": 2, "dtl": 3, "edge": 4}
                storagedriver.save()
                storagedrivers[sd_id] = storagedriver
                Helper._set_vpool_storage_driver_configuration(vpool=vpools[vpool_id], storagedriver=storagedriver)
        for mds_id, sd_id in structure.get("mds_services", ()):
            if mds_id not in mds_services:
                sd = storagedrivers[sd_id]
                s_id = "{0}-{1}".format(sd.storagerouter.name, mds_id)
                service = Service()
                service.name = s_id
                service.storagerouter = sd.storagerouter
                service.ports = [mds_id]
                service.type = service_type
                service.save()
                services[s_id] = service
                mds_service = MDSService()
                mds_service.service = service
                mds_service.number = 0
                mds_service.capacity = 10
                mds_service.vpool = sd.vpool
                mds_service.save()
                mds_services[mds_id] = mds_service
                StorageDriverController.add_storagedriverpartition(
                    sd,
                    {
                        "size": None,
                        "role": DiskPartition.ROLES.DB,
                        "sub_role": StorageDriverPartition.SUBROLE.MDS,
                        "partition": sd.storagerouter.disks[0].partitions[0],
                        "mds_service": mds_service,
                    },
                )
        for vdisk_id, storage_driver_id, vpool_id, mds_id in structure.get("vdisks", ()):
            if vdisk_id not in vdisks:
                vpool = vpools[vpool_id]
                devicename = "vdisk_{0}".format(vdisk_id)
                mds_backend_config = Helper._generate_mdsmetadatabackendconfig(
                    [] if mds_id is None else [mds_services[mds_id]]
                )
                volume_id = srclients[vpool_id].create_volume(devicename, mds_backend_config, 0, str(storage_driver_id))
                vdisk = VDisk()
                vdisk.name = str(vdisk_id)
                vdisk.devicename = devicename
                vdisk.volume_id = volume_id
                vdisk.vpool = vpool
                vdisk.size = 0
                vdisk.save()
                vdisk.reload_client("storagedriver")
                vdisks[vdisk_id] = vdisk
        for srd_id, sr_id, domain_id, backup in structure.get("storagerouter_domains", ()):
            if srd_id not in storagerouter_domains:
                sr_domain = StorageRouterDomain()
                sr_domain.backup = backup
                sr_domain.domain = domains[domain_id]
                sr_domain.storagerouter = storagerouters[sr_id]
                sr_domain.save()
                storagerouter_domains[srd_id] = sr_domain
        return {
            "vdisks": vdisks,
            "vpools": vpools,
            "domains": domains,
            "services": services,
            "service_type": service_type,
            "mds_services": mds_services,
            "storagerouters": storagerouters,
            "storagedrivers": storagedrivers,
            "storagerouter_domains": storagerouter_domains,
        }
コード例 #14
0
    def test_clone_snapshot(self):
        """
        Validates that a snapshot that has clones will not be deleted while other snapshots will be deleted
        """
        # Setup
        # There are 2 disks, second one cloned from a snapshot of the first
        vpool = VPool()
        vpool.name = 'vpool'
        vpool.status = 'RUNNING'
        vpool.save()
        storage_router = StorageRouter()
        storage_router.name = 'storage_router'
        storage_router.ip = '127.0.0.1'
        storage_router.machine_id = System.get_my_machine_id()
        storage_router.rdma_capable = False
        storage_router.save()
        disk = Disk()
        disk.name = 'physical_disk_1'
        disk.aliases = ['/dev/non-existent']
        disk.size = 500 * 1024**3
        disk.state = 'OK'
        disk.is_ssd = True
        disk.storagerouter = storage_router
        disk.save()
        disk_partition = DiskPartition()
        disk_partition.disk = disk
        disk_partition.aliases = ['/dev/disk/non-existent']
        disk_partition.size = 400 * 1024**3
        disk_partition.state = 'OK'
        disk_partition.offset = 1024
        disk_partition.roles = [DiskPartition.ROLES.SCRUB]
        disk_partition.mountpoint = '/var/tmp'
        disk_partition.save()
        storage_driver = StorageDriver()
        storage_driver.vpool = vpool
        storage_driver.storagerouter = storage_router
        storage_driver.name = 'storage_driver_1'
        storage_driver.mountpoint = '/'
        storage_driver.cluster_ip = storage_router.ip
        storage_driver.storage_ip = '127.0.0.1'
        storage_driver.storagedriver_id = 'storage_driver_1'
        storage_driver.ports = {
            'management': 1,
            'xmlrpc': 2,
            'dtl': 3,
            'edge': 4
        }
        storage_driver.save()
        service_type = ServiceType()
        service_type.name = 'MetadataServer'
        service_type.save()
        service = Service()
        service.name = 'service_1'
        service.storagerouter = storage_driver.storagerouter
        service.ports = [1]
        service.type = service_type
        service.save()
        mds_service = MDSService()
        mds_service.service = service
        mds_service.number = 0
        mds_service.capacity = 10
        mds_service.vpool = storage_driver.vpool
        mds_service.save()
        vdisk_1_1 = VDisk()
        vdisk_1_1.name = 'vdisk_1_1'
        vdisk_1_1.volume_id = 'vdisk_1_1'
        vdisk_1_1.vpool = vpool
        vdisk_1_1.devicename = 'dummy'
        vdisk_1_1.size = 0
        vdisk_1_1.save()
        vdisk_1_1.reload_client('storagedriver')

        [
            dynamic for dynamic in vdisk_1_1._dynamics
            if dynamic.name == 'snapshots'
        ][0].timeout = 0

        travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
        if travis is True:
            print 'Running in Travis, reducing output.'

        base = datetime.datetime.now().date()
        day = datetime.timedelta(1)
        base_timestamp = self._make_timestamp(base, day)
        minute = 60
        hour = minute * 60
        for h in [6, 12, 18]:
            timestamp = base_timestamp + (hour * h)
            VDiskController.create_snapshot(vdisk_guid=vdisk_1_1.guid,
                                            metadata={
                                                'label':
                                                'snapshot_{0}:30'.format(
                                                    str(h)),
                                                'is_consistent':
                                                True,
                                                'timestamp':
                                                str(timestamp),
                                                'machineguid':
                                                None
                                            })

        base_snapshot_guid = vdisk_1_1.snapshots[0]['guid']  # Oldest
        clone_vdisk = VDisk()
        clone_vdisk.name = 'clone_vdisk'
        clone_vdisk.volume_id = 'clone_vdisk'
        clone_vdisk.vpool = vpool
        clone_vdisk.devicename = 'dummy'
        clone_vdisk.parentsnapshot = base_snapshot_guid
        clone_vdisk.size = 0
        clone_vdisk.save()
        clone_vdisk.reload_client('storagedriver')

        for h in [6, 12, 18]:
            timestamp = base_timestamp + (hour * h)
            VDiskController.create_snapshot(vdisk_guid=clone_vdisk.guid,
                                            metadata={
                                                'label':
                                                'snapshot_{0}:30'.format(
                                                    str(h)),
                                                'is_consistent':
                                                True,
                                                'timestamp':
                                                str(timestamp),
                                                'machineguid':
                                                None
                                            })

        base_timestamp = self._make_timestamp(base, day * 2)
        ScheduledTaskController.delete_snapshots(timestamp=base_timestamp +
                                                 (minute * 30))
        self.assertIn(
            base_snapshot_guid, [snap['guid'] for snap in vdisk_1_1.snapshots],
            'Snapshot was deleted while there are still clones of it')
コード例 #15
0
ファイル: disk.py プロジェクト: jianyongchen/openvstorage
 def sync_with_reality(storagerouter_guid=None):
     """
     Syncs the Disks from all StorageRouters with the reality.
     """
     storagerouters = []
     if storagerouter_guid is not None:
         storagerouters.append(StorageRouter(storagerouter_guid))
     else:
         storagerouters = StorageRouterList.get_storagerouters()
     for storagerouter in storagerouters:
         try:
             client = SSHClient(storagerouter, username='******')
         except UnableToConnectException:
             logger.info('Could not connect to StorageRouter {0}, skipping'.format(storagerouter.ip))
             continue
         configuration = {}
         # Gather mount data
         mount_mapping = {}
         mount_data = client.run('mount')
         for mount in mount_data.splitlines():
             mount = mount.strip()
             match = re.search('/dev/(.+?) on (/.*?) type.*', mount)
             if match is not None:
                 mount_mapping[match.groups()[0]] = match.groups()[1]
         # Gather disk information
         with Remote(storagerouter.ip, [Context, os]) as remote:
             context = remote.Context()
             devices = [device for device in context.list_devices(subsystem='block')
                        if 'ID_TYPE' in device and device['ID_TYPE'] == 'disk']
             for device in devices:
                 is_partition = device['DEVTYPE'] == 'partition'
                 device_path = device['DEVNAME']
                 device_name = device_path.split('/')[-1]
                 partition_id = None
                 partition_name = None
                 extended_parition_info = None
                 if is_partition is True:
                     if 'ID_PART_ENTRY_NUMBER' in device:
                         extended_parition_info = True
                         partition_id = device['ID_PART_ENTRY_NUMBER']
                         partition_name = device_name
                         device_name = device_name[:0 - int(len(partition_id))]
                     else:
                         logger.debug('Partition {0} has no partition metadata'.format(device_path))
                         extended_parition_info = False
                         match = re.match('^(\D+?)(\d+)$', device_name)
                         if match is None:
                             logger.debug('Could not handle disk/partition {0}'.format(device_path))
                             continue  # Unable to handle this disk/partition
                         partition_name = device_name
                         partition_id = match.groups()[1]
                         device_name = match.groups()[0]
                 if device_name not in configuration:
                     configuration[device_name] = {'partitions': {}}
                 path = None
                 for path_type in ['by-id', 'by-uuid']:
                     if path is not None:
                         break
                     for item in device['DEVLINKS'].split(' '):
                         if path_type in item:
                             path = item
                 if path is None:
                     path = device_path
                 sectors = int(client.run('cat /sys/block/{0}/size'.format(device_name)))
                 sector_size = int(client.run('cat /sys/block/{0}/queue/hw_sector_size'.format(device_name)))
                 rotational = int(client.run('cat /sys/block/{0}/queue/rotational'.format(device_name)))
                 if is_partition is True:
                     if 'ID_PART_ENTRY_TYPE' in device and device['ID_PART_ENTRY_TYPE'] == '0x5':
                         continue  # This is an extended partition, let's skip that one
                     if extended_parition_info is True:
                         offset = int(device['ID_PART_ENTRY_OFFSET']) * sector_size
                         size = int(device['ID_PART_ENTRY_SIZE']) * sector_size
                     else:
                         match = re.match('^(\D+?)(\d+)$', device_path)
                         if match is None:
                             logger.debug('Could not handle disk/partition {0}'.format(device_path))
                             continue  # Unable to handle this disk/partition
                         fdisk_info = client.run('fdisk -l {0} | grep {1}'.format(match.groups()[0], device_path)).strip()
                         fdisk_data = filter(None, fdisk_info.split(' '))
                         offset = int(fdisk_data[1]) * sector_size
                         size = (int(fdisk_data[2]) - int(fdisk_data[1])) * sector_size
                     configuration[device_name]['partitions'][partition_id] = {'offset': offset,
                                                                               'size': size,
                                                                               'path': path,
                                                                               'state': 'OK'}
                     partition_data = configuration[device_name]['partitions'][partition_id]
                     if partition_name in mount_mapping:
                         mountpoint = mount_mapping[partition_name]
                         partition_data['mountpoint'] = mountpoint
                         partition_data['inode'] = remote.os.stat(mountpoint).st_dev
                         del mount_mapping[partition_name]
                         try:
                             client.run('touch {0}/{1}; rm {0}/{1}'.format(mountpoint, str(time.time())))
                         except CalledProcessError:
                             partition_data['state'] = 'FAILURE'
                             pass
                     if 'ID_FS_TYPE' in device:
                         partition_data['filesystem'] = device['ID_FS_TYPE']
                 else:
                     configuration[device_name].update({'name': device_name,
                                                        'path': path,
                                                        'vendor': device['ID_VENDOR'] if 'ID_VENDOR' in device else None,
                                                        'model': device['ID_MODEL'] if 'ID_MODEL' in device else None,
                                                        'size': sector_size * sectors,
                                                        'is_ssd': rotational == 0,
                                                        'state': 'OK'})
                 for partition_name in mount_mapping:
                     device_name = partition_name.split('/')[-1]
                     match = re.search('^(\D+?)(\d+)$', device_name)
                     if match is not None:
                         device_name = match.groups()[0]
                         partition_id = match.groups()[1]
                         if device_name not in configuration:
                             configuration[device_name] = {'partitions': {},
                                                           'state': 'MISSING'}
                         configuration[device_name]['partitions'][partition_id] = {'mountpoint': mount_mapping[partition_name],
                                                                                   'state': 'MISSING'}
         # Sync the model
         disk_names = []
         for disk in storagerouter.disks:
             if disk.name not in configuration:
                 for partition in disk.partitions:
                     partition.delete()
                 disk.delete()
             else:
                 disk_names.append(disk.name)
                 DiskController._update_disk(disk, configuration[disk.name])
                 partitions = []
                 partition_info = configuration[disk.name]['partitions']
                 for partition in disk.partitions:
                     if partition.id not in partition_info:
                         partition.delete()
                     else:
                         partitions.append(partition.id)
                         DiskController._update_partition(partition, partition_info[partition.id])
                 for partition_id in partition_info:
                     if partition_id not in partitions:
                         DiskController._create_partition(partition_id, partition_info[partition_id], disk)
         for disk_name in configuration:
             if disk_name not in disk_names and configuration[disk_name]['state'] not in ['MISSING']:
                 disk = Disk()
                 disk.storagerouter = storagerouter
                 disk.name = disk_name
                 DiskController._update_disk(disk, configuration[disk_name])
                 partition_info = configuration[disk_name]['partitions']
                 for partition_id in partition_info:
                     if partition_info[partition_id]['state'] not in ['MISSING']:
                         DiskController._create_partition(partition_id, partition_info[partition_id], disk)
コード例 #16
0
ファイル: disk.py プロジェクト: DarumasLegs/framework
    def sync_with_reality(storagerouter_guid=None):
        """
        Syncs the Disks from all StorageRouters with the reality.
        :param storagerouter_guid: Guid of the Storage Router to synchronize
        """
        storagerouters = []
        if storagerouter_guid is not None:
            storagerouters.append(StorageRouter(storagerouter_guid))
        else:
            storagerouters = StorageRouterList.get_storagerouters()
        for storagerouter in storagerouters:
            try:
                client = SSHClient(storagerouter, username='******')
            except UnableToConnectException:
                DiskController._logger.info('Could not connect to StorageRouter {0}, skipping'.format(storagerouter.ip))
                continue
            configuration = {}
            # Gather mount data
            mount_mapping = {}
            mount_data = client.run('mount')
            for mount in mount_data.splitlines():
                mount = mount.strip()
                match = re.search('(/dev/(.+?)) on (/.*?) type.*', mount)
                if match is not None:
                    dev_name = match.groups()[0]
                    uuid = client.run('blkid -o value -s UUID {0}'.format(dev_name))
                    if uuid:
                        mount_mapping[uuid] = match.groups()[2]
                    else:
                        mount_mapping[match.groups()[1]] = match.groups()[2]
            # Gather raid information
            try:
                md_information = client.run('mdadm --detail /dev/md*', suppress_logging=True)
            except CalledProcessError:
                md_information = ''
            raid_members = []
            for member in re.findall('(?: +[0-9]+){4} +[^/]+/dev/([a-z0-9]+)', md_information):
                raid_members.append(member)
            # Gather disk information
            with remote(storagerouter.ip, [Context, os]) as rem:
                context = rem.Context()
                devices = [device for device in context.list_devices(subsystem='block')
                           if ('ID_TYPE' in device and device['ID_TYPE'] == 'disk') or
                              ('DEVNAME' in device and ('loop' in device['DEVNAME'] or 'nvme' in device['DEVNAME'] or 'md' in device['DEVNAME']))]
                for device in devices:
                    is_partition = device['DEVTYPE'] == 'partition'
                    device_path = device['DEVNAME']
                    device_name = device_path.split('/')[-1]
                    partition_id = None
                    partition_name = None
                    extended_partition_info = None
                    if is_partition is True:
                        partition_name = device['ID_FS_UUID'] if 'ID_FS_UUID' in device else device_name
                        if 'ID_PART_ENTRY_NUMBER' in device:
                            extended_partition_info = True
                            partition_id = device['ID_PART_ENTRY_NUMBER']
                            if device_name.startswith('nvme') or device_name.startswith('loop'):
                                device_name = device_name[:0 - int(len(partition_id)) - 1]
                            elif device_name.startswith('md'):
                                device_name = device_name[:device_name.index('p')]
                            else:
                                device_name = device_name[:0 - int(len(partition_id))]
                        else:
                            DiskController._logger.debug('Partition {0} has no partition metadata'.format(device_path))
                            extended_partition_info = False
                            match = re.match('^(\D+?)(\d+)$', device_name)
                            if match is None:
                                DiskController._logger.debug('Could not handle disk/partition {0}'.format(device_path))
                                continue  # Unable to handle this disk/partition
                            partition_id = match.groups()[1]
                            device_name = match.groups()[0]
                    sectors = int(client.run('cat /sys/block/{0}/size'.format(device_name)))
                    sector_size = int(client.run('cat /sys/block/{0}/queue/hw_sector_size'.format(device_name)))
                    rotational = int(client.run('cat /sys/block/{0}/queue/rotational'.format(device_name)))

                    if sectors == 0:
                        continue
                    if device_name in raid_members:
                        continue
                    if device_name not in configuration:
                        configuration[device_name] = {'partitions': {}}
                    path = None
                    for path_type in ['by-id', 'by-uuid']:
                        if path is not None:
                            break
                        if 'DEVLINKS' in device:
                            for item in device['DEVLINKS'].split(' '):
                                if path_type in item:
                                    path = item
                    if path is None:
                        path = device_path
                    if is_partition is True:
                        if 'ID_PART_ENTRY_TYPE' in device and device['ID_PART_ENTRY_TYPE'] == '0x5':
                            continue  # This is an extended partition, let's skip that one
                        if extended_partition_info is True:
                            offset = int(device['ID_PART_ENTRY_OFFSET']) * sector_size
                            size = int(device['ID_PART_ENTRY_SIZE']) * sector_size
                        else:
                            match = re.match('^(\D+?)(\d+)$', device_path)
                            if match is None:
                                DiskController._logger.debug('Could not handle disk/partition {0}'.format(device_path))
                                continue  # Unable to handle this disk/partition
                            partitions_info = DiskTools.get_partitions_info(match.groups()[0])
                            if device_path in partitions_info:
                                partition_info = partitions_info[device_path]
                                offset = int(partition_info['start'])
                                size = int(partition_info['size'])
                            else:
                                DiskController._logger.warning('Could not retrieve partition info for disk/partition {0}'.format(device_path))
                                continue
                        configuration[device_name]['partitions'][partition_id] = {'offset': offset,
                                                                                  'size': size,
                                                                                  'path': path,
                                                                                  'state': 'OK'}
                        partition_data = configuration[device_name]['partitions'][partition_id]
                        if partition_name in mount_mapping:
                            mountpoint = mount_mapping[partition_name]
                            partition_data['mountpoint'] = mountpoint
                            partition_data['inode'] = rem.os.stat(mountpoint).st_dev
                            del mount_mapping[partition_name]
                            try:
                                client.run('touch {0}/{1}; rm {0}/{1}'.format(mountpoint, str(time.time())))
                            except CalledProcessError:
                                partition_data['state'] = 'FAILURE'
                                pass
                        if 'ID_FS_TYPE' in device:
                            partition_data['filesystem'] = device['ID_FS_TYPE']
                    else:
                        configuration[device_name].update({'name': device_name,
                                                           'path': path,
                                                           'vendor': device['ID_VENDOR'] if 'ID_VENDOR' in device else None,
                                                           'model': device['ID_MODEL'] if 'ID_MODEL' in device else None,
                                                           'size': sector_size * sectors,
                                                           'is_ssd': rotational == 0,
                                                           'state': 'OK'})
                    for partition_name in mount_mapping:
                        device_name = partition_name.split('/')[-1]
                        match = re.search('^(\D+?)(\d+)$', device_name)
                        if match is not None:
                            device_name = match.groups()[0]
                            partition_id = match.groups()[1]
                            if device_name not in configuration:
                                configuration[device_name] = {'partitions': {},
                                                              'state': 'MISSING'}
                            configuration[device_name]['partitions'][partition_id] = {'mountpoint': mount_mapping[partition_name],
                                                                                      'state': 'MISSING'}
            # Sync the model
            disk_names = []
            for disk in storagerouter.disks:
                if disk.name not in configuration:
                    for partition in disk.partitions:
                        partition.delete()
                    disk.delete()
                else:
                    disk_names.append(disk.name)
                    DiskController._update_disk(disk, configuration[disk.name])
                    partitions = []
                    partition_info = configuration[disk.name]['partitions']
                    for partition in disk.partitions:
                        if partition.id not in partition_info:
                            partition.delete()
                        else:
                            partitions.append(partition.id)
                            DiskController._update_partition(partition, partition_info[partition.id])
                    for partition_id in partition_info:
                        if partition_id not in partitions:
                            DiskController._create_partition(partition_id, partition_info[partition_id], disk)
            for disk_name in configuration:
                if disk_name not in disk_names and configuration[disk_name]['state'] not in ['MISSING']:
                    disk = Disk()
                    disk.storagerouter = storagerouter
                    disk.name = disk_name
                    DiskController._update_disk(disk, configuration[disk_name])
                    partition_info = configuration[disk_name]['partitions']
                    for partition_id in partition_info:
                        if partition_info[partition_id]['state'] not in ['MISSING']:
                            DiskController._create_partition(partition_id, partition_info[partition_id], disk)
コード例 #17
0
ファイル: helpers.py プロジェクト: yongshengma/framework
    def build_dal_structure(structure, previous_structure=None):
        """
        Builds a model structure
        Example:
            structure = DalHelper.build_service_structure(
                {'vpools': [1],
                 'domains': [],
                 'storagerouters': [1],
                 'storagedrivers': [(1, 1, 1)],  # (<id>, <vpool_id>, <storagerouter_id>)
                 'mds_services': [(1, 1)],  # (<id>, <storagedriver_id>)
                 'storagerouter_domains': []}  # (<id>, <storagerouter_id>, <domain_id>)
            )
        """
        Configuration.set(key=Configuration.EDITION_KEY,
                          value=PackageFactory.EDITION_ENTERPRISE)

        if previous_structure is None:
            previous_structure = {}
        vdisks = previous_structure.get('vdisks', {})
        vpools = previous_structure.get('vpools', {})
        domains = previous_structure.get('domains', {})
        services = previous_structure.get('services', {})
        mds_services = previous_structure.get('mds_services', {})
        storagerouters = previous_structure.get('storagerouters', {})
        storagedrivers = previous_structure.get('storagedrivers', {})
        storagerouter_domains = previous_structure.get('storagerouter_domains',
                                                       {})

        service_types = {}
        for service_type_name in ServiceType.SERVICE_TYPES.values():
            service_type = ServiceTypeList.get_by_name(service_type_name)
            if service_type is None:
                service_type = ServiceType()
                service_type.name = service_type_name
                service_type.save()
            service_types[service_type_name] = service_type
        srclients = {}
        for domain_id in structure.get('domains', []):
            if domain_id not in domains:
                domain = Domain()
                domain.name = 'domain_{0}'.format(domain_id)
                domain.save()
                domains[domain_id] = domain
        for vpool_id in structure.get('vpools', []):
            if vpool_id not in vpools:
                vpool = VPool()
                vpool.name = str(vpool_id)
                vpool.status = 'RUNNING'
                vpool.metadata = {'backend': {}, 'caching_info': {}}
                vpool.metadata_store_bits = 5
                vpool.save()
                vpools[vpool_id] = vpool
            else:
                vpool = vpools[vpool_id]
            srclients[vpool_id] = StorageRouterClient(vpool.guid, None)
            Configuration.set(
                '/ovs/vpools/{0}/mds_config|mds_tlogs'.format(vpool.guid), 100)
            Configuration.set(
                '/ovs/vpools/{0}/mds_config|mds_safety'.format(vpool.guid), 2)
            Configuration.set(
                '/ovs/vpools/{0}/mds_config|mds_maxload'.format(vpool.guid),
                75)
            Configuration.set(
                '/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(
                    vpool.guid),
                json.dumps({}, indent=4),
                raw=True)
        for sr_id in structure.get('storagerouters', []):
            if sr_id not in storagerouters:
                storagerouter = StorageRouter()
                storagerouter.name = str(sr_id)
                storagerouter.ip = '10.0.0.{0}'.format(sr_id)
                storagerouter.rdma_capable = False
                storagerouter.node_type = 'MASTER'
                storagerouter.machine_id = str(sr_id)
                storagerouter.save()
                storagerouters[sr_id] = storagerouter
                disk = Disk()
                disk.storagerouter = storagerouter
                disk.state = 'OK'
                disk.name = '/dev/uda'
                disk.size = 1 * 1024**4
                disk.is_ssd = True
                disk.aliases = ['/dev/uda']
                disk.save()
                partition = DiskPartition()
                partition.offset = 0
                partition.size = disk.size
                partition.aliases = ['/dev/uda-1']
                partition.state = 'OK'
                partition.mountpoint = '/tmp/unittest/sr_{0}/disk_1/partition_1'.format(
                    sr_id)
                partition.disk = disk
                partition.roles = [
                    DiskPartition.ROLES.DB, DiskPartition.ROLES.SCRUB
                ]
                partition.save()
            else:
                storagerouter = storagerouters[sr_id]

            # noinspection PyProtectedMember
            System._machine_id[storagerouter.ip] = str(sr_id)
            mds_start = 10000 + 100 * (sr_id - 1)
            mds_end = 10000 + 100 * sr_id - 1
            arakoon_start = 20000 + 100 * (sr_id - 1)
            storagedriver_start = 30000 + 100 * (sr_id - 1)
            storagedriver_end = 30000 + 100 * sr_id - 1
            Configuration.initialize_host(
                host_id=sr_id,
                port_info={
                    'mds': [mds_start, mds_end],
                    'arakoon': arakoon_start,
                    'storagedriver': [storagedriver_start, storagedriver_end]
                })

        for sd_id, vpool_id, sr_id in structure.get('storagedrivers', ()):
            if sd_id not in storagedrivers:
                storagedriver = StorageDriver()
                storagedriver.vpool = vpools[vpool_id]
                storagedriver.storagerouter = storagerouters[sr_id]
                storagedriver.name = str(sd_id)
                storagedriver.mountpoint = '/'
                storagedriver.cluster_ip = storagerouters[sr_id].ip
                storagedriver.storage_ip = '10.0.1.{0}'.format(sr_id)
                storagedriver.storagedriver_id = str(sd_id)
                storagedriver.ports = {
                    'management': 1,
                    'xmlrpc': 2,
                    'dtl': 3,
                    'edge': 4
                }
                storagedriver.save()
                storagedrivers[sd_id] = storagedriver
                DalHelper.set_vpool_storage_driver_configuration(
                    vpool=vpools[vpool_id], storagedriver=storagedriver)
        for mds_id, sd_id in structure.get('mds_services', ()):
            if mds_id not in mds_services:
                sd = storagedrivers[sd_id]
                s_id = '{0}-{1}'.format(sd.storagerouter.name, mds_id)
                service = Service()
                service.name = s_id
                service.storagerouter = sd.storagerouter
                service.ports = [mds_id]
                service.type = service_types['MetadataServer']
                service.save()
                services[s_id] = service
                mds_service = MDSService()
                mds_service.service = service
                mds_service.number = 0
                mds_service.capacity = 10
                mds_service.vpool = sd.vpool
                mds_service.save()
                mds_services[mds_id] = mds_service
                StorageDriverController.add_storagedriverpartition(
                    sd, {
                        'size': None,
                        'role': DiskPartition.ROLES.DB,
                        'sub_role': StorageDriverPartition.SUBROLE.MDS,
                        'partition': sd.storagerouter.disks[0].partitions[0],
                        'mds_service': mds_service
                    })
        for vdisk_id, storage_driver_id, vpool_id, mds_id in structure.get(
                'vdisks', ()):
            if vdisk_id not in vdisks:
                vpool = vpools[vpool_id]
                devicename = 'vdisk_{0}'.format(vdisk_id)
                mds_backend_config = DalHelper.generate_mds_metadata_backend_config(
                    [] if mds_id is None else [mds_services[mds_id]])
                volume_id = srclients[vpool_id].create_volume(
                    devicename, mds_backend_config, 0, str(storage_driver_id))
                vdisk = VDisk()
                vdisk.name = str(vdisk_id)
                vdisk.devicename = devicename
                vdisk.volume_id = volume_id
                vdisk.vpool = vpool
                vdisk.size = 0
                vdisk.save()
                vdisk.reload_client('storagedriver')
                vdisks[vdisk_id] = vdisk
        for srd_id, sr_id, domain_id, backup in structure.get(
                'storagerouter_domains', ()):
            if srd_id not in storagerouter_domains:
                sr_domain = StorageRouterDomain()
                sr_domain.backup = backup
                sr_domain.domain = domains[domain_id]
                sr_domain.storagerouter = storagerouters[sr_id]
                sr_domain.save()
                storagerouter_domains[srd_id] = sr_domain
        return {
            'vdisks': vdisks,
            'vpools': vpools,
            'domains': domains,
            'services': services,
            'mds_services': mds_services,
            'service_types': service_types,
            'storagerouters': storagerouters,
            'storagedrivers': storagedrivers,
            'storagerouter_domains': storagerouter_domains
        }