Esempio n. 1
0
 def test_simple_device_info(self):
     device_info = DeviceInfo(model=safe_load(device_info_simple))
     self.assertEquals(device_info.region_name, 'region2')
     self.assertEquals(device_info.region_id, '2')
     self.assertEquals(device_info.replication_ip, None)
     self.assertEquals(device_info.presence, 'present')
     self.assertEquals(device_info.meta, 'host.example.com:disk99:/dev/sdh')
     with self.assertRaises(AttributeError):
         _ = device_info.junk
     with self.assertRaises(AttributeError):
         device_info.junk = 'junk'
Esempio n. 2
0
 def test_simple_device_info(self):
     device_info = DeviceInfo(model=safe_load(device_info_simple))
     self.assertEquals(device_info.region_name, 'region2')
     self.assertEquals(device_info.region_id, 2)
     self.assertEquals(device_info.replication_ip, None)
     self.assertEquals(device_info.presence, 'present')
     self.assertEquals(device_info.meta, 'host.example.com:disk99:/dev/sdh')
     with self.assertRaises(AttributeError):
         _ = device_info.junk
     with self.assertRaises(AttributeError):
         device_info.junk = 'junk'
Esempio n. 3
0
    def test_lvm_numbering(self):
        input_model = ServersModel('my_cloud',
                                   'my_control_plane',
                                   config=safe_load(lvm_disk_model),
                                   consumes_model=standard_swf_rng_consumes)
        lv_devices = []
        lv_info = set()
        for lv_device in input_model._iter_volume_groups():
            lv_devices.append(DeviceInfo(lv_device))
            lv_info.add((lv_device.server_name, lv_device.server_ip,
                         lv_device.ring_name, lv_device.swift_drive_name))
        self.assertEqual(len(lv_devices), 11)

        # Do not change this without also examining test_drivedata.py
        lv_expected = [
            ('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-0', 'lvm0'),
            ('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-1', 'lvm0'),
            ('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-2', 'lvm1'),
            ('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-3', 'lvm1'),
            ('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-4', 'lvm2'),
            ('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-5', 'lvm2'),
            ('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-6', 'lvm2'),
            ('standard-ccp-c1-m2-mgmt', '192.168.222.3', 'object-7', 'lvm0'),
            ('standard-ccp-c1-m2-mgmt', '192.168.222.3', 'object-8', 'lvm1'),
            ('standard-ccp-c1-m2-mgmt', '192.168.222.3', 'object-9', 'lvm2'),
            ('standard-ccp-c1-m2-mgmt', '192.168.222.3', 'object-10', 'lvm2')
        ]
        for lv in set(lv_expected):
            self.assertTrue(lv in lv_info,
                            '%s missing from %s' % (lv, lv_info))
            lv_expected.remove(lv)
        self.assertEqual(0, len(lv_expected), 'still have %s' % lv_expected)
Esempio n. 4
0
    def test_lvm_numbering(self):
        input_model = InputModel(config=safe_load(lvm_disk_model),
                                 hosts_fd=padawan_net_hosts,
                                 consumes=padawan_swf_rng_consumes)
        lv_devices = []
        lv_info = set()
        for lv_device in input_model._iter_volume_groups():
            lv_devices.append(DeviceInfo(lv_device))
            lv_info.add((lv_device.server_name, lv_device.ring_name,
                         lv_device.swift_drive_name))
        self.assertEqual(len(lv_devices), 11)

        # Do not change this without also examining test_drivedata.py
        lv_expected = [('padawan-ccp-c1-m1', 'object-0', 'lvm0'),
                       ('padawan-ccp-c1-m1', 'object-1', 'lvm0'),
                       ('padawan-ccp-c1-m1', 'object-2', 'lvm1'),
                       ('padawan-ccp-c1-m1', 'object-3', 'lvm1'),
                       ('padawan-ccp-c1-m1', 'object-4', 'lvm2'),
                       ('padawan-ccp-c1-m1', 'object-5', 'lvm2'),
                       ('padawan-ccp-c1-m1', 'object-6', 'lvm2'),
                       ('padawan-ccp-c1-m2', 'object-7', 'lvm0'),
                       ('padawan-ccp-c1-m2', 'object-8', 'lvm1'),
                       ('padawan-ccp-c1-m2', 'object-9', 'lvm3'),
                       ('padawan-ccp-c1-m2', 'object-10', 'lvm3')]
        for lv in set(lv_expected):
            self.assertTrue(lv in lv_info,
                            '%s missing from %s' % (lv, lv_info))
            lv_expected.remove(lv)
        self.assertEqual(0, len(lv_expected), 'still have %s' % lv_expected)
Esempio n. 5
0
 def load_model(self, data):
     staged_rings = data.get('delta_rings')
     for staged_ring in staged_rings:
         ring_name = staged_ring.get('ring_name')
         ring_specification = RingSpecification(None)
         ring_specification.load_model(
             staged_ring.get('ring_specification'))
         self.delta_rings[ring_name] = ring_specification
     stage_ring_actions = data.get('delta_ring_actions')
     for stage_ring_action in stage_ring_actions:
         ring_name = stage_ring_action.get('ring_name')
         action = stage_ring_action.get('action')
         self.delta_ring_actions[ring_name] = action
     for staged_device in data.get('delta_devices'):
         device = DeviceInfo()
         device.load_from_model(staged_device)
         self.delta_devices.append(device)
Esempio n. 6
0
    def _get_devs_from_builder(self, builder_filename):
        ring_name = builder_filename[0:builder_filename.find('.builder')]
        try:
            builder = SwiftRingBuilder.load(builder_filename)
        except Exception as e:
            raise IOError('ERROR: swift-ring-builder Problem occurred while '
                          'reading builder file: %s. %s' %
                          (builder_filename, e))

        self.partitions = builder.parts
        self.replica_count = builder.replicas
        balance = 0
        if builder.devs:
            balance = builder.get_balance()
        self.balance = balance
        self.dispersion = 0.00
        if builder.dispersion:
            self.dispersion = float(builder.dispersion)
        self.min_part_hours = builder.min_part_hours
        self.remaining = str(timedelta(seconds=builder.min_part_seconds_left))
        self.overload = builder.overload

        if builder.devs:
            balance_per_dev = builder._build_balance_per_dev()
            for dev in builder._iter_devs():
                if dev in builder._remove_devs:
                    # This device was added and then later removed without
                    # doing a rebalance in the meantime. We can ignore
                    # since it's marked for deletion.
                    continue
                device_info = DeviceInfo({
                    'ring_name':
                    ring_name,
                    'zone_id':
                    dev['zone'],
                    'region_id':
                    dev['region'],
                    'server_ip':
                    dev['ip'],
                    'server_bind_port':
                    dev['port'],
                    'replication_ip':
                    dev['replication_ip'],
                    'replication_bind_port':
                    dev['replication_port'],
                    'swift_drive_name':
                    dev['device'],
                    'current_weight':
                    dev['weight'],
                    'balance':
                    balance_per_dev[dev['id']],
                    'meta':
                    dev['meta'],
                    'presence':
                    'present'
                })
                yield device_info
Esempio n. 7
0
 def load_model(self, data):
     staged_rings = data.get('delta_rings')
     for staged_ring in staged_rings:
         region_name = staged_ring.get('region_name')
         ring_name = staged_ring.get('ring_name')
         ring_specification = RingSpecification(None)
         ring_specification.load_model(staged_ring.get(
             'ring_specification'))
         self.delta_rings[(region_name, ring_name)] = ring_specification
     stage_ring_actions = data.get('delta_ring_actions')
     for stage_ring_action in stage_ring_actions:
         region_name = stage_ring_action.get('region_name')
         ring_name = stage_ring_action.get('ring_name')
         action = stage_ring_action.get('action')
         self.delta_ring_actions[(region_name, ring_name)] = action
     for staged_device in data.get('delta_devices'):
         device = DeviceInfo()
         device.load_from_model(staged_device)
         self.delta_devices.append(device)
Esempio n. 8
0
    def test_iter_volume_groups(self):
        input_model = InputModel(config=safe_load(padawan_input_model),
                                 hosts_fd=padawan_net_hosts,
                                 consumes=padawan_swf_rng_consumes)
        lv_devices = []
        lv_expected = []
        for lv_device in input_model._iter_volume_groups():
            lv_devices.append(DeviceInfo(lv_device))
        self.assertEqual(len(lv_devices), 3)

        # This could be done with a simple assertEquals(), but this allowed
        # me to pin-point differences between actual and expected results
        for lv_device in expected_lv_devices:
            lv_expected.append(DeviceInfo(lv_device))
        lv_expected = sorted(lv_expected, None, DeviceInfo.sortkey)
        lv_devices = sorted(lv_devices, None, DeviceInfo.sortkey)
        for i in range(0, len(lv_expected)):
            self.assertEqual(sorted(lv_expected[i].keys()),
                             sorted(lv_devices[i].keys()))
            for key in lv_expected[i].keys():
                self.assertEqual(lv_expected[i].get(key),
                                 lv_devices[i].get(key))
Esempio n. 9
0
    def test_dev_numbering(self):
        input_model = InputModel(config=safe_load(device_groups_disk_model),
                                 hosts_fd=padawan_net_hosts,
                                 consumes=padawan_swf_rng_consumes)
        dev_devices = []
        dev_info = set()
        for dev_device in input_model._iter_device_groups():
            dev_devices.append(DeviceInfo(dev_device))
            dev_info.add((dev_device.server_name, dev_device.ring_name,
                          dev_device.swift_drive_name, dev_device.device_name))
        self.assertEqual(len(dev_devices), 21)

        # Do not change this without also examining test_drivedata.py
        dev_expected = [
            ('padawan-ccp-c1-m1', 'account', 'disk0', '/dev/sdb'),
            ('padawan-ccp-c1-m1', 'container', 'disk0', '/dev/sdb'),
            ('padawan-ccp-c1-m1', 'account', 'disk1', '/dev/sdc'),
            ('padawan-ccp-c1-m1', 'container', 'disk1', '/dev/sdc'),
            ('padawan-ccp-c1-m1', 'object-0', 'disk2', '/dev/sdd'),
            ('padawan-ccp-c1-m1', 'object-0', 'disk3', '/dev/sde'),
            ('padawan-ccp-c1-m1', 'object-0', 'disk4', '/dev/sdf'),
            ('padawan-ccp-c1-m1', 'object-1', 'disk5', '/dev/sdg'),
            ('padawan-ccp-c1-m1', 'object-1', 'disk6', '/dev/sdh'),
            ('padawan-ccp-c1-m1', 'object-1', 'disk7', '/dev/sdi'),
            ('padawan-ccp-c1-m1', 'object-1', 'disk11', '/dev/sdm'),
            ('padawan-ccp-c1-m1', 'object-0', 'disk11', '/dev/sdm'),
            ('padawan-ccp-c1-m1', 'object-1', 'disk12', '/dev/sdn'),
            ('padawan-ccp-c1-m1', 'object-0', 'disk12', '/dev/sdn'),
            ('padawan-ccp-c1-m1', 'object-1', 'disk13', '/dev/sdo'),
            ('padawan-ccp-c1-m1', 'object-0', 'disk13', '/dev/sdo'),
            ('padawan-ccp-c1-m2', 'account', 'disk0', '/dev/sda'),
            ('padawan-ccp-c1-m2', 'container', 'disk1', '/dev/sdb'),
            ('padawan-ccp-c1-m2', 'object-0', 'disk2', '/dev/sdc'),
            ('padawan-ccp-c1-m2', 'object-1', 'disk3', '/dev/sdd'),
            ('padawan-ccp-c1-m2', 'object-2', 'disk4', '/dev/sde')
        ]
        for dev in set(dev_expected):
            self.assertTrue(dev in dev_info,
                            '%s missing from %s' % (dev, dev_info))
            dev_info.remove(dev)
        self.assertEqual(0, len(dev_info), 'still have %s' % dev_info)
Esempio n. 10
0
 def _parse_builder_output(self, builder_filename):
     ring_name = builder_filename[0:builder_filename.find('.builder')]
     try:
         output = subprocess.check_output(['swift-ring-builder',
                                           '%s' % builder_filename])
     except subprocess.CalledProcessError as err:
         print('ERROR: swift-ring-builder %s: %s' % (builder_filename, err))
         sys.exit(1)
     done_with_headers = False
     for line in output.split('\n'):
         items = line.split()
         if not items:
             break
         if items[1] == 'partitions,':
             self.partitions = int(items[0])
             self.replica_count = float(items[2])
             self.balance = float(items[10])
         if items[0] == 'Devices:':
             done_with_headers = True
             continue
         if done_with_headers:
             (deviceid, region_id, zone_id, server, port, replicationip,
              replicationport, drive, weight, partitions, balance,
              meta) = items
             device_info = DeviceInfo({'ring_name': ring_name,
                                       'zone_id': zone_id,
                                       'region_id': region_id,
                                       'server_ip': server,
                                       'server_bind_port': port,
                                       'replication_ip': replicationip,
                                       'replication_bind_port':
                                       replicationport,
                                       'swift_drive_name': drive,
                                       'weight': weight,
                                       'balance': balance,
                                       'meta': meta,
                                       'presence': 'present'})
             yield device_info
Esempio n. 11
0
    def test_dev_numbering(self):
        input_model = ServersModel('my_cloud',
                                   'my_control_plane',
                                   config=safe_load(device_groups_disk_model),
                                   consumes_model=standard_swf_rng_consumes)
        dev_devices = []
        dev_info = set()
        for dev_device in input_model._iter_device_groups():
            dev_devices.append(DeviceInfo(dev_device))
            dev_info.add((dev_device.server_name, dev_device.server_ip,
                          dev_device.ring_name, dev_device.swift_drive_name,
                          dev_device.device_name))
        self.assertEqual(len(dev_devices), 21)

        # Do not change this without also examining test_drivedata.py
        dev_expected = [('standard-ccp-c1-m1-mgmt', '192.168.245.4', 'account',
                         'disk0', '/dev/sdb'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.245.4',
                         'container', 'disk0', '/dev/sdb'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.245.4', 'account',
                         'disk1', '/dev/sdc'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.245.4',
                         'container', 'disk1', '/dev/sdc'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.222.4',
                         'object-0', 'disk2', '/dev/sdd'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.222.4',
                         'object-0', 'disk3', '/dev/sde'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.222.4',
                         'object-0', 'disk4', '/dev/sdf'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.222.4',
                         'object-1', 'disk5', '/dev/sdg'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.222.4',
                         'object-1', 'disk6', '/dev/sdh'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.222.4',
                         'object-1', 'disk7', '/dev/sdi'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.222.4',
                         'object-1', 'disk8', '/dev/sdj'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.222.4',
                         'object-0', 'disk8', '/dev/sdj'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.222.4',
                         'object-1', 'disk9', '/dev/sdk'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.222.4',
                         'object-0', 'disk9', '/dev/sdk'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.222.4',
                         'object-1', 'disk10', '/dev/sdl'),
                        ('standard-ccp-c1-m1-mgmt', '192.168.222.4',
                         'object-0', 'disk10', '/dev/sdl'),
                        ('standard-ccp-c1-m2-mgmt', '192.168.245.3', 'account',
                         'disk0', '/dev/sda'),
                        ('standard-ccp-c1-m2-mgmt', '192.168.245.3',
                         'container', 'disk1', '/dev/sdb'),
                        ('standard-ccp-c1-m2-mgmt', '192.168.222.3',
                         'object-0', 'disk2', '/dev/sdc'),
                        ('standard-ccp-c1-m2-mgmt', '192.168.222.3',
                         'object-1', 'disk3', '/dev/sdd'),
                        ('standard-ccp-c1-m2-mgmt', '192.168.222.3',
                         'object-2', 'disk4', '/dev/sde')]
        for dev in set(dev_expected):
            self.assertTrue(dev in dev_info,
                            '%s missing from %s' % (dev, dev_info))
            dev_expected.remove(dev)
        self.assertEqual(0, len(dev_expected), 'still have %s' % dev_expected)
Esempio n. 12
0
 def _iter_volume_groups(self):
     for server in self.config.get('global').get('all_servers'):
         server_name = server.get('name')
         disk_model = server.get('disk_model')
         region_name = server.get('region')
         rack_id = server.get('rack')
         lv_index = 0
         for volume_group in disk_model.get('volume_groups', []):
             vg_name = volume_group.get('name')
             physical_volumes = volume_group.get('physical_volumes')
             for logical_volume in volume_group.get('logical_volumes', []):
                 lv_name = logical_volume.get('name')
                 percent = logical_volume.get('size')
                 consumer = logical_volume.get('consumer')
                 if consumer and consumer.get('name', 'other') == 'swift':
                     attrs = consumer.get('attrs')
                     if not attrs:
                         raise SwiftModelException('The attrs item is'
                                                   ' missing from '
                                                   ' logical volume'
                                                   ' %s in disk model %s' %
                                                   (logical_volume.get(
                                                       'name'),
                                                    disk_model.get('name')))
                     if attrs.get('rings', None) is None:
                         raise SwiftModelException('The rings item is'
                                                   ' missing from logical'
                                                   ' volume'
                                                   ' %s in disk model %s' %
                                                   (logical_volume.get(
                                                       'name'),
                                                    disk_model.get('name')))
                     for ring in attrs.get('rings'):
                         if isinstance(ring, str):
                             ring_name = ring
                         else:
                             ring_name = ring.get('name')
                         server_ip, bind_port = self._get_server_bind(
                             ring_name, server_name)
                         if not server_ip:
                             # TODO: this may be worth warning
                             break
                         swift_drive_name = LVM_MOUNT + str(lv_index)
                         device_name = '/dev/' + vg_name + '/' + lv_name
                         device_info = DeviceInfo({
                             'region_name': region_name,
                             'rack_id': rack_id,
                             'region_id': 1,    # later, the rack id may
                             'zone_id': 1,      # change these defaults
                             'server_name': Consumes.basename(
                                 server_name),
                             'server_ip': server_ip,
                             'server_bind_port': bind_port,
                             'replication_ip': server_ip,
                             'replication_bind_port': bind_port,
                             'swift_drive_name': swift_drive_name,
                             'device_name': device_name,
                             'ring_name': ring_name,
                             'group_type': 'lvm',
                             'block_devices': {'percent': percent,
                                               'physicals':
                                                   physical_volumes},
                             'presence': 'present'})
                         yield device_info
                     lv_index += 1
Esempio n. 13
0
    def _iter_device_groups(self):
        for server in self.config.get('global').get('all_servers'):
            server_name = server.get('name')
            disk_model = server.get('disk_model')
            region_name = server.get('region')
            rack_id = server.get('rack')
            device_index = 0
            for device_group in disk_model.get('device_groups', []):
                consumer = device_group.get('consumer')
                if consumer and consumer.get('name', 'other') == 'swift':
                    attrs = consumer.get('attrs')
                    if not attrs:
                        raise SwiftModelException('The attrs item is'
                                                  ' missing from device-groups'
                                                  ' %s in disk model %s' %
                                                  (device_group.get('name'),
                                                   disk_model.get('name')))
                    devices = device_group.get('devices')
                    if attrs.get('rings', None) is None:
                        raise SwiftModelException('The rings item is'
                                                  ' missing from device-groups'
                                                  ' %s in disk model %s' %
                                                  (device_group.get('name'),
                                                   disk_model.get('name')))
                    for device in devices:
                        for ring in attrs.get('rings'):
                            if isinstance(ring, str):
                                ring_name = ring
                            else:
                                ring_name = ring.get('name')
                            server_ip, bind_port = self._get_server_bind(
                                ring_name, server_name)
                            if not server_ip:
                                # When a swift service (example swift-account)
                                # is configured in the input model to run a
                                # node, we expect the node to be in the
                                # "consumes" variable. e.g., consumes_SWF_ACC
                                # should have this node in its list. Since we
                                # failed to get the network name/port, it means
                                # that it is not.
                                # In model terms, we have a disk model that
                                # calls out that a device hosts a ring (e.g.
                                # account), but the node is not configured
                                # to run SWF-ACC.
                                # TODO: this may be worth warning
                                break

                            swift_drive_name = DISK_MOUNT + str(device_index)
                            device_info = DeviceInfo({
                                'region_name': region_name,
                                'rack_id': rack_id,
                                'region_id': 1,      # later, the rack id may
                                'zone_id': 1,        # change these defaults
                                'server_name': Consumes.basename(server_name),
                                'server_ip': server_ip,
                                'server_bind_port': bind_port,
                                'replication_ip': server_ip,
                                'replication_bind_port': bind_port,
                                'swift_drive_name': swift_drive_name,
                                'device_name': device.get('name'),
                                'ring_name': ring_name,
                                'group_type': 'device',
                                'block_devices': {'percent': '100%',
                                                  'physicals':
                                                      [device.get('name')]},
                                'presence': 'present'})

                            yield device_info
                        device_index += 1
Esempio n. 14
0
 def _iter_volume_groups(self):
     for server in self.servers:
         server_name = server.get('ardana_ansible_host', server.get('name'))
         cloud = server.get('cloud')
         control_plane = server.get('control_plane')
         network_names = server.get('network_names')
         disk_model = server.get('disk_model')
         server_groups = server.get('server_group_list', [])
         lv_index = 0
         for volume_group in disk_model.get('volume_groups', []):
             vg_name = volume_group.get('name')
             physical_volumes = volume_group.get('physical_volumes')
             for logical_volume in volume_group.get('logical_volumes', []):
                 lv_name = logical_volume.get('name')
                 percent = logical_volume.get('size')
                 consumer = logical_volume.get('consumer')
                 if consumer and consumer.get('name', 'other') == 'swift':
                     attrs = consumer.get('attrs')
                     if not attrs:
                         raise SwiftModelException(
                             'The attrs item is'
                             ' missing from '
                             ' logical volume'
                             ' %s in disk model %s' %
                             (logical_volume.get('name'),
                              disk_model.get('name')))
                     if not attrs.get('rings'):
                         raise SwiftModelException(
                             'The rings item is'
                             ' missing from logical'
                             ' volume'
                             ' %s in disk model %s' %
                             (logical_volume.get('name'),
                              disk_model.get('name')))
                     for ring in attrs.get('rings'):
                         if isinstance(ring, str):
                             ring_name = ring
                         else:
                             ring_name = ring.get('name')
                         server_ip, bind_port = self._get_server_bind(
                             ring_name, network_names)
                         if not server_ip:
                             # TODO: this may be worth warning
                             break
                         swift_drive_name = LVM_MOUNT + str(lv_index)
                         device_name = '/dev/' + vg_name + '/' + lv_name
                         device_info = DeviceInfo({
                             'cloud': cloud,
                             'control_plane': control_plane,
                             'server_groups': server_groups,
                             'region_id': 1,  # later, the server group may
                             'zone_id': 1,  # change these defaults
                             'server_name': server_name,
                             'network_names': network_names,
                             'server_ip': server_ip,
                             'server_bind_port': bind_port,
                             'replication_ip': server_ip,
                             'replication_bind_port': bind_port,
                             'swift_drive_name': swift_drive_name,
                             'device_name': device_name,
                             'ring_name': ring_name,
                             'group_type': 'lvm',
                             'block_devices': {
                                 'percent': percent,
                                 'physicals': physical_volumes
                             },
                             'presence': 'present'
                         })
                         yield device_info
                     lv_index += 1