def test_simple_device_info(self):
device_info = DeviceInfo(model=safe_load(device_info_simple))
self.assertEquals(device_info.region_name, 'region2')
self.assertEquals(device_info.region_id, '2')
self.assertEquals(device_info.replication_ip, None)
self.assertEquals(device_info.presence, 'present')
self.assertEquals(device_info.meta, 'host.example.com:disk99:/dev/sdh')
with self.assertRaises(AttributeError):
_ = device_info.junk
with self.assertRaises(AttributeError):
device_info.junk = 'junk'
def test_simple_device_info(self):
device_info = DeviceInfo(model=safe_load(device_info_simple))
self.assertEquals(device_info.region_name, 'region2')
self.assertEquals(device_info.region_id, 2)
self.assertEquals(device_info.replication_ip, None)
self.assertEquals(device_info.presence, 'present')
self.assertEquals(device_info.meta, 'host.example.com:disk99:/dev/sdh')
with self.assertRaises(AttributeError):
_ = device_info.junk
with self.assertRaises(AttributeError):
device_info.junk = 'junk'
def test_lvm_numbering(self):
input_model = ServersModel('my_cloud',
'my_control_plane',
config=safe_load(lvm_disk_model),
consumes_model=standard_swf_rng_consumes)
lv_devices = []
lv_info = set()
for lv_device in input_model._iter_volume_groups():
lv_devices.append(DeviceInfo(lv_device))
lv_info.add((lv_device.server_name, lv_device.server_ip,
lv_device.ring_name, lv_device.swift_drive_name))
self.assertEqual(len(lv_devices), 11)
# Do not change this without also examining test_drivedata.py
lv_expected = [
('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-0', 'lvm0'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-1', 'lvm0'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-2', 'lvm1'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-3', 'lvm1'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-4', 'lvm2'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-5', 'lvm2'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4', 'object-6', 'lvm2'),
('standard-ccp-c1-m2-mgmt', '192.168.222.3', 'object-7', 'lvm0'),
('standard-ccp-c1-m2-mgmt', '192.168.222.3', 'object-8', 'lvm1'),
('standard-ccp-c1-m2-mgmt', '192.168.222.3', 'object-9', 'lvm2'),
('standard-ccp-c1-m2-mgmt', '192.168.222.3', 'object-10', 'lvm2')
]
for lv in set(lv_expected):
self.assertTrue(lv in lv_info,
'%s missing from %s' % (lv, lv_info))
lv_expected.remove(lv)
self.assertEqual(0, len(lv_expected), 'still have %s' % lv_expected)
def test_lvm_numbering(self):
input_model = InputModel(config=safe_load(lvm_disk_model),
hosts_fd=padawan_net_hosts,
consumes=padawan_swf_rng_consumes)
lv_devices = []
lv_info = set()
for lv_device in input_model._iter_volume_groups():
lv_devices.append(DeviceInfo(lv_device))
lv_info.add((lv_device.server_name, lv_device.ring_name,
lv_device.swift_drive_name))
self.assertEqual(len(lv_devices), 11)
# Do not change this without also examining test_drivedata.py
lv_expected = [('padawan-ccp-c1-m1', 'object-0', 'lvm0'),
('padawan-ccp-c1-m1', 'object-1', 'lvm0'),
('padawan-ccp-c1-m1', 'object-2', 'lvm1'),
('padawan-ccp-c1-m1', 'object-3', 'lvm1'),
('padawan-ccp-c1-m1', 'object-4', 'lvm2'),
('padawan-ccp-c1-m1', 'object-5', 'lvm2'),
('padawan-ccp-c1-m1', 'object-6', 'lvm2'),
('padawan-ccp-c1-m2', 'object-7', 'lvm0'),
('padawan-ccp-c1-m2', 'object-8', 'lvm1'),
('padawan-ccp-c1-m2', 'object-9', 'lvm3'),
('padawan-ccp-c1-m2', 'object-10', 'lvm3')]
for lv in set(lv_expected):
self.assertTrue(lv in lv_info,
'%s missing from %s' % (lv, lv_info))
lv_expected.remove(lv)
self.assertEqual(0, len(lv_expected), 'still have %s' % lv_expected)
def load_model(self, data):
staged_rings = data.get('delta_rings')
for staged_ring in staged_rings:
ring_name = staged_ring.get('ring_name')
ring_specification = RingSpecification(None)
ring_specification.load_model(
staged_ring.get('ring_specification'))
self.delta_rings[ring_name] = ring_specification
stage_ring_actions = data.get('delta_ring_actions')
for stage_ring_action in stage_ring_actions:
ring_name = stage_ring_action.get('ring_name')
action = stage_ring_action.get('action')
self.delta_ring_actions[ring_name] = action
for staged_device in data.get('delta_devices'):
device = DeviceInfo()
device.load_from_model(staged_device)
self.delta_devices.append(device)
def _get_devs_from_builder(self, builder_filename):
ring_name = builder_filename[0:builder_filename.find('.builder')]
try:
builder = SwiftRingBuilder.load(builder_filename)
except Exception as e:
raise IOError('ERROR: swift-ring-builder Problem occurred while '
'reading builder file: %s. %s' %
(builder_filename, e))
self.partitions = builder.parts
self.replica_count = builder.replicas
balance = 0
if builder.devs:
balance = builder.get_balance()
self.balance = balance
self.dispersion = 0.00
if builder.dispersion:
self.dispersion = float(builder.dispersion)
self.min_part_hours = builder.min_part_hours
self.remaining = str(timedelta(seconds=builder.min_part_seconds_left))
self.overload = builder.overload
if builder.devs:
balance_per_dev = builder._build_balance_per_dev()
for dev in builder._iter_devs():
if dev in builder._remove_devs:
# This device was added and then later removed without
# doing a rebalance in the meantime. We can ignore
# since it's marked for deletion.
continue
device_info = DeviceInfo({
'ring_name':
ring_name,
'zone_id':
dev['zone'],
'region_id':
dev['region'],
'server_ip':
dev['ip'],
'server_bind_port':
dev['port'],
'replication_ip':
dev['replication_ip'],
'replication_bind_port':
dev['replication_port'],
'swift_drive_name':
dev['device'],
'current_weight':
dev['weight'],
'balance':
balance_per_dev[dev['id']],
'meta':
dev['meta'],
'presence':
'present'
})
yield device_info
def load_model(self, data):
staged_rings = data.get('delta_rings')
for staged_ring in staged_rings:
region_name = staged_ring.get('region_name')
ring_name = staged_ring.get('ring_name')
ring_specification = RingSpecification(None)
ring_specification.load_model(staged_ring.get(
'ring_specification'))
self.delta_rings[(region_name, ring_name)] = ring_specification
stage_ring_actions = data.get('delta_ring_actions')
for stage_ring_action in stage_ring_actions:
region_name = stage_ring_action.get('region_name')
ring_name = stage_ring_action.get('ring_name')
action = stage_ring_action.get('action')
self.delta_ring_actions[(region_name, ring_name)] = action
for staged_device in data.get('delta_devices'):
device = DeviceInfo()
device.load_from_model(staged_device)
self.delta_devices.append(device)
def test_iter_volume_groups(self):
input_model = InputModel(config=safe_load(padawan_input_model),
hosts_fd=padawan_net_hosts,
consumes=padawan_swf_rng_consumes)
lv_devices = []
lv_expected = []
for lv_device in input_model._iter_volume_groups():
lv_devices.append(DeviceInfo(lv_device))
self.assertEqual(len(lv_devices), 3)
# This could be done with a simple assertEquals(), but this allowed
# me to pin-point differences between actual and expected results
for lv_device in expected_lv_devices:
lv_expected.append(DeviceInfo(lv_device))
lv_expected = sorted(lv_expected, None, DeviceInfo.sortkey)
lv_devices = sorted(lv_devices, None, DeviceInfo.sortkey)
for i in range(0, len(lv_expected)):
self.assertEqual(sorted(lv_expected[i].keys()),
sorted(lv_devices[i].keys()))
for key in lv_expected[i].keys():
self.assertEqual(lv_expected[i].get(key),
lv_devices[i].get(key))
def test_dev_numbering(self):
input_model = InputModel(config=safe_load(device_groups_disk_model),
hosts_fd=padawan_net_hosts,
consumes=padawan_swf_rng_consumes)
dev_devices = []
dev_info = set()
for dev_device in input_model._iter_device_groups():
dev_devices.append(DeviceInfo(dev_device))
dev_info.add((dev_device.server_name, dev_device.ring_name,
dev_device.swift_drive_name, dev_device.device_name))
self.assertEqual(len(dev_devices), 21)
# Do not change this without also examining test_drivedata.py
dev_expected = [
('padawan-ccp-c1-m1', 'account', 'disk0', '/dev/sdb'),
('padawan-ccp-c1-m1', 'container', 'disk0', '/dev/sdb'),
('padawan-ccp-c1-m1', 'account', 'disk1', '/dev/sdc'),
('padawan-ccp-c1-m1', 'container', 'disk1', '/dev/sdc'),
('padawan-ccp-c1-m1', 'object-0', 'disk2', '/dev/sdd'),
('padawan-ccp-c1-m1', 'object-0', 'disk3', '/dev/sde'),
('padawan-ccp-c1-m1', 'object-0', 'disk4', '/dev/sdf'),
('padawan-ccp-c1-m1', 'object-1', 'disk5', '/dev/sdg'),
('padawan-ccp-c1-m1', 'object-1', 'disk6', '/dev/sdh'),
('padawan-ccp-c1-m1', 'object-1', 'disk7', '/dev/sdi'),
('padawan-ccp-c1-m1', 'object-1', 'disk11', '/dev/sdm'),
('padawan-ccp-c1-m1', 'object-0', 'disk11', '/dev/sdm'),
('padawan-ccp-c1-m1', 'object-1', 'disk12', '/dev/sdn'),
('padawan-ccp-c1-m1', 'object-0', 'disk12', '/dev/sdn'),
('padawan-ccp-c1-m1', 'object-1', 'disk13', '/dev/sdo'),
('padawan-ccp-c1-m1', 'object-0', 'disk13', '/dev/sdo'),
('padawan-ccp-c1-m2', 'account', 'disk0', '/dev/sda'),
('padawan-ccp-c1-m2', 'container', 'disk1', '/dev/sdb'),
('padawan-ccp-c1-m2', 'object-0', 'disk2', '/dev/sdc'),
('padawan-ccp-c1-m2', 'object-1', 'disk3', '/dev/sdd'),
('padawan-ccp-c1-m2', 'object-2', 'disk4', '/dev/sde')
]
for dev in set(dev_expected):
self.assertTrue(dev in dev_info,
'%s missing from %s' % (dev, dev_info))
dev_info.remove(dev)
self.assertEqual(0, len(dev_info), 'still have %s' % dev_info)
def _parse_builder_output(self, builder_filename):
ring_name = builder_filename[0:builder_filename.find('.builder')]
try:
output = subprocess.check_output(['swift-ring-builder',
'%s' % builder_filename])
except subprocess.CalledProcessError as err:
print('ERROR: swift-ring-builder %s: %s' % (builder_filename, err))
sys.exit(1)
done_with_headers = False
for line in output.split('\n'):
items = line.split()
if not items:
break
if items[1] == 'partitions,':
self.partitions = int(items[0])
self.replica_count = float(items[2])
self.balance = float(items[10])
if items[0] == 'Devices:':
done_with_headers = True
continue
if done_with_headers:
(deviceid, region_id, zone_id, server, port, replicationip,
replicationport, drive, weight, partitions, balance,
meta) = items
device_info = DeviceInfo({'ring_name': ring_name,
'zone_id': zone_id,
'region_id': region_id,
'server_ip': server,
'server_bind_port': port,
'replication_ip': replicationip,
'replication_bind_port':
replicationport,
'swift_drive_name': drive,
'weight': weight,
'balance': balance,
'meta': meta,
'presence': 'present'})
yield device_info
def test_dev_numbering(self):
input_model = ServersModel('my_cloud',
'my_control_plane',
config=safe_load(device_groups_disk_model),
consumes_model=standard_swf_rng_consumes)
dev_devices = []
dev_info = set()
for dev_device in input_model._iter_device_groups():
dev_devices.append(DeviceInfo(dev_device))
dev_info.add((dev_device.server_name, dev_device.server_ip,
dev_device.ring_name, dev_device.swift_drive_name,
dev_device.device_name))
self.assertEqual(len(dev_devices), 21)
# Do not change this without also examining test_drivedata.py
dev_expected = [('standard-ccp-c1-m1-mgmt', '192.168.245.4', 'account',
'disk0', '/dev/sdb'),
('standard-ccp-c1-m1-mgmt', '192.168.245.4',
'container', 'disk0', '/dev/sdb'),
('standard-ccp-c1-m1-mgmt', '192.168.245.4', 'account',
'disk1', '/dev/sdc'),
('standard-ccp-c1-m1-mgmt', '192.168.245.4',
'container', 'disk1', '/dev/sdc'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4',
'object-0', 'disk2', '/dev/sdd'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4',
'object-0', 'disk3', '/dev/sde'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4',
'object-0', 'disk4', '/dev/sdf'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4',
'object-1', 'disk5', '/dev/sdg'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4',
'object-1', 'disk6', '/dev/sdh'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4',
'object-1', 'disk7', '/dev/sdi'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4',
'object-1', 'disk8', '/dev/sdj'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4',
'object-0', 'disk8', '/dev/sdj'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4',
'object-1', 'disk9', '/dev/sdk'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4',
'object-0', 'disk9', '/dev/sdk'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4',
'object-1', 'disk10', '/dev/sdl'),
('standard-ccp-c1-m1-mgmt', '192.168.222.4',
'object-0', 'disk10', '/dev/sdl'),
('standard-ccp-c1-m2-mgmt', '192.168.245.3', 'account',
'disk0', '/dev/sda'),
('standard-ccp-c1-m2-mgmt', '192.168.245.3',
'container', 'disk1', '/dev/sdb'),
('standard-ccp-c1-m2-mgmt', '192.168.222.3',
'object-0', 'disk2', '/dev/sdc'),
('standard-ccp-c1-m2-mgmt', '192.168.222.3',
'object-1', 'disk3', '/dev/sdd'),
('standard-ccp-c1-m2-mgmt', '192.168.222.3',
'object-2', 'disk4', '/dev/sde')]
for dev in set(dev_expected):
self.assertTrue(dev in dev_info,
'%s missing from %s' % (dev, dev_info))
dev_expected.remove(dev)
self.assertEqual(0, len(dev_expected), 'still have %s' % dev_expected)
def _iter_volume_groups(self):
for server in self.config.get('global').get('all_servers'):
server_name = server.get('name')
disk_model = server.get('disk_model')
region_name = server.get('region')
rack_id = server.get('rack')
lv_index = 0
for volume_group in disk_model.get('volume_groups', []):
vg_name = volume_group.get('name')
physical_volumes = volume_group.get('physical_volumes')
for logical_volume in volume_group.get('logical_volumes', []):
lv_name = logical_volume.get('name')
percent = logical_volume.get('size')
consumer = logical_volume.get('consumer')
if consumer and consumer.get('name', 'other') == 'swift':
attrs = consumer.get('attrs')
if not attrs:
raise SwiftModelException('The attrs item is'
' missing from '
' logical volume'
' %s in disk model %s' %
(logical_volume.get(
'name'),
disk_model.get('name')))
if attrs.get('rings', None) is None:
raise SwiftModelException('The rings item is'
' missing from logical'
' volume'
' %s in disk model %s' %
(logical_volume.get(
'name'),
disk_model.get('name')))
for ring in attrs.get('rings'):
if isinstance(ring, str):
ring_name = ring
else:
ring_name = ring.get('name')
server_ip, bind_port = self._get_server_bind(
ring_name, server_name)
if not server_ip:
# TODO: this may be worth warning
break
swift_drive_name = LVM_MOUNT + str(lv_index)
device_name = '/dev/' + vg_name + '/' + lv_name
device_info = DeviceInfo({
'region_name': region_name,
'rack_id': rack_id,
'region_id': 1, # later, the rack id may
'zone_id': 1, # change these defaults
'server_name': Consumes.basename(
server_name),
'server_ip': server_ip,
'server_bind_port': bind_port,
'replication_ip': server_ip,
'replication_bind_port': bind_port,
'swift_drive_name': swift_drive_name,
'device_name': device_name,
'ring_name': ring_name,
'group_type': 'lvm',
'block_devices': {'percent': percent,
'physicals':
physical_volumes},
'presence': 'present'})
yield device_info
lv_index += 1
def _iter_device_groups(self):
for server in self.config.get('global').get('all_servers'):
server_name = server.get('name')
disk_model = server.get('disk_model')
region_name = server.get('region')
rack_id = server.get('rack')
device_index = 0
for device_group in disk_model.get('device_groups', []):
consumer = device_group.get('consumer')
if consumer and consumer.get('name', 'other') == 'swift':
attrs = consumer.get('attrs')
if not attrs:
raise SwiftModelException('The attrs item is'
' missing from device-groups'
' %s in disk model %s' %
(device_group.get('name'),
disk_model.get('name')))
devices = device_group.get('devices')
if attrs.get('rings', None) is None:
raise SwiftModelException('The rings item is'
' missing from device-groups'
' %s in disk model %s' %
(device_group.get('name'),
disk_model.get('name')))
for device in devices:
for ring in attrs.get('rings'):
if isinstance(ring, str):
ring_name = ring
else:
ring_name = ring.get('name')
server_ip, bind_port = self._get_server_bind(
ring_name, server_name)
if not server_ip:
# When a swift service (example swift-account)
# is configured in the input model to run a
# node, we expect the node to be in the
# "consumes" variable. e.g., consumes_SWF_ACC
# should have this node in its list. Since we
# failed to get the network name/port, it means
# that it is not.
# In model terms, we have a disk model that
# calls out that a device hosts a ring (e.g.
# account), but the node is not configured
# to run SWF-ACC.
# TODO: this may be worth warning
break
swift_drive_name = DISK_MOUNT + str(device_index)
device_info = DeviceInfo({
'region_name': region_name,
'rack_id': rack_id,
'region_id': 1, # later, the rack id may
'zone_id': 1, # change these defaults
'server_name': Consumes.basename(server_name),
'server_ip': server_ip,
'server_bind_port': bind_port,
'replication_ip': server_ip,
'replication_bind_port': bind_port,
'swift_drive_name': swift_drive_name,
'device_name': device.get('name'),
'ring_name': ring_name,
'group_type': 'device',
'block_devices': {'percent': '100%',
'physicals':
[device.get('name')]},
'presence': 'present'})
yield device_info
device_index += 1
def _iter_volume_groups(self):
for server in self.servers:
server_name = server.get('ardana_ansible_host', server.get('name'))
cloud = server.get('cloud')
control_plane = server.get('control_plane')
network_names = server.get('network_names')
disk_model = server.get('disk_model')
server_groups = server.get('server_group_list', [])
lv_index = 0
for volume_group in disk_model.get('volume_groups', []):
vg_name = volume_group.get('name')
physical_volumes = volume_group.get('physical_volumes')
for logical_volume in volume_group.get('logical_volumes', []):
lv_name = logical_volume.get('name')
percent = logical_volume.get('size')
consumer = logical_volume.get('consumer')
if consumer and consumer.get('name', 'other') == 'swift':
attrs = consumer.get('attrs')
if not attrs:
raise SwiftModelException(
'The attrs item is'
' missing from '
' logical volume'
' %s in disk model %s' %
(logical_volume.get('name'),
disk_model.get('name')))
if not attrs.get('rings'):
raise SwiftModelException(
'The rings item is'
' missing from logical'
' volume'
' %s in disk model %s' %
(logical_volume.get('name'),
disk_model.get('name')))
for ring in attrs.get('rings'):
if isinstance(ring, str):
ring_name = ring
else:
ring_name = ring.get('name')
server_ip, bind_port = self._get_server_bind(
ring_name, network_names)
if not server_ip:
# TODO: this may be worth warning
break
swift_drive_name = LVM_MOUNT + str(lv_index)
device_name = '/dev/' + vg_name + '/' + lv_name
device_info = DeviceInfo({
'cloud': cloud,
'control_plane': control_plane,
'server_groups': server_groups,
'region_id': 1, # later, the server group may
'zone_id': 1, # change these defaults
'server_name': server_name,
'network_names': network_names,
'server_ip': server_ip,
'server_bind_port': bind_port,
'replication_ip': server_ip,
'replication_bind_port': bind_port,
'swift_drive_name': swift_drive_name,
'device_name': device_name,
'ring_name': ring_name,
'group_type': 'lvm',
'block_devices': {
'percent': percent,
'physicals': physical_volumes
},
'presence': 'present'
})
yield device_info
lv_index += 1