def __init__(self, fd=None, config=None, hosts_fd=None, consumes=None): self.config = {} self.servers = [] self.host_ip_map = {} if fd: self._read_config(fd) if config: self.config = config self.servers = self.config.get('global').get('all_servers') self.devices = [] if hosts_fd: self._load_host_ip_mapping(hosts_fd) if consumes: self.consumes = Consumes(consumes)
def __init__(self, cloud, control_plane, config=None, consumes_model=None): ''' :param cloud: used in unit tests :param control_plane: used in unit tests :param config: used in unit tests :param consumes_model: used in unit tests :return: ''' self.servers = [] self.consumes = None if consumes_model: # Used by unit tests self.register_consumes(Consumes(consumes_model)) # Unit tests can load up a single site here; supervisor uses # add_servers(). if config: servers = [] if config.get('control_plane_servers'): servers = config.get('control_plane_servers') elif config.get('global'): if config.get('global').get('all_servers'): servers = config.get('global').get('all_servers') self.add_servers(cloud, control_plane, servers)
def aliases(self, ip_address): """ Given an ip address, return a list of alternate ip addresses :param ip_address: The ip address to lookup. :return: list of ip addresses """ aliases = [] network_name = self.host_ip_map.get(ip_address) if not network_name: return [] basename = Consumes.basename(network_name) for item in self.host_ip_map.keys(): item_name = self.host_ip_map.get(item) if item_name: item_basename = Consumes.basename(item_name) if item_basename == basename: aliases.append(item) return aliases
class InputModel(object): """ Access the drive data in the input model The input model looks like: global: all_servers: - name: server1 disk_model: device_groups: - consumer: name: swift attrs: rings: - object-0 # Allows raw name or - name: object-0 # name as key (future # extension devices: - /dev/sda - /dev/sdb - .etc... - consumer: name: <something-else> # We ignore attrs: devices: The main output is from iter_devices(). This returns a list of all swift drives with following items: region_name the Keystone region name, i.e., which system the device refers to. (e.g., region1) region_id The swift region id (e.g., 1) zone_id The swift zone id (e.g., 2) server_name The name of the server server_ip The IP address of the server server_bind_port: The port number to use (e.g., 6000) replication_ip The IP address of the drive on the replication network. Or None if no replication network replication_bind_port Port to use if a replication network is used swift_drive_name Name used in ring files (e.g. swdisk1) device_name Dhe name of the device (e.g., /dev/sdb) ring_name The ringname (e.g., object-1) group_type 'device' or 'volume' presence: Currently always 'present' (reserved for future use -- would allow controlled removal of drives or server from an input model) """ def __init__(self, fd=None, config=None, hosts_fd=None, consumes=None): self.config = {} self.servers = [] self.host_ip_map = {} if fd: self._read_config(fd) if config: self.config = config self.servers = self.config.get('global').get('all_servers') self.devices = [] if hosts_fd: self._load_host_ip_mapping(hosts_fd) if consumes: self.consumes = Consumes(consumes) def iter_devices(self): for device_info in self._iter_device_groups(): yield device_info for device_info in self._iter_volume_groups(): yield device_info def _iter_device_groups(self): for server in self.config.get('global').get('all_servers'): server_name = server.get('name') disk_model = server.get('disk_model') region_name = server.get('region') rack_id = server.get('rack') device_index = 0 for device_group in disk_model.get('device_groups', []): consumer = device_group.get('consumer') if consumer and consumer.get('name', 'other') == 'swift': attrs = consumer.get('attrs') if not attrs: raise SwiftModelException('The attrs item is' ' missing from device-groups' ' %s in disk model %s' % (device_group.get('name'), disk_model.get('name'))) devices = device_group.get('devices') if attrs.get('rings', None) is None: raise SwiftModelException('The rings item is' ' missing from device-groups' ' %s in disk model %s' % (device_group.get('name'), disk_model.get('name'))) for device in devices: for ring in attrs.get('rings'): if isinstance(ring, str): ring_name = ring else: ring_name = ring.get('name') server_ip, bind_port = self._get_server_bind( ring_name, server_name) if not server_ip: # When a swift service (example swift-account) # is configured in the input model to run a # node, we expect the node to be in the # "consumes" variable. e.g., consumes_SWF_ACC # should have this node in its list. Since we # failed to get the network name/port, it means # that it is not. # In model terms, we have a disk model that # calls out that a device hosts a ring (e.g. # account), but the node is not configured # to run SWF-ACC. # TODO: this may be worth warning break swift_drive_name = DISK_MOUNT + str(device_index) device_info = DeviceInfo({ 'region_name': region_name, 'rack_id': rack_id, 'region_id': 1, # later, the rack id may 'zone_id': 1, # change these defaults 'server_name': Consumes.basename(server_name), 'server_ip': server_ip, 'server_bind_port': bind_port, 'replication_ip': server_ip, 'replication_bind_port': bind_port, 'swift_drive_name': swift_drive_name, 'device_name': device.get('name'), 'ring_name': ring_name, 'group_type': 'device', 'block_devices': {'percent': '100%', 'physicals': [device.get('name')]}, 'presence': 'present'}) yield device_info device_index += 1 def _iter_volume_groups(self): for server in self.config.get('global').get('all_servers'): server_name = server.get('name') disk_model = server.get('disk_model') region_name = server.get('region') rack_id = server.get('rack') lv_index = 0 for volume_group in disk_model.get('volume_groups', []): vg_name = volume_group.get('name') physical_volumes = volume_group.get('physical_volumes') for logical_volume in volume_group.get('logical_volumes', []): lv_name = logical_volume.get('name') percent = logical_volume.get('size') consumer = logical_volume.get('consumer') if consumer and consumer.get('name', 'other') == 'swift': attrs = consumer.get('attrs') if not attrs: raise SwiftModelException('The attrs item is' ' missing from ' ' logical volume' ' %s in disk model %s' % (logical_volume.get( 'name'), disk_model.get('name'))) if attrs.get('rings', None) is None: raise SwiftModelException('The rings item is' ' missing from logical' ' volume' ' %s in disk model %s' % (logical_volume.get( 'name'), disk_model.get('name'))) for ring in attrs.get('rings'): if isinstance(ring, str): ring_name = ring else: ring_name = ring.get('name') server_ip, bind_port = self._get_server_bind( ring_name, server_name) if not server_ip: # TODO: this may be worth warning break swift_drive_name = LVM_MOUNT + str(lv_index) device_name = '/dev/' + vg_name + '/' + lv_name device_info = DeviceInfo({ 'region_name': region_name, 'rack_id': rack_id, 'region_id': 1, # later, the rack id may 'zone_id': 1, # change these defaults 'server_name': Consumes.basename( server_name), 'server_ip': server_ip, 'server_bind_port': bind_port, 'replication_ip': server_ip, 'replication_bind_port': bind_port, 'swift_drive_name': swift_drive_name, 'device_name': device_name, 'ring_name': ring_name, 'group_type': 'lvm', 'block_devices': {'percent': percent, 'physicals': physical_volumes}, 'presence': 'present'}) yield device_info lv_index += 1 def _get_server_bind(self, ring_name, server_name): network_name, network_port = self.consumes.get_network_name_port( ring_name, server_name) if not network_name: return None, None server_ip = self.ip_address(network_name, 'ring: %s host: %s' % ( ring_name, server_name)) return (server_ip, network_port) def __repr__(self): output = '\nInput Model\n' output += '-----------\n\n' output += '\n Servers\n' output += ' Number: %s' % len(self.config.get('global').get( 'all_servers')) output += '\n Device Information\n' for di in self.iter_devices(): output += '\n device info: %s' % di return output def _read_config(self, fd): self.config = safe_load(fd) def _load_host_ip_mapping(self, hosts_fd): """ Load host to IP mapping data For now, this is simply a hosts file (/etc/hosts or .net/hosts.hf). Later, this will load the HLM 2.0 model. """ for line in hosts_fd: line = line.strip() try: if line.startswith('#'): continue words = line.split() ip_address = words[0] hostname = words[1] self.host_ip_map[ip_address] = hostname self.host_ip_map[hostname] = ip_address except Exception: continue # Ignore all parsing errors def ip_address(self, hostname, context): """ Returns IP address of host from hosts file """ ip_address = self.host_ip_map.get(hostname) if not ip_address: raise SwiftModelException('Cannot find ip address of %s for' ' %s' % (hostname, context)) return ip_address def aliases(self, ip_address): """ Given an ip address, return a list of alternate ip addresses :param ip_address: The ip address to lookup. :return: list of ip addresses """ aliases = [] network_name = self.host_ip_map.get(ip_address) if not network_name: return [] basename = Consumes.basename(network_name) for item in self.host_ip_map.keys(): item_name = self.host_ip_map.get(item) if item_name: item_basename = Consumes.basename(item_name) if item_basename == basename: aliases.append(item) return aliases
def _iter_volume_groups(self): for server in self.config.get('global').get('all_servers'): server_name = server.get('name') disk_model = server.get('disk_model') region_name = server.get('region') rack_id = server.get('rack') lv_index = 0 for volume_group in disk_model.get('volume_groups', []): vg_name = volume_group.get('name') physical_volumes = volume_group.get('physical_volumes') for logical_volume in volume_group.get('logical_volumes', []): lv_name = logical_volume.get('name') percent = logical_volume.get('size') consumer = logical_volume.get('consumer') if consumer and consumer.get('name', 'other') == 'swift': attrs = consumer.get('attrs') if not attrs: raise SwiftModelException('The attrs item is' ' missing from ' ' logical volume' ' %s in disk model %s' % (logical_volume.get( 'name'), disk_model.get('name'))) if attrs.get('rings', None) is None: raise SwiftModelException('The rings item is' ' missing from logical' ' volume' ' %s in disk model %s' % (logical_volume.get( 'name'), disk_model.get('name'))) for ring in attrs.get('rings'): if isinstance(ring, str): ring_name = ring else: ring_name = ring.get('name') server_ip, bind_port = self._get_server_bind( ring_name, server_name) if not server_ip: # TODO: this may be worth warning break swift_drive_name = LVM_MOUNT + str(lv_index) device_name = '/dev/' + vg_name + '/' + lv_name device_info = DeviceInfo({ 'region_name': region_name, 'rack_id': rack_id, 'region_id': 1, # later, the rack id may 'zone_id': 1, # change these defaults 'server_name': Consumes.basename( server_name), 'server_ip': server_ip, 'server_bind_port': bind_port, 'replication_ip': server_ip, 'replication_bind_port': bind_port, 'swift_drive_name': swift_drive_name, 'device_name': device_name, 'ring_name': ring_name, 'group_type': 'lvm', 'block_devices': {'percent': percent, 'physicals': physical_volumes}, 'presence': 'present'}) yield device_info lv_index += 1
def _iter_device_groups(self): for server in self.config.get('global').get('all_servers'): server_name = server.get('name') disk_model = server.get('disk_model') region_name = server.get('region') rack_id = server.get('rack') device_index = 0 for device_group in disk_model.get('device_groups', []): consumer = device_group.get('consumer') if consumer and consumer.get('name', 'other') == 'swift': attrs = consumer.get('attrs') if not attrs: raise SwiftModelException('The attrs item is' ' missing from device-groups' ' %s in disk model %s' % (device_group.get('name'), disk_model.get('name'))) devices = device_group.get('devices') if attrs.get('rings', None) is None: raise SwiftModelException('The rings item is' ' missing from device-groups' ' %s in disk model %s' % (device_group.get('name'), disk_model.get('name'))) for device in devices: for ring in attrs.get('rings'): if isinstance(ring, str): ring_name = ring else: ring_name = ring.get('name') server_ip, bind_port = self._get_server_bind( ring_name, server_name) if not server_ip: # When a swift service (example swift-account) # is configured in the input model to run a # node, we expect the node to be in the # "consumes" variable. e.g., consumes_SWF_ACC # should have this node in its list. Since we # failed to get the network name/port, it means # that it is not. # In model terms, we have a disk model that # calls out that a device hosts a ring (e.g. # account), but the node is not configured # to run SWF-ACC. # TODO: this may be worth warning break swift_drive_name = DISK_MOUNT + str(device_index) device_info = DeviceInfo({ 'region_name': region_name, 'rack_id': rack_id, 'region_id': 1, # later, the rack id may 'zone_id': 1, # change these defaults 'server_name': Consumes.basename(server_name), 'server_ip': server_ip, 'server_bind_port': bind_port, 'replication_ip': server_ip, 'replication_bind_port': bind_port, 'swift_drive_name': swift_drive_name, 'device_name': device.get('name'), 'ring_name': ring_name, 'group_type': 'device', 'block_devices': {'percent': '100%', 'physicals': [device.get('name')]}, 'presence': 'present'}) yield device_info device_index += 1