def get_mds_storagedriver_config_set(vpool): """ Builds a configuration for all StorageRouters from a given VPool with following goals: * Primary MDS is the local one * All slaves are on different hosts * Maximum `mds.safety` nodes are returned """ mds_per_storagerouter = {} mds_per_load = {} for storagedriver in vpool.storagedrivers: storagerouter = storagedriver.storagerouter mds_service, load = MDSServiceController.get_preferred_mds(storagerouter, vpool, include_load=True) mds_per_storagerouter[storagerouter.guid] = {'host': storagerouter.ip, 'port': mds_service.service.ports[0]} if load not in mds_per_load: mds_per_load[load] = [] mds_per_load[load].append(storagerouter.guid) safety = Configuration.getInt('ovs.storagedriver.mds.safety') config_set = {} for storagerouter_guid in mds_per_storagerouter: config_set[storagerouter_guid] = [mds_per_storagerouter[storagerouter_guid]] for load in sorted(mds_per_load.keys()): if len(config_set[storagerouter_guid]) >= safety: break sr_guids = mds_per_load[load] random.shuffle(sr_guids) for sr_guid in sr_guids: if len(config_set[storagerouter_guid]) >= safety: break if sr_guid != storagerouter_guid: config_set[storagerouter_guid].append(mds_per_storagerouter[sr_guid]) return config_set
def test_single_node(self): base_port = Configuration.getInt('ovs.ports.arakoon') cluster = 'one' node = sorted(TestArakoonInstaller.nodes.keys())[0] ArakoonInstaller.create_cluster(cluster, node, []) contents = SSHClient.load(node).file_read(self._get_config_path(cluster)) expected = TestArakoonInstaller.expected_global.format(cluster, TestArakoonInstaller.nodes[node]) expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1) self.assertEqual(contents.strip(), expected.strip())
def mds_checkup(): """ Validates the current MDS setup/configuration and takes actions where required """ mds_dict = {} for vpool in VPoolList.get_vpools(): for mds_service in vpool.mds_services: storagerouter = mds_service.service.storagerouter if vpool not in mds_dict: mds_dict[vpool] = {} if storagerouter not in mds_dict[vpool]: mds_dict[vpool][storagerouter] = [] mds_dict[vpool][storagerouter].append(mds_service) for vpool in mds_dict: # 1. First, make sure there's at least one MDS on every StorageRouter that's not overloaded # If not, create an extra MDS for that StorageRouter for storagerouter in mds_dict[vpool]: mds_services = mds_dict[vpool][storagerouter] has_room = False for mds_service in mds_services: load, _ = MDSServiceController.get_mds_load(mds_service) if load < Configuration.getInt('ovs.storagedriver.mds.maxload'): has_room = True break if has_room is False: client = SSHClient.load(storagerouter.ip) mds_service = MDSServiceController.prepare_mds_service(client, storagerouter, vpool, fresh_only=False, start=True) if mds_service is None: raise RuntimeError('Could not add MDS node') mds_dict[vpool][storagerouter].append(mds_service) mds_config_set = MDSServiceController.get_mds_storagedriver_config_set(vpool) for storagerouter in mds_dict[vpool]: client = SSHClient.load(storagerouter.ip) storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.name) storagedriver_config.load(client) if storagedriver_config.is_new is False: storagedriver_config.clean() # Clean out obsolete values storagedriver_config.configure_filesystem( fs_metadata_backend_mds_nodes=mds_config_set[storagerouter.guid] ) storagedriver_config.save(client) # 2. Per VPool, execute a safety check, making sure the master/slave configuration is optimal. for vdisk in vpool.vdisks: MDSServiceController.ensure_safety(vdisk)
def test_multi_node(self): base_port = Configuration.getInt('ovs.ports.arakoon') cluster = 'one' nodes = sorted(TestArakoonInstaller.nodes.keys()) ArakoonInstaller.create_cluster(cluster, nodes[0], []) for node in nodes[1:]: ArakoonInstaller.extend_cluster(nodes[0], node, cluster, []) expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes)) for node in nodes: expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1) expected = expected.strip() for node in nodes: contents = SSHClient.load(node).file_read(self._get_config_path(cluster)) self.assertEqual(contents.strip(), expected.strip()) node = nodes[0] ArakoonInstaller.shrink_cluster(nodes[1], node, cluster) expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes[1:])) for node in nodes[1:]: expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1) expected = expected.strip() for node in nodes[1:]: contents = SSHClient.load(node).file_read(self._get_config_path(cluster)) self.assertEqual(contents.strip(), expected.strip())
def ensure_safety(vdisk, excluded_storagerouters=None): """ Ensures (or tries to ensure) the safety of a given vdisk (except hypervisor). Assumptions: * A local overloaded master is better than a non-local non-overloaded master * Prefer master/services to be on different hosts, a subsequent slave on the same node doesn't add safety * Don't actively overload services (e.g. configure an MDS as slave causing it to get overloaded) * Too much safety is not wanted (it adds loads to nodes while not required) """ logger.debug('Ensuring MDS safety for vdisk {0}'.format(vdisk.guid)) vdisk.reload_client() if excluded_storagerouters is None: excluded_storagerouters = [] maxload = Configuration.getInt('ovs.storagedriver.mds.maxload') safety = Configuration.getInt('ovs.storagedriver.mds.safety') tlogs = Configuration.getInt('ovs.storagedriver.mds.tlogs') services = [mds_service.service for mds_service in vdisk.vpool.mds_services if mds_service.service.storagerouter not in excluded_storagerouters] nodes = set(service.storagerouter.ip for service in services) services_load = {} service_per_key = {} for service in services: load, load_plus = MDSServiceController.get_mds_load(service.mds_service) services_load[service.guid] = load, load_plus service_per_key['{0}:{1}'.format(service.storagerouter.ip, service.ports[0])] = service # List current configuration and filter out excluded services reconfigure_required = False reconfigure_reasons = [] vdisk.invalidate_dynamics(['info', 'storagedriver_id', 'storagerouter_guid']) configs = vdisk.info['metadata_backend_config'] for config in configs: config['key'] = '{0}:{1}'.format(config['ip'], config['port']) master_service = None if len(configs) > 0: config = configs[0] if config['key'] in service_per_key: master_service = service_per_key.get(config['key']) configs.remove(config) else: reconfigure_required = True reconfigure_reasons.append('Master ({0}:{1}) cannot be used anymore'.format(config['ip'], config['port'])) slave_services = [] for config in configs: if config['key'] in service_per_key: slave_services.append(service_per_key[config['key']]) else: reconfigure_required = True reconfigure_reasons.append('Slave ({0}:{1}) cannot be used anymore'.format(config['ip'], config['port'])) # Fix services_load services_per_load = {} for service in services: if service == master_service or service in slave_services: load = services_load[service.guid][0] else: load = services_load[service.guid][1] services_load[service.guid] = load if load not in services_per_load: services_per_load[load] = [] services_per_load[load].append(service) # Further checks if a reconfiguration is required. service_nodes = [] if master_service is not None: service_nodes.append(master_service.storagerouter.ip) for service in slave_services: ip = service.storagerouter.ip if ip in service_nodes: reconfigure_required = True reconfigure_reasons.append('Multiple MDS services on the same node') else: service_nodes.append(ip) if len(service_nodes) > safety: # Too much safety reconfigure_required = True reconfigure_reasons.append('Too much safety') if len(service_nodes) < safety and len(service_nodes) < len(nodes): # Insufficient MDS services configured while there should be sufficient nodes available reconfigure_required = True reconfigure_reasons.append('Not enough safety') if master_service is not None and services_load[master_service.guid] > maxload: # The master service is overloaded reconfigure_required = True reconfigure_reasons.append('Master overloaded') if master_service is not None and master_service.storagerouter_guid != vdisk.storagerouter_guid: # The master is not local reconfigure_required = True reconfigure_reasons.append('Master is not local') if any(service for service in slave_services if services_load[service.guid] > maxload): # There's a slave service overloaded reconfigure_required = True reconfigure_reasons.append('One or more slaves overloaded') if reconfigure_required is False: logger.debug('No reconfiguration required for vdisk {0}'.format(vdisk.guid)) MDSServiceController.sync_vdisk_to_reality(vdisk) return logger.debug('Reconfiguration required for vdisk {0}:'.format(vdisk.guid)) for reason in reconfigure_reasons: logger.debug('Reason: {0} - vdisk {1}'.format(reason, vdisk.guid)) # Prepare fresh configuration new_services = [] # Check whether the master (if available) is non-local to the vdisk and/or is overloaded master_ok = master_service is not None if master_ok is True: master_ok = master_service.storagerouter_guid == vdisk.storagerouter_guid and services_load[master_service.guid] <= maxload if master_ok: # Add this master to the fresh configuration new_services.append(master_service) else: # Try to find the best non-overloaded local MDS (slave) candidate_master = None candidate_master_load = 0 local_mds = None local_mds_load = 0 for service in services: load = services_load[service.guid] if load <= maxload and service.storagerouter_guid == vdisk.storagerouter_guid: if local_mds is None or local_mds_load > load: # This service is a non-overloaded local MDS local_mds = service local_mds_load = load if service in slave_services: if candidate_master is None or candidate_master_load > load: # This service is a non-overloaded local slave candidate_master = service candidate_master_load = load if candidate_master is not None: # A non-overloaded local slave was found. client = MetadataServerClient.load(candidate_master) amount_of_tlogs = client.catch_up(str(vdisk.volume_id), True) if amount_of_tlogs < tlogs: # Almost there. Catching up right now, and continue as soon as it's up-to-date start = time.time() client.catch_up(str(vdisk.volume_id), False) logger.debug('MDS catch up for vdisk {0} took {1}s'.format(vdisk.guid, round(time.time() - start, 2))) # It's up to date, so add it as a new master new_services.append(candidate_master) if master_service is not None: # The current master (if available) is now candidate for become one of the slaves slave_services.append(master_service) else: # It's not up to date, keep the previous master (if available) and give the local slave # some more time to catch up if master_service is not None: new_services.append(master_service) new_services.append(candidate_master) if candidate_master in slave_services: slave_services.remove(candidate_master) else: # There's no non-overloaded local slave found. Keep the current master (if available) and add # a local MDS (if available) as slave if master_service is not None: new_services.append(master_service) if local_mds is not None: new_services.append(local_mds) if local_mds in slave_services: slave_services.remove(local_mds) # At this point, there might (or might not) be a (new) master, and a (catching up) slave. The rest of the non-local # MDS nodes must now be added to the configuration until the safety is reached. There's always one extra # slave recycled to make sure there's always an (almost) up-to-date slave ready for failover loads = sorted(load for load in services_per_load.keys() if load <= maxload) nodes = set(service.storagerouter.ip for service in new_services) slave_added = False if len(nodes) < safety: for load in loads: for service in services_per_load[load]: if slave_added is False and service in slave_services and service.storagerouter.ip not in nodes: new_services.append(service) slave_services.remove(service) nodes.add(service.storagerouter.ip) slave_added = True if len(nodes) < safety: for load in loads: for service in services_per_load[load]: if len(nodes) < safety and service.storagerouter.ip not in nodes: new_services.append(service) nodes.add(service.storagerouter.ip) # Build the new configuration and update the vdisk configs = [] for service in new_services: client = MetadataServerClient.load(service) client.create_namespace(str(vdisk.volume_id)) configs.append(MDSNodeConfig(address=str(service.storagerouter.ip), port=service.ports[0])) vdisk.storagedriver_client.update_metadata_backend_config( volume_id=str(vdisk.volume_id), metadata_backend_config=MDSMetaDataBackendConfig(configs) ) MDSServiceController.sync_vdisk_to_reality(vdisk) logger.debug('Ensuring MDS safety for vdisk {0} completed'.format(vdisk.guid))