def _voldrv_arakoon_checkup(create_cluster): def _add_service(service_storagerouter, arakoon_ports, service_name): """ Add a service to the storage router """ new_service = Service() new_service.name = service_name new_service.type = service_type new_service.ports = arakoon_ports new_service.storagerouter = service_storagerouter new_service.save() return new_service current_ips = [] current_services = [] service_type = ServiceTypeList.get_by_name( ServiceType.SERVICE_TYPES.ARAKOON) cluster_name = Configuration.get( '/ovs/framework/arakoon_clusters').get('voldrv') if cluster_name is not None: arakoon_service_name = ArakoonInstaller.get_service_name_for_cluster( cluster_name=cluster_name) for service in service_type.services: if service.name == arakoon_service_name: current_services.append(service) if service.is_internal is True: current_ips.append(service.storagerouter.ip) all_sr_ips = [ storagerouter.ip for storagerouter in StorageRouterList.get_slaves() ] available_storagerouters = {} for storagerouter in StorageRouterList.get_masters(): storagerouter.invalidate_dynamics(['partition_config']) if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0: available_storagerouters[storagerouter] = DiskPartition( storagerouter.partition_config[DiskPartition.ROLES.DB][0]) all_sr_ips.append(storagerouter.ip) if create_cluster is True and len( current_services) == 0: # Create new cluster metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim( cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD) if metadata is None: # No externally managed cluster found, we create 1 ourselves if not available_storagerouters: raise RuntimeError( 'Could not find any Storage Router with a DB role') storagerouter, partition = available_storagerouters.items()[0] arakoon_voldrv_cluster = 'voldrv' arakoon_installer = ArakoonInstaller( cluster_name=arakoon_voldrv_cluster) arakoon_installer.create_cluster( cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD, ip=storagerouter.ip, base_dir=partition.folder, log_sinks=LogHandler.get_sink_path( 'arakoon-server_{0}'.format(arakoon_voldrv_cluster)), crash_log_sinks=LogHandler.get_sink_path( 'arakoon-server-crash_{0}'.format( arakoon_voldrv_cluster))) arakoon_installer.start_cluster() ports = arakoon_installer.ports[storagerouter.ip] metadata = arakoon_installer.metadata current_ips.append(storagerouter.ip) else: ports = [] storagerouter = None cluster_name = metadata['cluster_name'] Configuration.set('/ovs/framework/arakoon_clusters|voldrv', cluster_name) StorageDriverController._logger.info( 'Claiming {0} managed arakoon cluster: {1}'.format( 'externally' if storagerouter is None else 'internally', cluster_name)) StorageDriverController._configure_arakoon_to_volumedriver( cluster_name=cluster_name) current_services.append( _add_service( service_storagerouter=storagerouter, arakoon_ports=ports, service_name=ArakoonInstaller.get_service_name_for_cluster( cluster_name=cluster_name))) cluster_name = Configuration.get( '/ovs/framework/arakoon_clusters').get('voldrv') if cluster_name is None: return metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name( cluster_name=cluster_name) if 0 < len(current_services) < len( available_storagerouters) and metadata['internal'] is True: for storagerouter, partition in available_storagerouters.iteritems( ): if storagerouter.ip in current_ips: continue arakoon_installer = ArakoonInstaller(cluster_name=cluster_name) arakoon_installer.load() arakoon_installer.extend_cluster( new_ip=storagerouter.ip, base_dir=partition.folder, log_sinks=LogHandler.get_sink_path( 'arakoon-server_{0}'.format(cluster_name)), crash_log_sinks=LogHandler.get_sink_path( 'arakoon-server-crash_{0}'.format(cluster_name))) _add_service( service_storagerouter=storagerouter, arakoon_ports=arakoon_installer.ports[storagerouter.ip], service_name=ArakoonInstaller.get_service_name_for_cluster( cluster_name=cluster_name)) current_ips.append(storagerouter.ip) arakoon_installer.restart_cluster_after_extending( new_ip=storagerouter.ip) StorageDriverController._configure_arakoon_to_volumedriver( cluster_name=cluster_name)
def ensure_nsm_clusters_load(cls, alba_backend, nsms_per_storagerouter=None, min_internal_nsms=1, external_nsm_cluster_names=None, version_str=None, ssh_clients=None): # type: (AlbaBackend, Optional[Dict[StorageRouter, int]], Optional[int], Optional[List[str], Optional[str]], Optional[StorageRouter, SSHClient]) -> None """ Ensure that all NSM clusters are not overloaded :param alba_backend: Alba Backend to ensure NSM Cluster load for :type alba_backend: AlbaBackend :param nsms_per_storagerouter: Amount of NSMs mapped by StorageRouter :type nsms_per_storagerouter: Dict[StorageRouter, int] :param min_internal_nsms: Minimum amount of NSM hosts that need to be provided :type min_internal_nsms: int :param external_nsm_cluster_names: Information about the additional clusters to claim (only for externally managed Arakoon clusters) :type external_nsm_cluster_names: list :param version_str: Alba version string :type version_str: str :param ssh_clients: SSHClients to use :type ssh_clients: Dict[Storagerouter, SSHClient] :return: None :rtype: NoneType """ if ssh_clients is None: ssh_clients = {} if external_nsm_cluster_names is None: external_nsm_cluster_names = [] nsms_per_storagerouter = nsms_per_storagerouter if nsms_per_storagerouter is not None else cls.get_nsms_per_storagerouter( alba_backend) version_str = version_str or AlbaArakoonInstaller.get_alba_version_string( ) nsm_loads = cls.get_nsm_loads(alba_backend) internal = AlbaArakoonInstaller.is_internally_managed(alba_backend) abm_cluster_name = alba_backend.abm_cluster.name safety = Configuration.get( '/ovs/framework/plugins/alba/config|nsm.safety') maxload = Configuration.get( '/ovs/framework/plugins/alba/config|nsm.maxload') overloaded = min(nsm_loads.values()) >= maxload if not overloaded: # At least 1 NSM is not overloaded yet AlbaArakoonController._logger.debug( 'ALBA Backend {0} - NSM load OK'.format(alba_backend.name)) if internal: # When load is not OK, deploy at least 1 additional NSM nsms_to_add = max(0, min_internal_nsms - len(nsm_loads)) else: nsms_to_add = len(external_nsm_cluster_names) if nsms_to_add == 0: return else: AlbaArakoonController._logger.warning( 'ALBA Backend {0} - NSM load is NOT OK'.format( alba_backend.name)) if internal: # When load is not OK, deploy at least 1 additional NSM nsms_to_add = max(1, min_internal_nsms - len(nsm_loads)) else: # For externally managed clusters we only claim the specified clusters, if none provided, we just log it nsms_to_add = len(external_nsm_cluster_names) if nsms_to_add == 0: cls._logger.critical( 'ALBA Backend {0} - All NSM clusters are overloaded'. format(alba_backend.name)) return # Deploy new (internal) or claim existing (external) NSM clusters cls._logger.debug( 'ALBA Backend {0} - Currently {1} NSM cluster{2}'.format( alba_backend.name, len(nsm_loads), '' if len(nsm_loads) == 1 else 's')) AlbaArakoonController._logger.debug( 'ALBA Backend {0} - Trying to add {1} NSM cluster{2}'.format( alba_backend.name, nsms_to_add, '' if nsms_to_add == 1 else 's')) base_number = max(nsm_loads.keys()) + 1 for index, number in enumerate( xrange(base_number, base_number + nsms_to_add)): if not internal: # External clusters master_client = None if not ssh_clients: for storagerouter in StorageRouterList.get_masters(): try: master_client = SSHClient(storagerouter) except UnableToConnectException: cls._logger.warning( 'StorageRouter {0} with IP {1} is not reachable' .format(storagerouter.name, storagerouter.ip)) else: for storagerouter, ssh_client in ssh_clients.iteritems(): if storagerouter.node_type == 'MASTER': master_client = ssh_client if not master_client: raise ValueError('Could not find an online master node') # @todo this might raise an indexerror? nsm_cluster_name = external_nsm_cluster_names[index] cls._logger.debug( 'ALBA Backend {0} - Claiming NSM cluster {1}'.format( alba_backend.name, nsm_cluster_name)) metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim( cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.NSM, cluster_name=nsm_cluster_name) if metadata is None: cls._logger.critical( 'ALBA Backend {0} - NSM cluster with name {1} could not be found' .format(alba_backend.name, nsm_cluster_name)) continue cls._logger.debug( 'ALBA Backend {0} - Modeling services'.format( alba_backend.name)) AlbaArakoonInstaller.model_arakoon_service( alba_backend=alba_backend, cluster_name=nsm_cluster_name, number=number) cls._logger.debug('ALBA Backend {0} - Registering NSM'.format( alba_backend.name)) NSMInstaller.register_nsm(abm_name=abm_cluster_name, nsm_name=nsm_cluster_name, ip=master_client.ip) AlbaArakoonController._logger.debug( 'ALBA Backend {0} - Extended cluster'.format( alba_backend.name)) else: # Internal clusters nsm_cluster_name = '{0}-nsm_{1}'.format( alba_backend.name, number) cls._logger.debug( 'ALBA Backend {0} - Adding NSM cluster {1}'.format( alba_backend.name, nsm_cluster_name)) # One of the NSM nodes is overloaded. This means the complete NSM is considered overloaded # Figure out which StorageRouters are the least occupied loads = sorted(nsms_per_storagerouter.values())[:safety] storagerouters = [] for storagerouter, load in nsms_per_storagerouter.iteritems(): if load in loads: storagerouters.append(storagerouter) if len(storagerouters) == safety: break # Creating a new NSM cluster for sub_index, storagerouter in enumerate(storagerouters): nsms_per_storagerouter[storagerouter] += 1 partition = AlbaArakoonInstaller.get_db_partition( storagerouter) arakoon_installer = ArakoonInstaller( cluster_name=nsm_cluster_name) # @todo Use deploy and extend code. (Disable register nsm in those parts) if sub_index == 0: arakoon_installer.create_cluster( cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.NSM, ip=storagerouter.ip, base_dir=partition.folder, plugins={NSM_PLUGIN: version_str}) else: cls._logger.debug( 'ALBA Backend {0} - Extending NSM cluster {1}'. format(alba_backend.name, nsm_cluster_name)) arakoon_installer.load() arakoon_installer.extend_cluster( new_ip=storagerouter.ip, base_dir=partition.folder, plugins={NSM_PLUGIN: version_str}) cls._logger.debug( 'ALBA Backend {0} - Linking plugins'.format( alba_backend.name)) ssh_client = ssh_clients.get(storagerouter) or SSHClient( StorageRouter) AlbaArakoonInstaller.link_plugins( client=ssh_client, data_dir=partition.folder, plugins=[NSM_PLUGIN], cluster_name=nsm_cluster_name) cls._logger.debug( 'ALBA Backend {0} - Modeling services'.format( alba_backend.name)) AlbaArakoonInstaller.model_arakoon_service( alba_backend=alba_backend, cluster_name=nsm_cluster_name, ports=arakoon_installer.ports[storagerouter.ip], storagerouter=storagerouter, number=number) if sub_index == 0: cls._logger.debug( 'ALBA Backend {0} - Starting cluster'.format( alba_backend.name)) arakoon_installer.start_cluster() else: AlbaArakoonController._logger.debug( 'ALBA Backend {0} - Restarting cluster'.format( alba_backend.name)) arakoon_installer.restart_cluster_after_extending( new_ip=storagerouter.ip) cls._logger.debug('ALBA Backend {0} - Registering NSM'.format( alba_backend.name)) NSMInstaller.register_nsm(abm_name=abm_cluster_name, nsm_name=nsm_cluster_name, ip=storagerouters[0].ip) cls._logger.debug( 'ALBA Backend {0} - Added NSM cluster {1}'.format( alba_backend.name, nsm_cluster_name))