def get_overrides(self, namespace=None): replicas = 2 if utils.is_aio_system(self.dbapi): if utils.is_aio_simplex_system(self.dbapi): replicas = 1 if (utils.is_aio_system(self.dbapi) and not self._is_distributed_cloud_role_system_controller()): esJavaOpts = \ "-Djava.net.preferIPv6Addresses=true -Xmx512m -Xms512m" else: esJavaOpts = \ "-Djava.net.preferIPv6Addresses=true -Xmx1024m -Xms1024m" overrides = { common.HELM_NS_MONITOR: { 'replicas': replicas, 'esJavaOpts': esJavaOpts, 'nodeSelector': { common.LABEL_MONITOR_CLIENT: "enabled" }, 'resources': self._get_client_resources_overrides(), } } if namespace in self.SUPPORTED_NAMESPACES: return overrides[namespace] elif namespace: raise exception.InvalidHelmNamespace(chart=self.CHART, namespace=namespace) else: return overrides
def get_monitors_status(self, db_api): num_inv_monitors = 0 if cutils.is_aio_system(db_api): required_monitors = constants.MIN_STOR_MONITORS_AIO else: required_monitors = constants.MIN_STOR_MONITORS_MULTINODE quorum_names = [] inventory_monitor_names = [] # first check that the monitors are available in sysinv monitor_list = db_api.ceph_mon_get_list() for mon in monitor_list: ihost = db_api.ihost_get(mon['forihostid']) host_action = ihost['ihost_action'] or "" locking = (host_action.startswith(constants.LOCK_ACTION) or host_action.startswith(constants.FORCE_LOCK_ACTION)) if (ihost['administrative'] == constants.ADMIN_UNLOCKED and ihost['operational'] == constants.OPERATIONAL_ENABLED and not locking): num_inv_monitors += 1 inventory_monitor_names.append(ihost['hostname']) LOG.info("Active ceph monitors in inventory = %s" % str(inventory_monitor_names)) # check that the cluster is actually operational. # if we can get the monitor quorum from ceph, then # the cluster is truly operational if num_inv_monitors >= required_monitors: try: quorum_names = self._osd_quorum_names() except Exception: # if the cluster is not responding to requests # we set the quorum_names to an empty list , indicating a problem quorum_names = [] LOG.error("Ceph cluster not responding to requests.") LOG.info("Active ceph monitors in ceph cluster = %s" % str(quorum_names)) # There may be cases where a host is in an unlocked-available state, # but the monitor is down due to crashes or manual removal. # For such cases, we determine the list of active ceph monitors to be # the intersection of the sysinv reported unlocked-available monitor # hosts and the monitors reported in the quorum via the ceph API. active_monitors = list( set(inventory_monitor_names) & set(quorum_names)) num_active_monitors = len(active_monitors) if (num_inv_monitors and num_active_monitors == 0 and cutils.is_initial_config_complete() and not cutils.is_aio_system(db_api)): # The active controller always has a monitor. # We are on standard or storage, initial configuration # was completed and Ceph is down so we can't check if # it is working. Assume it is. num_active_monitors = 1 LOG.info("Active ceph monitors = %s" % str(active_monitors)) return num_active_monitors, required_monitors, active_monitors
def get_overrides(self, namespace=None): combined_data_and_master = False replicas = 2 if utils.is_aio_system(self.dbapi): if (utils.is_aio_duplex_system(self.dbapi) and self._count_hosts_by_label( common.LABEL_MONITOR_MASTER) < 3): # For AIO-DX without master labelled worker nodes, # configure elasticsearch data pods as master capable, # so they will form a cluster of 3 masters with the single # elasticsearch master pod. combined_data_and_master = True if utils.is_aio_simplex_system(self.dbapi): replicas = 1 if (utils.is_aio_system(self.dbapi) and not self._is_distributed_cloud_role_system_controller()): esJavaOpts = \ "-Djava.net.preferIPv6Addresses=true -Xmx1536m -Xms1536m" else: esJavaOpts = \ "-Djava.net.preferIPv6Addresses=true -Xmx4096m -Xms4096m" overrides = { common.HELM_NS_MONITOR: { 'nodeGroup': 'data', 'replicas': replicas, 'esJavaOpts': esJavaOpts, 'resources': self._get_data_resources_overrides(), 'volumeClaimTemplate': { 'accessModes': ["ReadWriteOnce"], 'resources': { 'requests': { 'storage': str(self.DATA_VOLUME_SIZE_GB) + 'Gi' } }, 'storageClass': 'general' }, 'nodeSelector': { common.LABEL_MONITOR_DATA: "enabled" }, 'antiAffinity': "hard", } } if combined_data_and_master: overrides[common.HELM_NS_MONITOR]['roles'] = {'master': 'true'} overrides[common.HELM_NS_MONITOR]['minimumMasterNodes'] = 1 if namespace in self.SUPPORTED_NAMESPACES: return overrides[namespace] elif namespace: raise exception.InvalidHelmNamespace(chart=self.CHART, namespace=namespace) else: return overrides
def _get_data_overrides(self): # Note memory values are to be system engineered. if utils.is_aio_system(self.dbapi): heap_size = "512m" memory_size = "512Mi" else: heap_size = "1536m" memory_size = "1536Mi" conf = { 'replicas': self._count_hosts_by_label(common.LABEL_MONITOR_DATA), 'heapSize': heap_size, 'resources': { 'limits': { 'cpu': "1" }, 'requests': { 'cpu': "25m", 'memory': memory_size, }, }, 'persistence': { 'storageClass': 'general', 'size': "100Gi" }, 'nodeSelector': { common.LABEL_MONITOR_DATA: "enabled" }, } return conf
def get_overrides(self, namespace=None): minimumMasterNodes = 1 replicas = 3 if utils.is_aio_system(self.dbapi): if self._count_hosts_by_label(common.LABEL_MONITOR_MASTER) < 3: # For AIO-SX, we will get here by definition, as there will # only be 1 master labelled host. # For AIO-DX without master labelled worker, we only # need 1 elasticsearch master pod, as the 2 data # pods will be master capable to form a cluster of 3 masters. replicas = 1 if (utils.is_aio_system(self.dbapi) and not self._is_distributed_cloud_role_system_controller()): esJavaOpts = "-Djava.net.preferIPv6Addresses=true -Xmx256m -Xms256m" else: esJavaOpts = "-Djava.net.preferIPv6Addresses=true -Xmx512m -Xms512m" overrides = { common.HELM_NS_MONITOR: { 'nodeGroup': 'master', 'replicas': replicas, 'esJavaOpts': esJavaOpts, 'minimumMasterNodes': minimumMasterNodes, 'nodeSelector': {common.LABEL_MONITOR_MASTER: "enabled"}, 'resources': self._get_master_resource_overrides(), 'volumeClaimTemplate': { 'accessModes': ["ReadWriteOnce"], 'resources': { 'requests': {'storage': '4Gi'} }, 'storageClass': 'general' }, } } if namespace in self.SUPPORTED_NAMESPACES: return overrides[namespace] elif namespace: raise exception.InvalidHelmNamespace(chart=self.CHART, namespace=namespace) else: return overrides
def _check_host(stor): ihost_id = stor['forihostid'] ihost = pecan.request.dbapi.ihost_get(ihost_id) stor_model = ceph.get_ceph_storage_model() # semantic check: whether OSD can be added to this host. if stor_model == constants.CEPH_STORAGE_MODEL: if ihost.personality != constants.STORAGE: msg = ("Storage model is '%s'. Storage devices can only be added " "to storage nodes." % stor_model) raise wsme.exc.ClientSideError(_(msg)) elif stor_model == constants.CEPH_CONTROLLER_MODEL: if ihost.personality != constants.CONTROLLER: msg = ("Storage model is '%s'. Storage devices can only be added " "to controller nodes." % stor_model) raise wsme.exc.ClientSideError(_(msg)) elif stor_model == constants.CEPH_UNDEFINED_MODEL: msg = ("Please install storage-0 or configure a Ceph monitor " "on a worker node before adding storage devices.") raise wsme.exc.ClientSideError(_(msg)) # semantic check: whether host is operationally acceptable if (stor_model == constants.CEPH_CONTROLLER_MODEL or stor_model == constants.CEPH_AIO_SX_MODEL): if (ihost['administrative'] == constants.ADMIN_UNLOCKED and ihost['operational'] != constants.OPERATIONAL_ENABLED): msg = _("Host %s must be unlocked and operational state " "enabled." % ihost['hostname']) raise wsme.exc.ClientSideError(msg) else: if ihost['administrative'] != constants.ADMIN_LOCKED: raise wsme.exc.ClientSideError(_("Host %s must be locked." % ihost['hostname'])) # semantic check: whether system has a ceph backend if not StorageBackendConfig.has_backend_configured( pecan.request.dbapi, constants.SB_TYPE_CEPH ): raise wsme.exc.ClientSideError(_( "System must have a %s backend" % constants.SB_TYPE_CEPH)) # semantic check: whether at least 2 unlocked hosts are monitors if not cutils.is_aio_system(pecan.request.dbapi): ceph_helper = ceph.CephApiOperator() num_monitors, required_monitors, __ = \ ceph_helper.get_monitors_status(pecan.request.dbapi) # CGTS 503 for now update monitors requirement until controller-0 is # inventoried # CGTS 1448 if num_monitors < required_monitors: raise wsme.exc.ClientSideError(_( "Only %d storage monitor available. " "At least %s unlocked and enabled hosts with monitors are " "required. Please ensure hosts with monitors are unlocked " "and enabled.") % (num_monitors, required_monitors))
def _get_master_overrides(self): if utils.is_aio_system(self.dbapi): heap_size = "256m" else: heap_size = "512m" conf = { 'replicas': self._count_hosts_by_label(common.LABEL_MONITOR_CONTROLLER), 'heapSize': heap_size, 'nodeSelector': { common.LABEL_MONITOR_CONTROLLER: "enabled" }, } return conf
def _get_resources_overrides(self): if (utils.is_aio_system(self.dbapi) and not self._is_distributed_cloud_role_system_controller()): cpu_limits = "500m" memory_limits = "1024Mi" else: cpu_limits = "500m" memory_limits = "2048Mi" return { 'limits': { 'cpu': cpu_limits, 'memory': memory_limits }, }
def _get_master_resource_overrides(self): if (utils.is_aio_system(self.dbapi) and not self._is_distributed_cloud_role_system_controller()): cpu_requests = "200m" memory_size = "256Mi" else: cpu_requests = "500m" memory_size = "512Mi" resources = { 'requests': { 'cpu': cpu_requests, 'memory': memory_size }, 'limits': { 'cpu': "1", 'memory': "1024Mi" }, } return resources
def _get_client_resources_overrides(self): if (utils.is_aio_system(self.dbapi) and not self._is_distributed_cloud_role_system_controller()): cpu_requests = "50m" cpu_limits = "1" # high watermark memory_size = "1024Mi" else: cpu_requests = "100m" cpu_limits = "1" # high watermark memory_size = "2048Mi" resources = { 'requests': { 'cpu': cpu_requests, 'memory': memory_size }, 'limits': { 'cpu': cpu_limits, 'memory': memory_size } } return resources
def _get_data_resources_overrides(self): # Default values based upon AIO+4 and Standard+20 system test if (utils.is_aio_system(self.dbapi) and not self._is_distributed_cloud_role_system_controller()): cpu_requests = "200m" cpu_limits = "1" memory_size = "4096Mi" else: cpu_requests = "500m" cpu_limits = "2" memory_size = "6144Mi" resources = { 'requests': { 'cpu': cpu_requests, 'memory': memory_size }, 'limits': { 'cpu': cpu_limits, 'memory': memory_size } } return resources
def fix_crushmap(dbapi=None): """ Set Ceph's CRUSH Map based on storage model """ def _create_crushmap_flag_file(): try: open(crushmap_flag_file, "w").close() except IOError as e: LOG.warn(('Failed to create flag file: {}. ' 'Reason: {}').format(crushmap_flag_file, e)) if not dbapi: dbapi = pecan.request.dbapi crushmap_flag_file = os.path.join(constants.SYSINV_CONFIG_PATH, constants.CEPH_CRUSH_MAP_APPLIED) if not os.path.isfile(crushmap_flag_file): _operator = CephApiOperator() if not cutils.is_aio_system(dbapi): # At least two monitors have to be running on a standard deployment, # otherwise don't even try to load the crushmap. active_mons, required_mons, __ = _operator.get_monitors_status( dbapi) if required_mons > active_mons: LOG.info("Not enough monitors yet available to fix crushmap.") return False # For AIO system, crushmap should be already loaded through puppet. # If it was loaded, set the crushmap flag to avoid loading it twice. default_ceph_tier_name = constants.SB_TIER_DEFAULT_NAMES[ constants.SB_TIER_TYPE_CEPH] + constants.CEPH_CRUSH_TIER_SUFFIX rule_is_present, __, __ = _operator._crush_rule_status( default_ceph_tier_name) if rule_is_present: _create_crushmap_flag_file() return False try: # For AIO system, crushmap should alreadby be loaded through # puppet. If for any reason it is not, as a precaution we set # the crushmap here. # Check if a backup crushmap exists. If it does, that means # it is during restore. We need to restore the backup crushmap # instead of generating it. For non-AIO system, it is stored in # /opt/platform/sysinv which is a drbd fs. For AIO systems because # when unlocking controller-0 for the first time, the crushmap is # set thru ceph puppet when /opt/platform is not mounted yet, we # store the crushmap in /etc/sysinv. if cutils.is_aio_system(dbapi): backup = os.path.join( constants.CEPH_CRUSH_MAP_BACKUP_DIR_FOR_AIO, constants.CEPH_CRUSH_MAP_BACKUP) else: backup = os.path.join(constants.SYSINV_CONFIG_PATH, constants.CEPH_CRUSH_MAP_BACKUP) crushmap_bin = "/etc/sysinv/crushmap.bin" if os.path.exists(backup): shutil.copyfile(backup, crushmap_bin) else: stor_model = get_ceph_storage_model(dbapi) if stor_model == constants.CEPH_AIO_SX_MODEL: crushmap_txt = "/etc/sysinv/crushmap-aio-sx.txt" elif stor_model == constants.CEPH_CONTROLLER_MODEL: crushmap_txt = "/etc/sysinv/crushmap-controller-model.txt" elif stor_model == constants.CEPH_STORAGE_MODEL: crushmap_txt = "/etc/sysinv/crushmap-storage-model.txt" else: reason = "Error: Undefined ceph storage model %s" % stor_model raise exception.CephCrushMapNotApplied(reason=reason) LOG.info("Updating crushmap with: %s" % crushmap_txt) # Compile crushmap subprocess.check_output("crushtool -c %s " "-o %s" % (crushmap_txt, crushmap_bin), stderr=subprocess.STDOUT, shell=True) # Set crushmap subprocess.check_output("ceph osd setcrushmap -i %s" % crushmap_bin, stderr=subprocess.STDOUT, shell=True) if os.path.exists(backup): os.remove(backup) except (IOError, subprocess.CalledProcessError) as e: # May not be critical, depends on where this is called. reason = "Error: %s Output: %s" % (str(e), e.output) raise exception.CephCrushMapNotApplied(reason=reason) _create_crushmap_flag_file() return True return False
def _check(self, op, tier): # Semantic checks LOG.debug("storage_tier: Semantic check for %s operation" % op) # Check storage tier parameters _check_parameters(tier) if op == "add": # See if this storage tier already exists tiers = pecan.request.dbapi.storage_tier_get_all(name=tier['name']) if len(tiers) != 0: raise wsme.exc.ClientSideError( _("Storage tier (%s) " "already present." % tier['name'])) # Deny adding secondary tier if initial configuration is not done. if not cutils.is_initial_config_complete(): msg = _( "Operation denied. Adding secondary tiers to a cluster requires " "initial configuration to be complete and controller node unlocked." ) raise wsme.exc.ClientSideError(msg) if cutils.is_aio_system(pecan.request.dbapi): # Deny adding secondary tiers if primary tier backend is not configured # for cluster. clusterId = tier.get('forclusterid') or tier.get('cluster_uuid') cluster_tiers = pecan.request.dbapi.storage_tier_get_by_cluster( clusterId) configured = False if cluster_tiers else True for t in cluster_tiers: if t.forbackendid: bk = pecan.request.dbapi.storage_backend_get( t.forbackendid) if bk.state != constants.SB_STATE_CONFIGURED: msg = _("Operation denied. Storage backend '%s' " "of tier '%s' must be in '%s' state." % (bk.name, t['name'], constants.SB_STATE_CONFIGURED)) raise wsme.exc.ClientSideError(msg) configured = True if not configured: msg = _( "Operation denied. Adding secondary tiers to a cluster requires " "primary tier storage backend of this cluster to be configured." ) raise wsme.exc.ClientSideError(msg) else: # Deny adding secondary tier if ceph is down on standard num_monitors, required_monitors, __ = \ self._ceph.get_monitors_status(pecan.request.dbapi) if num_monitors < required_monitors: raise wsme.exc.ClientSideError( _("Operation denied. Ceph is not operational. " "Only %d storage monitor available. " "At least %s unlocked and enabled hosts with " "monitors are required. Please ensure hosts " "with monitors are unlocked and enabled.") % (num_monitors, required_monitors)) elif op == "delete": if tier['name'] == constants.SB_TIER_DEFAULT_NAMES[ constants.SB_TIER_TYPE_CEPH]: raise wsme.exc.ClientSideError( _("Storage Tier %s cannot be " "deleted.") % tier['name']) if tier['status'] != constants.SB_TIER_STATUS_DEFINED: raise wsme.exc.ClientSideError( _("Storage Tier %s cannot be " "deleted. It is %s") % (tier['name'], tier['status'])) elif op == "modify": pass else: raise wsme.exc.ClientSideError( _("Internal Error: Invalid storage tier operation: %s" % op)) return tier