def get(self, request, fsid): servers = self.client.server_list_cluster(fsid, async=True) osd_data = self.client.get_sync_object(fsid, OsdMap.str, async=True) osds = self.client.list(fsid, OSD, {}, async=True) pg_summary = self.client.get_sync_object(fsid, PgSummary.str, async=True) osds = osds.get() servers = servers.get() osd_data = osd_data.get() pg_summary = pg_summary.get() osd_map = OsdMap(None, osd_data) server_info = self.client.server_by_service([ServiceId(fsid, OSD, str(osd['osd'])) for osd in osds], async=True) server_info = server_info.get() osds, osds_by_pg_state = self.generate(pg_summary, osd_map, server_info, servers) if not osds or not osds_by_pg_state: return Response([], status.HTTP_202_ACCEPTED) pg_states = request.QUERY_PARAMS.get('pg_states', None) if pg_states: osds = self._filter_by_pg_state(osds, pg_states, osds_by_pg_state) osd_list = DataObject({ # 'osds': [DataObject({'osd': o}) for o in osds], 'osds': osds, 'osds_by_pg_state': osds_by_pg_state }) return Response(OSDListSerializer(osd_list).data)
def retrieve(self, request, fsid, osd_id): osd = self.client.get_sync_object( fsid, 'osd_map', ['osds_by_id', int(osd_id)]) crush_node = self.client.get_sync_object( fsid, 'osd_map', ['osd_tree_node_by_id', int(osd_id)]) osd['reweight'] = float(crush_node['reweight']) osd['server'] = self.client.server_by_service( [ServiceId(fsid, OSD, osd_id)])[0][1] pools = self.client.get_sync_object( fsid, 'osd_map', ['osd_pools', int(osd_id)]) osd['pools'] = pools osd_metadata = self.client.get_sync_object( fsid, 'osd_map', ['metadata_by_id', int(osd_id)]) try: osd['backend_device_node'] = osd_metadata[ 'backend_filestore_dev_node'] except KeyError: osd['backend_device_node'] = None try: osd['backend_partition_path'] = osd_metadata[ 'backend_filestore_partition_path'] except KeyError: osd['backend_partition_path'] = None osd_commands = self.client.get_valid_commands(fsid, OSD, [int(osd_id)]) osd.update(osd_commands[int(osd_id)]) parent_map = self.client.get_sync_object(fsid, 'osd_map', ['parent_bucket_by_node_id']) osd.update( {'crush_node_ancestry': lookup_ancestry(osd['osd'], parent_map)}) return Response(self.serializer_class(DataObject(osd)).data)
def retrieve_status(self, request, fsid, mon_id): service_info = self.client.status_by_service( [ServiceId(fsid, 'mon', mon_id)])[0] if service_info is None: raise Http404("Mon not found '%s'" % mon_id) return Response(service_info['status'])
def status_by_service(self, services): result = self._manager.servers.get_services( [ServiceId(*s) for s in services]) return [({ 'running': ss.running, 'server': ss.server_state.fqdn, 'status': ss.status } if ss else None) for ss in result]
def _get_fqdn(self, fsid, service_type, service_id): """ Resolve a service to a FQDN if possible, else return None """ server = self._manager.servers.get_by_service(ServiceId(fsid, service_type, str(service_id))) if server is None: log.warn("No server found for service %s %s" % (service_type, service_id)) return server.fqdn if server else None
def server_by_service(self, services): """ Return a list of 2-tuples mapping of service ID to server FQDN Note that we would rather return a dict but tuple dict keys are awkward to serialize """ result = self._manager.servers.list_by_service([ServiceId(*s) for s in services]) return result
def list(self, request, fsid): # Get data needed for filtering list_filter = {} if 'pool' in request.GET: try: pool_id = int(request.GET['pool']) except ValueError: return Response("Pool ID must be an integer", status=status.HTTP_400_BAD_REQUEST) list_filter['pool'] = pool_id if 'id__in[]' in request.GET: try: ids = request.GET.getlist("id__in[]") list_filter['id__in'] = [int(i) for i in ids] except ValueError: return Response("Invalid OSD ID in list", status=status.HTTP_400_BAD_REQUEST) # Get data osds = self.client.list(fsid, OSD, list_filter, async=True) osd_to_pools = self.client.get_sync_object(fsid, 'osd_map', ['osd_pools'], async=True) crush_nodes = self.client.get_sync_object(fsid, 'osd_map', ['osd_tree_node_by_id'], async=True) osds = osds.get() # Get data depending on OSD list server_info = self.client.server_by_service([ServiceId(fsid, OSD, str(osd['osd'])) for osd in osds], async=True) osd_commands = self.client.get_valid_commands(fsid, OSD, [x['osd'] for x in osds], async=True) # Preparation complete, await all data to serialize result osd_to_pools = osd_to_pools.get() crush_nodes = crush_nodes.get() server_info = server_info.get() osd_commands = osd_commands.get() # Build OSD data objects for o in osds: # An OSD being in the OSD map does not guarantee its presence in the CRUSH # map, as "osd crush rm" and "osd rm" are separate operations. try: o.update({'reweight': float(crush_nodes[o['osd']]['reweight'])}) except KeyError: log.warning("No CRUSH data available for OSD {0}".format(o['osd'])) o.update({'reweight': 0.0}) for o, (service_id, fqdn) in zip(osds, server_info): o['server'] = fqdn for o in osds: o['pools'] = osd_to_pools[o['osd']] for o in osds: o.update(osd_commands[o['osd']]) return Response(self.serializer_class([DataObject(o) for o in osds], many=True).data)
def on_mon_map(self, mon_map, mon_status): """ When a new mon map is received, use it to eliminate any mon ServiceState records that no longer exist in the real world. """ log.debug("ServerMonitor.on_mon_map: %s" % str([m['name'] for m in mon_map['mons']])) # We're no longer getting these via salt so we fake them # based on what we know in the mon_map if mon_status is None: mon_status = {} for mon in mon_map['mons']: services = { mon['name']: { 'fsid': mon_map['fsid'], 'type': 'mon', 'status': { 'election_epoch': mon_status.get('election_epoch'), 'quorum': mon_map['quorum'], 'rank': mon['rank'] }, 'id': mon['name'] } } mon_addr = mon.get('addr') mon_name = mon['name'] if mon_addr is not None: mon_addr = mon_addr.split('/')[0].split(':')[ 0] # deal with CIDR notation try: mon_name = socket.getfqdn(mon_addr) except socket.gaierror: pass self.on_server_heartbeat(mon_name, { 'boot_time': 0, 'ceph_version': None, 'services': services }) map_mons = set([ ServiceId(mon_map['fsid'], 'mon', m['name']) for m in mon_map['mons'] ]) known_mons = set([ s.id for s in self.fsid_services[mon_map['fsid']] if s.service_type == 'mon' ]) for stale_mon_id in known_mons - map_mons: self.forget_service(self.services[stale_mon_id])
def retrieve(self, request, fsid, osd_id): osd = self.client.get_sync_object(fsid, 'osd_map', ['osds_by_id', int(osd_id)]) crush_node = self.client.get_sync_object(fsid, 'osd_map', ['osd_tree_node_by_id', int(osd_id)]) osd['reweight'] = float(crush_node['reweight']) osd['server'] = self.client.server_by_service([ServiceId(fsid, OSD, osd_id)])[0][1] pools = self.client.get_sync_object(fsid, 'osd_map', ['osd_pools', int(osd_id)]) osd['pools'] = pools osd_commands = self.client.get_valid_commands(fsid, OSD, [int(osd_id)]) osd.update(osd_commands[int(osd_id)]) return Response(self.serializer_class(DataObject(osd)).data)
def fixup_osd(osd): osd_id = osd['osd'] data = dict((k, osd[k]) for k in OSD_FIELDS) data.update({'id': osd_id}) data.update({'osd': osd_id}) data.update({'pg_states': dict(pg_states_by_osd[osd_id])}) data.update({'pools': list(pools_by_osd[osd_id])}) server = server_monitor.get_by_service( ServiceId(cluster_monitor.fsid, 'osd', str(osd_id))) data.update({'host': server.hostname if server else None}) data.update({'fqdn': server.fqdn if server else None}) return data
def on_mon_map(self, mon_map): """ When a new mon map is received, use it to eliminate any mon ServiceState records that no longer exist in the real world. """ map_mons = set([ ServiceId(mon_map['fsid'], 'mon', m['name']) for m in mon_map['mons'] ]) known_mons = set([ s.id for s in self.fsid_services[mon_map['fsid']] if s.service_type == 'mon' ]) for stale_mon_id in known_mons - map_mons: self.forget_service(self.services[stale_mon_id])
def on_mds_map(self, fsid, mds_map): """ When a new MDS map is received, use it to eliminate any MDS ServiceState records that no longer exist in the real world. :param fsid: Pass in fsid string because mds map doesn't include it :param mds_map: The MDS map sync object """ map_mds = set([ ServiceId(fsid, 'mds', i['name']) for i in mds_map['info'].values() ]) known_mds = set([ s.id for s in self.fsid_services[fsid] if s.service_type == 'mds' ]) for stale_mds_id in known_mds - map_mds: self.forget_service(self.services[stale_mds_id])
def _get_up_mon_servers(self, fsid): # Resolve FSID to list of mon FQDNs servers = self.client.server_list_cluster(fsid) # Sort to get most recently contacted server first; drop any # for whom last_contact is None servers = [s for s in servers if s['last_contact']] servers = sorted(servers, key=lambda t: dateutil_parse(t['last_contact']), reverse=True) mon_fqdns = [] for server in servers: for service in server['services']: service_id = ServiceId(*(service['id'])) if service[ 'running'] and service_id.service_type == MON and service_id.fsid == fsid: mon_fqdns.append(server['fqdn']) return mon_fqdns
def get_cluster_log(self, request, fsid): """ Retrieve the cluster log from one of a cluster's mons (expect it to be in /var/log/ceph/ceph.log) """ lines = request.GET.get('lines', 40) # Resolve FSID to name name = self.client.get_cluster(fsid)['name'] # Resolve FSID to list of mon FQDNs servers = self.client.server_list_cluster(fsid) # Sort to get most recently contacted server first; drop any # for whom last_contact is None servers = [s for s in servers if s['last_contact']] servers = sorted(servers, key=lambda t: dateutil_parse(t['last_contact']), reverse=True) mon_fqdns = [] for server in servers: for service in server['services']: service_id = ServiceId(*(service['id'])) if service[ 'running'] and service_id.service_type == MON and service_id.fsid == fsid: mon_fqdns.append(server['fqdn']) client = salt.client.LocalClient( config.get('cthulhu', 'salt_config_path')) log.debug("LogTailViewSet: mons for %s are %s" % (fsid, mon_fqdns)) # For each mon FQDN, try to go get ceph/$cluster.log, if we succeed return it, if we fail try the next one # NB this path is actually customizable in ceph as `mon_cluster_log_file` but we assume user hasn't done that. for mon_fqdn in mon_fqdns: results = client.cmd(mon_fqdn, "log_tail.tail", ["ceph/{name}.log".format(name=name), lines]) if results: return Response({'lines': results[mon_fqdn]}) else: log.info("Failed to get log from %s" % mon_fqdn) # If none of the mons gave us what we wanted, return a 503 service unavailable return Response("mon log data unavailable", status=status.HTTP_503_SERVICE_UNAVAILABLE)
def _get_mons(self, fsid): mon_status = self.client.get_sync_object(fsid, 'mon_status') if not mon_status: raise Http404("No mon data available") mons = mon_status['monmap']['mons'] service_ids = [ServiceId(fsid, MON, mon['name']) for mon in mons] services_info = self.client.status_by_service(service_ids) # Use this to invalidate any statuses we can prove are outdated lowest_valid_epoch = mon_status['election_epoch'] # Step 1: account for the possibility that our cluster-wide mon_status object # could be out of date with respect to local mon_status data that we get # from each mon service. for mon, service_info in zip(mons, services_info): if service_info and service_info['status']: local_epoch = service_info['status']['election_epoch'] if local_epoch > lowest_valid_epoch: # Evidence that the cluster mon status is out of date, and we have # a more recent one to replace it with. log.warn( "Using mon '%s' local status as it is most recent" % (mon['name'])) mon_status = service_info['status'] lowest_valid_epoch = mon_status['election_epoch'] elif local_epoch == lowest_valid_epoch and service_info[ 'status']['quorum'] != mon_status['quorum']: # Evidence that the cluster mon status is out of date, and we # have to assume that anyone it claimed was in quorum no longer is. log.warn( "Disregarding cluster mon status because '%s' disagrees" % (mon['name'])) lowest_valid_epoch = local_epoch + 1 # Step 2: Reconcile what the cluster mon status thinks about this mon with # what it thinks about itself. for mon, service_info in zip(mons, services_info): mon['server'] = service_info['server'] if service_info else None cluster_opinion = mon['rank'] in mon_status[ 'quorum'] and mon_status['election_epoch'] >= lowest_valid_epoch if service_info is None or service_info['status'] is None: # Handle local data being unavailable, e.g. if our agent # is not installed on one or more mons mon['status'] = None mon['in_quorum'] = cluster_opinion continue status = service_info['status'] mon['status'] = status local_opinion = service_info['running'] and (status['rank'] in status['quorum']) and \ status['election_epoch'] >= lowest_valid_epoch if cluster_opinion != local_opinion: log.warn("mon %s/%s local state disagrees with cluster state" % (mon['name'], mon['rank'])) if status['election_epoch'] == 0 or not service_info['running']: # You're claiming not to be in quorum, I believe you because I have # no way of knowing the cluster map is more up to date than your info. mon['in_quorum'] = local_opinion elif status['election_epoch'] < mon_status['election_epoch']: # The cluster map is unambiguously more up to date than your info, so # I believe it. mon['in_quorum'] = cluster_opinion else: # Your data is newer than the cluster map, I believe you. mon['in_quorum'] = local_opinion else: mon['in_quorum'] = cluster_opinion # Step 3: special case, handle when our local inferrences about mon status # make it impossible for us to believe what the cluster mon status is telling us. if len([m for m in mons if m['in_quorum']]) < (len(mons) / 2 + 1): log.warn( "Asserting that there is no quorum even if cluster map says there is" ) # I think the cluster map is lying about there being a quorum at all for m in mons: m['in_quorum'] = False return mons
def on_server_heartbeat(self, fqdn, server_heartbeat): """ Call back for when a ceph.service message is received from a salt minion. This is actually a fairly simple operation of updating the in memory ServerState to reflect what is in the message, but it's convoluted because we may be seeing a new server, a known server, or a server which was known but unmanaged. """ log.debug("ServerMonitor.on_server_heartbeat: %s" % fqdn) new_server = True newly_managed_server = False try: server_state = self.servers[fqdn] new_server = False except KeyError: # Look up the grains for this server, we need to know its hostname in order # to resolve this vs. the OSD map. hostname = self._get_grains(fqdn)['host'] if hostname in self.hostname_to_server: server_state = self.hostname_to_server[hostname] if not server_state.managed: # Take over a ServerState that was created from OSD map server_state.managed = True old_fqdn = server_state.fqdn # OSD map servers would have faked up FQDN as hostname, so clear that out del self.servers[old_fqdn] server_state.fqdn = fqdn self.servers[server_state.fqdn] = server_state self._persister.update_server(old_fqdn, fqdn=fqdn, managed=True) new_server = False log.info("Server %s went from unmanaged to managed" % fqdn) newly_managed_server = True else: # We will go on to treat these as distinct servers even though # they have the same hostname log.warn( "Hostname clash: FQDNs '%s' and '%s' both have hostname %s" % (fqdn, server_state.fqdn, hostname)) else: # The case where hostname == FQDN, we may already have this FQDN in our # map from an unmanaged server being reported by hostname. if not server_state.managed: newly_managed_server = True server_state.managed = True self._persister.update_server(server_state.fqdn, managed=True) log.info("Server %s went from unmanaged to managed" % fqdn) boot_time = datetime.datetime.fromtimestamp( server_heartbeat['boot_time'], tz=tz.tzutc()) if new_server: hostname = self._get_grains(fqdn)['host'] server_state = ServerState( fqdn, hostname, managed=True, last_contact=now(), boot_time=boot_time, ceph_version=server_heartbeat['ceph_version']) self.inject_server(server_state) self._persister.create_server( Server(fqdn=server_state.fqdn, hostname=server_state.hostname, managed=server_state.managed, last_contact=server_state.last_contact)) log.info("Saw server %s for the first time" % server_state) server_state.last_contact = now() self._persister.update_server(server_state.fqdn, last_contact=server_state.last_contact) if server_state.boot_time != boot_time: log.warn("{0} boot time changed, old {1} new {2}".format( server_state.fqdn, server_state.boot_time, boot_time)) old_boot_time = server_state.boot_time server_state.boot_time = boot_time self._persister.update_server(server_state.fqdn, boot_time=server_state.boot_time) if old_boot_time is not None: # i.e. a reboot, not an unmanaged->managed transition if server_state.boot_time < old_boot_time: log.warn("Server boot time went backwards") elif server_state.boot_time - old_boot_time < REBOOT_THRESHOLD: log.warn("Server boot time changed, but only a little") else: # A substantial forward change in boot time, that's a reboot: emit # a user visible event log.warn("{0} rebooted!".format(fqdn)) self._eventer.on_reboot(server_state, False) if server_state.ceph_version != server_heartbeat['ceph_version']: # Interpret "no package installed but some services running" as meaning we're # in the process of upgrading. upgrading = server_heartbeat[ 'ceph_version'] is None and server_heartbeat['services'] if server_heartbeat['ceph_version'] is None and upgrading: # Ignore version=None while upgrading to avoid generating spurious # "ceph uninstalled" events pass else: server_state.ceph_version = server_heartbeat['ceph_version'] self._persister.update_server( server_state.fqdn, ceph_version=server_state.ceph_version) if not (new_server or newly_managed_server): self._eventer.on_new_version(server_state) seen_id_tuples = set() for service_name, service in server_heartbeat['services'].items(): id_tuple = ServiceId(service['fsid'], service['type'], service['id']) seen_id_tuples.add(id_tuple) self._register_service(server_state, id_tuple, running=True, status=service['status']) # For any service which was last reported on this server but # is now gone, mark it as not running for unseen_id_tuple in set( server_state.services.keys()) ^ seen_id_tuples: service_state = self.services[unseen_id_tuple] if service_state.running: log.info("Service %s stopped on server %s" % (service_state, server_state)) service_state.running = False if new_server or newly_managed_server: # We do this at the end so that by the time we emit the event # the ServiceState objects have been created self._eventer.on_server(server_state)
def on_osd_map(self, osd_map): """ For when a new OSD map is received: we may infer the existence of hosts from the CRUSH map if the hosts are not all sending us data with salt. :param osd_map: The data from an OsdMap sync object """ log.debug("ServerMonitor.on_osd_map: epoch %s" % osd_map['epoch']) hostname_to_osds = self.get_hostname_to_osds(osd_map) log.debug("ServerMonitor.on_osd_map: got service data for %s servers" % len(hostname_to_osds)) osds_in_map = set() for hostname, osds in hostname_to_osds.items(): id_to_osd = dict([(ServiceId(osd_map['fsid'], 'osd', str(o['osd'])), o) for o in osds]) osds_in_map |= set(id_to_osd.keys()) # Identify if this is a CRUSH alias rather than a real hostname, by # checking if any of the OSDs mentioned are already recorded as children # of a managed host. crush_alias_to = None if hostname not in self.hostname_to_server: for service_id, osd in id_to_osd.items(): try: service_state = self.services[service_id] if service_state.server_state.managed: crush_alias_to = service_state.server_state except KeyError: pass if crush_alias_to: log.info("'{0}' is a CRUSH alias to {1}".format( hostname, crush_alias_to)) continue # Look up or create ServerState for the server named in the CRUSH map try: server_state = self.hostname_to_server[hostname] except KeyError: # Fake FQDN to equal hostname server_state = ServerState(hostname, hostname, managed=False, last_contact=None, boot_time=None, ceph_version=None) self.inject_server(server_state) self._persister.create_server( Server(fqdn=server_state.fqdn, hostname=server_state.hostname, managed=server_state.managed)) # Register all the OSDs reported under this hostname with the ServerState for service_id, osd in id_to_osd.items(): if not server_state.managed: # Only pay attention to these services for unmanaged servers, # for managed servers rely on ceph/server salt messages self._register_service(server_state, service_id, bool(osd['up']), None) # Remove ServiceState for any OSDs for this FSID which are not # mentioned in hostname_to_osds known_osds = set([ s.id for s in self.fsid_services[osd_map['fsid']] if s.service_type == 'osd' ]) for stale_service_id in known_osds - osds_in_map: self.forget_service(self.services[stale_service_id])
def id(self): return ServiceId(self.fsid, self.service_type, self.service_id)