def _is_favorite(self, minion_id):
"""
Check if this minion is the one which we are currently treating
as the primary source of updates, and promote it to be the
favourite if the favourite has not sent a heartbeat since
rlyeh->favorite_timeout_s.
:return True if this minion was the favorite or has just been
promoted.
"""
t_now = now()
self._last_heartbeat[minion_id] = t_now
if self._favorite_mon is None:
log.debug("%s is my new favourite" % minion_id)
self._set_favorite(minion_id)
return True
elif minion_id != self._favorite_mon:
# Consider whether this minion should become my new favourite: has it been
# too long since my current favourite reported in?
time_since = t_now - self._last_heartbeat[self._favorite_mon]
favorite_timeout_s = self._servers.get_contact_period(
self._favorite_mon) * FAVORITE_TIMEOUT_FACTOR
if time_since > datetime.timedelta(seconds=favorite_timeout_s):
log.debug(
"My old favourite, %s, has not sent a heartbeat for %s: %s is my new favourite"
% (self._favorite_mon, time_since, minion_id))
self._set_favorite(minion_id)
return minion_id == self._favorite_mon
def _emit_stats(self):
try:
if not self._socket:
log.info("Opening carbon socket {0}:{1}".format(
self.CARBON_HOST, self.CARBON_PORT))
self._socket = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
self._socket.connect((self.CARBON_HOST, self.CARBON_PORT))
carbon_data = ""
t = int(time.time())
usage = resource.getrusage(resource.RUSAGE_SELF)
for usage_field in ("utime", "stime", "maxrss", "ixrss", "idrss",
"isrss", "minflt", "majflt", "nswap",
"inblock", "oublock", "msgsnd", "msgrcv",
"nsignals", "nvcsw", "nivcsw"):
val = getattr(usage, "ru_{0}".format(usage_field))
log.debug("{0}: {1}".format(usage_field, val))
carbon_data += "calamari.rlyeh.ru_{0} {1} {2}\n".format(
usage_field, val, t)
self._socket.sendall(carbon_data)
except socket.gaierror, resource.error:
log.exception("Failed to send debugging statistics")
self._close()
def on_heartbeat(self, fqdn, data):
pass
if not data['fsid'] in self._manager.clusters:
self._manager.on_discovery(fqdn, data)
else:
log.debug("%s: heartbeat from existing cluster %s" %
(self.__class__.__name__, data['fsid']))
def on_fetch_complete(self, minion_id, sync_type, version, data):
"""
:return A SyncObject if this version was new to us, else None
"""
log.debug("SyncObjects.on_fetch_complete %s/%s/%s" %
(minion_id, sync_type.str, version))
self._fetching_at[sync_type] = None
# A fetch might give us a newer version than we knew we had asked for
if sync_type.cmp(version, self._known_versions[sync_type]) > 0:
self._known_versions[sync_type] = version
# Don't store this if we already got something newer
if sync_type.cmp(version, self.get_version(sync_type)) <= 0:
log.warn("Ignoring outdated update %s/%s from %s" %
(sync_type.str, version, minion_id))
new_object = None
else:
log.info("Got new version %s/%s" % (sync_type.str, version))
new_object = self.set_map(sync_type, version, data)
# This might not be the latest: if it's not, send out another fetch
# right away
if sync_type.cmp(self._known_versions[sync_type], version) > 0:
self.fetch(minion_id, sync_type)
return new_object
def on_heartbeat(self, fqdn, data):
pass
if not data['fsid'] in self._manager.clusters:
self._manager.on_discovery(fqdn, data)
else:
log.debug("%s: heartbeat from existing cluster %s" % (
self.__class__.__name__, data['fsid']))
def wrap(*args, **kwargs):
log.debug("RpcInterface >> %s(%s, %s)" % (item, args, kwargs))
try:
rc = attr(*args, **kwargs)
log.debug("RpcInterface << %s" % item)
except:
log.exception("RpcInterface !! %s" % item)
raise
return rc
def _run(self):
self._plugin_monitor.start()
self._ready.set()
log.debug("ClusterMonitor._run: ready")
remote.listen(self._complete,
on_heartbeat=self.on_heartbeat,
fsid=self.fsid,
on_job=self.on_job_complete)
log.info("%s complete" % self.__class__.__name__)
self._plugin_monitor.stop()
self._plugin_monitor.join()
self.done.set()
def on_job_complete(self, fqdn, jid, success, result, cmd, args):
# It would be much nicer to put the FSID at the start of
# the tag, if salt would only let us add custom tags to our jobs.
# Instead we enforce a convention that calamari jobs include
# fsid in their return value.
if 'fsid' not in result or result['fsid'] != self.fsid:
# Something for a different ClusterMonitor
log.debug("Ignoring job return, not for my FSID")
return
if cmd == 'ceph.get_cluster_object':
# A ceph.get_cluster_object response
if not success:
log.error("on_sync_object: failure from %s: %s" %
(fqdn, result))
return
self.on_sync_object(fqdn, result)
else:
log.warning("Unexpected function '%s' (%s)" % (cmd, cmd))
def _emit_stats(self):
try:
if not self._socket:
log.info("Opening carbon socket {0}:{1}".format(self.CARBON_HOST, self.CARBON_PORT))
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.connect((self.CARBON_HOST, self.CARBON_PORT))
carbon_data = ""
t = int(time.time())
usage = resource.getrusage(resource.RUSAGE_SELF)
for usage_field in ("utime", "stime", "maxrss", "ixrss", "idrss", "isrss", "minflt", "majflt",
"nswap", "inblock", "oublock", "msgsnd", "msgrcv", "nsignals", "nvcsw", "nivcsw"):
val = getattr(usage, "ru_{0}".format(usage_field))
log.debug("{0}: {1}".format(usage_field, val))
carbon_data += "calamari.rlyeh.ru_{0} {1} {2}\n".format(usage_field, val, t)
self._socket.sendall(carbon_data)
except socket.gaierror, resource.error:
log.exception("Failed to send debugging statistics")
self._close()
def on_sync_object(self, minion_id, data):
if minion_id != self._favorite_mon:
log.debug("Ignoring map from %s, it is not my favourite (%s)" %
(minion_id, self._favorite_mon))
assert data['fsid'] == self.fsid
sync_object = data['data']
sync_type = SYNC_OBJECT_STR_TYPE[data['type']]
new_object = self.inject_sync_object(minion_id, data['type'],
data['version'], sync_object)
if new_object:
self._requests.on_map(self.fsid, sync_type, new_object)
self._persister.update_sync_object(
self.fsid, self.name,
sync_type.str, new_object.version if isinstance(
new_object.version, int) else None, now(), sync_object)
else:
log.warn(
"ClusterMonitor.on_sync_object: stale object received from %s"
% minion_id)
def fetch(self, minion_id, sync_type):
log.debug("SyncObjects.fetch: %s/%s" % (minion_id, sync_type))
if minion_id is None:
# We're probably being replayed to from the database
log.warn("SyncObjects.fetch called with minion_id=None")
return
self._fetching_at[sync_type] = now()
try:
# TODO clean up unused 'since' argument
jid = remote.run_job(
minion_id, 'ceph.get_cluster_object', {
'cluster_name': self._cluster_name,
'sync_type': sync_type.str,
'since': None
})
except Unavailable:
# Don't throw an exception because if a fetch fails we should end up
# issuing another on next heartbeat
log.error("Failed to start fetch job %s/%s" %
(minion_id, sync_type))
else:
log.debug("SyncObjects.fetch: jid=%s" % jid)
def on_heartbeat(self, minion_id, cluster_data):
"""
Handle a ceph.heartbeat from a minion.
Heartbeats come from all servers, but we're mostly interested in those
which come from a mon (and therefore have the 'clusters' attribute populated)
as these tells us whether there are any new versions of cluster maps
for us to fetch.
"""
if not self._is_favorite(minion_id):
log.debug(
'Ignoring cluster data from %s, it is not my favourite (%s)' %
(minion_id, self._favorite_mon))
return
self.update_time = datetime.datetime.utcnow().replace(tzinfo=utc)
log.debug('Checking for version increments in heartbeat from %s' %
minion_id)
for sync_type in SYNC_OBJECT_TYPES:
self._sync_objects.on_version(
minion_id, sync_type, cluster_data['versions'][sync_type.str])
def on_version(self, reported_by, sync_type, new_version):
"""
Notify me that a particular version of a particular map exists.
I may choose to initiate RPC to retrieve the map
"""
log.debug("SyncObjects.on_version %s/%s/%s" %
(reported_by, sync_type.str, new_version))
old_version = self.get_version(sync_type)
if sync_type.cmp(new_version, old_version) > 0:
known_version = self._known_versions[sync_type]
if sync_type.cmp(new_version, known_version) > 0:
# We are out of date: request an up to date copy
log.info("Advanced known version %s/%s %s->%s" %
(self._cluster_name, sync_type.str, known_version,
new_version))
self._known_versions[sync_type] = new_version
else:
log.info("on_version: %s is newer than %s" %
(new_version, old_version))
# If we already have a request out for this type of map, then consider
# cancelling it if we've already waited for a while.
if self._fetching_at[sync_type] is not None:
if now() - self._fetching_at[sync_type] < self.FETCH_TIMEOUT:
log.info("Fetch already underway for %s" % sync_type.str)
return
else:
log.warn("Abandoning fetch for %s started at %s" %
(sync_type.str, self._fetching_at[sync_type]))
log.info(
"on_version: fetching %s/%s from %s, currently got %s, know %s"
% (sync_type, new_version, reported_by, old_version,
known_version))
self.fetch(reported_by, sync_type)
def _recover(self):
if sqlalchemy is None:
return
session = Session()
for server in session.query(Server).all():
log.debug("Recovered server %s" % server.fqdn)
assert server.boot_time is None or server.boot_time.tzinfo is not None # expect timezone-aware DB backend
self.servers.inject_server(
ServerState(fqdn=server.fqdn,
hostname=server.hostname,
managed=server.managed,
last_contact=server.last_contact,
boot_time=server.boot_time,
ceph_version=server.ceph_version))
for service in session.query(Service).all():
if service.server:
server = session.query(Server).get(service.server)
else:
server = None
log.debug("Recovered service %s/%s/%s on %s" %
(service.fsid, service.service_type, service.service_id,
server.fqdn if server else None))
self.servers.inject_service(
ServiceState(fsid=service.fsid,
service_type=service.service_type,
service_id=service.service_id),
server.fqdn if server else None)
# I want the most recent version of every sync_object
fsids = [(row[0], row[1]) for row in session.query(
SyncObject.fsid, SyncObject.cluster_name).distinct(SyncObject.fsid)
]
for fsid, name in fsids:
cluster_monitor = ClusterMonitor(fsid, name, self.persister,
self.servers, self.eventer,
self.requests)
self.clusters[fsid] = cluster_monitor
object_types = [
row[0]
for row in session.query(SyncObject.sync_type).filter_by(
fsid=fsid).distinct()
]
for sync_type in object_types:
latest_record = session.query(SyncObject).filter_by(
fsid=fsid,
sync_type=sync_type).order_by(SyncObject.version.desc(),
SyncObject.when.desc())[0]
# FIXME: bit of a hack because records persisted only store their 'version'
# if it's a real counter version, underlying problem is that we have
# underlying data (health, pg_brief) without usable version counters.
def md5(raw):
hasher = hashlib.md5()
hasher.update(raw)
return hasher.hexdigest()
if latest_record.version:
version = latest_record.version
else:
version = md5(latest_record.data)
when = latest_record.when
when = when.replace(tzinfo=tzutc())
if cluster_monitor.update_time is None or when > cluster_monitor.update_time:
cluster_monitor.update_time = when
cluster_monitor.inject_sync_object(
None, sync_type, version,
msgpack.unpackb(latest_record.data))
for monitor in self.clusters.values():
log.info("Recovery: Cluster %s with update time %s" %
(monitor.fsid, monitor.update_time))
monitor.start()
def _recover(self):
if sqlalchemy is None:
return
session = Session()
for server in session.query(Server).all():
log.debug("Recovered server %s" % server.fqdn)
assert server.boot_time is None or server.boot_time.tzinfo is not None # expect timezone-aware DB backend
self.servers.inject_server(ServerState(
fqdn=server.fqdn,
hostname=server.hostname,
managed=server.managed,
last_contact=server.last_contact,
boot_time=server.boot_time,
ceph_version=server.ceph_version
))
for service in session.query(Service).all():
if service.server:
server = session.query(Server).get(service.server)
else:
server = None
log.debug("Recovered service %s/%s/%s on %s" % (
service.fsid, service.service_type, service.service_id, server.fqdn if server else None
))
self.servers.inject_service(ServiceState(
fsid=service.fsid,
service_type=service.service_type,
service_id=service.service_id
), server.fqdn if server else None)
# I want the most recent version of every sync_object
fsids = [(row[0], row[1]) for row in session.query(SyncObject.fsid, SyncObject.cluster_name).distinct(SyncObject.fsid)]
for fsid, name in fsids:
cluster_monitor = ClusterMonitor(fsid, name, self.persister, self.servers,
self.eventer, self.requests)
self.clusters[fsid] = cluster_monitor
object_types = [row[0] for row in session.query(SyncObject.sync_type).filter_by(fsid=fsid).distinct()]
for sync_type in object_types:
latest_record = session.query(SyncObject).filter_by(
fsid=fsid, sync_type=sync_type).order_by(
SyncObject.version.desc(), SyncObject.when.desc())[0]
# FIXME: bit of a hack because records persisted only store their 'version'
# if it's a real counter version, underlying problem is that we have
# underlying data (health, pg_brief) without usable version counters.
def md5(raw):
hasher = hashlib.md5()
hasher.update(raw)
return hasher.hexdigest()
if latest_record.version:
version = latest_record.version
else:
version = md5(latest_record.data)
when = latest_record.when
when = when.replace(tzinfo=tzutc())
if cluster_monitor.update_time is None or when > cluster_monitor.update_time:
cluster_monitor.update_time = when
cluster_monitor.inject_sync_object(None, sync_type, version, msgpack.unpackb(latest_record.data))
for monitor in self.clusters.values():
log.info("Recovery: Cluster %s with update time %s" % (monitor.fsid, monitor.update_time))
monitor.start()
