def __init__(self,
health_checker,
sandbox=None,
interval_secs=10,
initial_interval_secs=None,
max_consecutive_failures=0,
clock=time):
self._health_checks = 0
self._total_latency = 0
self._stats_lock = threading.Lock()
self._clock = clock
self.threaded_health_checker = ThreadedHealthChecker(
self._timing_wrapper(health_checker), sandbox, interval_secs,
initial_interval_secs, max_consecutive_failures, clock)
self.metrics.register(
LambdaGauge(
'consecutive_failures', lambda: self.threaded_health_checker.
current_consecutive_failures))
self.metrics.register(
LambdaGauge('snoozed',
lambda: int(self.threaded_health_checker.snoozed)))
self.metrics.register(
LambdaGauge('total_latency_secs', lambda: self._total_latency))
self.metrics.register(
LambdaGauge('checks', lambda: self._health_checks))
def _init_metrics(self):
self._session_expirations = AtomicGauge('session_expirations')
self._connection_losses = AtomicGauge('connection_losses')
self.metrics.register(self._session_expirations)
self.metrics.register(self._connection_losses)
self.metrics.register(
LambdaGauge('session_id', lambda: self.session_id))
self.metrics.register(
LambdaGauge('live', lambda: int(self._live.is_set())))
def register_diagnostics():
import os, sys, time
rm = RootMetrics().scope('sys')
now = time.time()
rm.register(LambdaGauge('uptime', lambda: time.time() - now))
rm.register(Label('argv', repr(sys.argv)))
rm.register(Label('path', repr(sys.path)))
rm.register(LambdaGauge('modules', lambda: ', '.join(sys.modules.keys())))
rm.register(Label('version', sys.version))
rm.register(Label('platform', sys.platform))
rm.register(Label('executable', sys.executable))
rm.register(Label('prefix', sys.prefix))
rm.register(Label('exec_prefix', sys.exec_prefix))
rm.register(Label('uname', ' '.join(os.uname())))
def __init__(self, clock=time):
self._clock = clock
self._self = psutil.Process(os.getpid())
self._orphan = False
self.metrics.register(LambdaGauge('orphan', lambda: int(self._orphan)))
self._metrics = dict((metric, MutatorGauge(metric, 0)) for metric in self.MUTATOR_METRICS)
for metric in self._metrics.values():
self.metrics.register(metric)
ExceptionalThread.__init__(self)
self.daemon = True
def __init__(self, client, path, timeout_secs, endpoint, additional=None, shard=None, name=None):
self.__client = client
self.__connect_event = client.start_async()
self.__timeout_secs = timeout_secs
self.__announcer = Announcer(ServerSet(client, path), endpoint, additional=additional,
shard=shard)
self.__name = name or self.DEFAULT_NAME
self.__status = None
self.start_event = threading.Event()
self.metrics.register(LambdaGauge('disconnected_time', self.__announcer.disconnected_time))
def __init__(self, *args, **kw):
super(TwitterKazooClient, self).__init__(*args, **kw)
self.connecting = threading.Event()
self.__session_expirations = AtomicGauge('session_expirations')
self.__connection_losses = AtomicGauge('connection_losses')
self.__session_id = LambdaGauge('session_id', lambda:
(self._session_id or 0))
self.metrics.register(self.__session_expirations)
self.metrics.register(self.__connection_losses)
self.metrics.register(self.__session_id)
self.add_listener(self._observable_listener)
def register_diagnostics(metrics):
rm = metrics.scope('sys')
now = time.time()
rm.register(LambdaGauge('uptime', lambda: time.time() - now))
rm.register(Label('argv', repr(sys.argv)))
rm.register(Label('path', repr(sys.path)))
rm.register(Label('version', sys.version))
rm.register(Label('platform', sys.platform))
rm.register(Label('executable', sys.executable))
rm.register(Label('prefix', sys.prefix))
rm.register(Label('exec_prefix', sys.exec_prefix))
rm.register(Label('uname', ' '.join(os.uname())))
def __init__(self,
serverset,
endpoint,
additional=None,
shard=None,
name=None):
self.__announcer = Announcer(serverset,
endpoint,
additional=additional,
shard=shard)
self.__name = name or self.DEFAULT_NAME
self.metrics.register(
LambdaGauge('disconnected_time',
self.__announcer.disconnected_time))
def _setup_metrics(self):
self._metrics = self.Metrics()
self._metrics.cluster_count = self.metrics.register(
AtomicGauge('cluster_count', 0))
# Total resources requested by the scheduler's clients. When a cluster is created its resources
# are added to the total; when it's deleted its resources are subtracted from the total.
# NOTE: These are 'requested' resources that are independent of resources offered by Mesos or
# allocated to or used by Mysos tasks running on Mesos cluster.
self._metrics.total_requested_cpus = self.metrics.register(
MutatorGauge('total_requested_cpus', 0.))
self._metrics.total_requested_mem_mb = self.metrics.register(
MutatorGauge('total_requested_mem_mb', 0.))
self._metrics.total_requested_disk_mb = self.metrics.register(
MutatorGauge('total_requested_disk_mb', 0.))
# 1: registered; 0: not registered.
self._metrics.framework_registered = self.metrics.register(
MutatorGauge('framework_registered', 0))
self._startup_time = datetime.utcnow()
self._metrics.uptime = self.metrics.register(
LambdaGauge(
'uptime', lambda:
(datetime.utcnow() - self._startup_time).total_seconds()))
# Counters for tasks in terminal states.
self._metrics.tasks_lost = self.metrics.register(
AtomicGauge('tasks_lost', 0))
self._metrics.tasks_finished = self.metrics.register(
AtomicGauge('tasks_finished', 0))
self._metrics.tasks_failed = self.metrics.register(
AtomicGauge('tasks_failed', 0))
self._metrics.tasks_killed = self.metrics.register(
AtomicGauge('tasks_killed', 0))
self._metrics.resource_offers = self.metrics.register(
AtomicGauge('resource_offers', 0))
self._metrics.offers_incompatible_role = self.metrics.register(
AtomicGauge('offers_incompatible_role', 0))
self._metrics.tasks_launched = self.metrics.register(
AtomicGauge('tasks_launched', 0))
# 'offers_unused' are due to idle scheduler or resources don't fit, i.e.,
# 'resource_offers' - 'tasks_launched' - 'offers_incompatible_role'.
self._metrics.offers_unused = self.metrics.register(
AtomicGauge('offers_unused', 0))
def __init__(self, clock=time):
self._clock = clock
self._self = psutil.Process(os.getpid())
try:
self._version = self.get_release_from_binary(
os.path.join(self._self.cwd(), self._self.cmdline()[1]))
except (IndexError, psutil.Error):
self._version = 'UNKNOWN'
self.metrics.register(NamedGauge('version', self._version))
self._orphan = False
self.metrics.register(LambdaGauge('orphan', lambda: int(self._orphan)))
self._metrics = dict((metric, MutatorGauge(metric, 0)) for metric in self.MUTATOR_METRICS)
for metric in self._metrics.values():
self.metrics.register(metric)
ExceptionalThread.__init__(self)
self.daemon = True
def register_metrics(self):
self.metrics.register(
LambdaGauge('disk_used', lambda: self._disk_sample))
self.metrics.register(
LambdaGauge('disk_reserved', lambda: self._max_disk))
self.metrics.register(
LambdaGauge('disk_percent',
lambda: 1.0 * self._disk_sample / self._max_disk))
self.metrics.register(
LambdaGauge('cpu_used', lambda: self._ps_sample.rate))
self.metrics.register(
LambdaGauge('cpu_reserved', lambda: self._max_cpu))
self.metrics.register(
LambdaGauge('cpu_percent',
lambda: 1.0 * self._ps_sample.rate / self._max_cpu))
self.metrics.register(
LambdaGauge('ram_used', lambda: self._ps_sample.rss))
self.metrics.register(
LambdaGauge('ram_reserved', lambda: self._max_ram))
self.metrics.register(
LambdaGauge('ram_percent',
lambda: 1.0 * self._ps_sample.rss / self._max_ram))
def __init__(self, *args, **kw):
if 'connection_retry' not in kw:
# The default backoff delay limit in kazoo is 3600 seconds, which is generally
# too conservative for our use cases. If not supplied by the caller, provide
# a backoff that will truncate earlier.
kw['connection_retry'] = KazooRetry(
max_delay=DEFAULT_RETRY_MAX_DELAY_SECS, **DEFAULT_RETRY_DICT)
super(TwitterKazooClient, self).__init__(*args, **kw)
self.connecting = threading.Event()
self.__session_expirations = AtomicGauge('session_expirations')
self.__connection_losses = AtomicGauge('connection_losses')
self.__session_id = LambdaGauge('session_id', lambda: (self._session_id or 0))
self.metrics.register(self.__session_expirations)
self.metrics.register(self.__connection_losses)
self.metrics.register(self.__session_id)
self.add_listener(self._observable_listener)
