def watchme(cls):
if cls.__name__ not in Metrics.watched_objects:
Metrics.watched_objects[cls.__name__] = {}
for method in dir(cls):
try:
if callable(getattr(cls, method)) and not method.startswith('_'):
hist = Histogram(f'{cls.__name__}_{method}_latency', f'Total latency of {cls.__name__}.{method}')
Metrics.watched_objects[cls.__name__] = hist
setattr(cls, method, hist.time()(getattr(cls, method)))
except TypeError:
print(f'Can not decorate {method}')
return cls
class Collector(object):
""" collector is a model running in thread and responsible for collecting
some metrics, we use thread because we do not want to let hanging in one
collector can not have impact on other collectors. This is base class,
real collector should inhernit this class and implement collect_impl,
metrics are returned as an array."""
def __init__(self, name, sleep_time, atomic_ref, iteration_counter):
self.name = name
self.sleep_time = sleep_time
self.atomic_ref = atomic_ref
self.iteration_counter = iteration_counter
histogram_key = "collector_%s_iteration_latency_seconds" % self.name
histogram_desc = "latency for execute one interation of %s collector (seconds)" % \
self.name
self.collector_histogram = Histogram(
histogram_key,
histogram_desc,
buckets=(.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5,
5.0, 7.5, 10.0, 12.5, 15.0, 17.5, 20.0, float("inf")))
logger.debug("init %s with sleep_time %d", self.name, self.sleep_time)
def collect(self):
while True:
logger.debug("collecting metrics from %s", self.name)
with self.collector_histogram.time():
self.iteration_counter.labels(name=self.name).inc()
try:
self.atomic_ref.set(self.collect_impl(),
datetime.datetime.now())
except Exception:
logger.exception("%s collector get an exception",
self.name)
logger.debug(
"finished collect metrics from %s, will sleep for %s",
self.name, self.sleep_time)
time.sleep(self.sleep_time)
def collect_impl(self):
""" implementations are expected to return an array of
prometheus_client's metrics or None on exception """
pass
from prometheus_client import Histogram
from prometheus_client import start_http_server, Summary
import random
import time
h = Histogram('request_latency_seconds', 'Description of histogram')
h.observe(4.7) # Observe 4.7 (seconds in this case)
@h.time()
def f():
pass
with h.time():
pass
if __name__ == '__main__':
# Start up the server to expose the metrics.
start_http_server(8000)
# Generate some requests.
while True:
h.observe(random.uniform(0, 5))
'''
http://localhost:8000/
# HELP python_gc_collected_objects Objects collected during gc
# TYPE python_gc_collected_objects histogram
# HELP python_gc_uncollectable_objects Uncollectable object found during GC
# TYPE python_gc_uncollectable_objects histogram
def run_wrapper(self, evidence):
"""Wrapper to manage TurbiniaTaskResults and exception handling.
This wrapper should be called to invoke the run() methods so it can handle
the management of TurbiniaTaskResults and the exception handling. Otherwise
details from exceptions in the worker cannot be propagated back to the
Turbinia TaskManager.
This method should handle (in no particular order):
- Exceptions thrown from run()
- Verifying valid TurbiniaTaskResult object is returned
- Check for bad results (non TurbiniaTaskResults) returned from run()
- Auto-close results that haven't been closed
- Verifying that the results are serializeable
- Locking to make sure only one task is active at a time
Args:
evidence (dict): To be decoded into Evidence object
Returns:
A TurbiniaTaskResult object
"""
# Avoid circular dependency.
from turbinia.jobs import manager as job_manager
log.debug('Task {0:s} {1:s} awaiting execution'.format(
self.name, self.id))
evidence = evidence_decode(evidence)
try:
self.result = self.setup(evidence)
self.result.update_task_status(self, 'queued')
turbinia_worker_tasks_queued_total.inc()
except TurbiniaException as exception:
message = (
'{0:s} Task setup failed with exception: [{1!s}]'.format(
self.name, exception))
# Logging explicitly here because the result is in an unknown state
trace = traceback.format_exc()
log.error(message)
log.error(trace)
if self.result:
if hasattr(exception, 'message'):
self.result.set_error(exception.message,
traceback.format_exc())
else:
self.result.set_error(exception.__class__,
traceback.format_exc())
self.result.status = message
else:
self.result = self.create_result(message=message,
trace=traceback.format_exc())
return self.result.serialize()
with filelock.FileLock(config.LOCK_FILE):
log.info('Starting Task {0:s} {1:s}'.format(self.name, self.id))
original_result_id = None
turbinia_worker_tasks_started_total.inc()
task_runtime_metrics = Histogram(
'{0:s}_duration_seconds'.format(self.name),
'Seconds to run {0:s}'.format(self.name))
with task_runtime_metrics.time():
try:
original_result_id = self.result.id
# Check if Task's job is available for the worker.
active_jobs = list(job_manager.JobsManager.GetJobNames())
if self.job_name.lower() not in active_jobs:
message = (
'Task will not run due to the job: {0:s} being disabled '
'on the worker.'.format(self.job_name))
self.result.log(message, level=logging.ERROR)
self.result.status = message
return self.result.serialize()
self.evidence_setup(evidence)
if self.turbinia_version != turbinia.__version__:
message = (
'Worker and Server versions do not match: {0:s} != {1:s}'
.format(self.turbinia_version,
turbinia.__version__))
self.result.log(message, level=logging.ERROR)
self.result.status = message
return self.result.serialize()
self.result.update_task_status(self, 'running')
self._evidence_config = evidence.config
self.result = self.run(evidence, self.result)
# pylint: disable=broad-except
except Exception as exception:
message = (
'{0:s} Task failed with exception: [{1!s}]'.format(
self.name, exception))
# Logging explicitly here because the result is in an unknown state
trace = traceback.format_exc()
log_and_report(message, trace)
if self.result:
self.result.log(message, level=logging.ERROR)
self.result.log(trace)
if hasattr(exception, 'message'):
self.result.set_error(exception.message,
traceback.format_exc())
else:
self.result.set_error(exception.__class__,
traceback.format_exc())
self.result.status = message
else:
log.error(
'No TurbiniaTaskResult object found after task execution.'
)
self.result = self.validate_result(self.result)
if self.result:
self.result.update_task_status(self)
# Trying to close the result if possible so that we clean up what we can.
# This has a higher likelihood of failing because something must have gone
# wrong as the Task should have already closed this.
if self.result and not self.result.closed:
message = 'Trying last ditch attempt to close result'
log.warning(message)
self.result.log(message)
if self.result.status:
status = self.result.status
else:
status = 'No previous status'
message = (
'Task Result was auto-closed from task executor on {0:s} likely '
'due to previous failures. Previous status: [{1:s}]'.
format(self.result.worker_name, status))
self.result.log(message)
try:
self.result.close(self, False, message)
# Using broad except here because lots can go wrong due to the reasons
# listed above.
# pylint: disable=broad-except
except Exception as exception:
log.error('TurbiniaTaskResult close failed: {0!s}'.format(
exception))
if not self.result.status:
self.result.status = message
# Check the result again after closing to make sure it's still good.
self.result = self.validate_result(self.result)
if original_result_id != self.result.id:
log.debug(
'Result object {0:s} is different from original {1!s} after task '
'execution which indicates errors during execution'.format(
self.result.id, original_result_id))
else:
log.debug(
'Returning original result object {0:s} after task execution'.
format(self.result.id))
return self.result.serialize()
class TestHistogram(unittest.TestCase):
def setUp(self):
self.registry = CollectorRegistry()
self.histogram = Histogram('h', 'help', registry=self.registry)
self.labels = Histogram('hl', 'help', ['l'], registry=self.registry)
def test_histogram(self):
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(0, self.registry.get_sample_value('h_sum'))
self.histogram.observe(2)
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(2, self.registry.get_sample_value('h_sum'))
self.histogram.observe(2.5)
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(2, self.registry.get_sample_value('h_count'))
self.assertEqual(4.5, self.registry.get_sample_value('h_sum'))
self.histogram.observe(float("inf"))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(3, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(3, self.registry.get_sample_value('h_count'))
self.assertEqual(float("inf"), self.registry.get_sample_value('h_sum'))
def test_setting_buckets(self):
h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2])
self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds)
h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2, float("inf")])
self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds)
self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[])
self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[float("inf")])
self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[3, 1])
def test_labels(self):
self.labels.labels('a').observe(2)
self.assertEqual(0, self.registry.get_sample_value('hl_bucket', {'le': '1.0', 'l': 'a'}))
self.assertEqual(1, self.registry.get_sample_value('hl_bucket', {'le': '2.5', 'l': 'a'}))
self.assertEqual(1, self.registry.get_sample_value('hl_bucket', {'le': '5.0', 'l': 'a'}))
self.assertEqual(1, self.registry.get_sample_value('hl_bucket', {'le': '+Inf', 'l': 'a'}))
self.assertEqual(1, self.registry.get_sample_value('hl_count', {'l': 'a'}))
self.assertEqual(2, self.registry.get_sample_value('hl_sum', {'l': 'a'}))
def test_function_decorator(self):
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
@self.histogram.time()
def f():
pass
f()
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
def test_block_decorator(self):
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
with self.histogram.time():
pass
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
# {rising: 1, falling: -1, high: 2, low: -2}
if abs(df) > 0.25:
status = math.copysign(1, df)
elif dfdt(t - 1) > 0:
status = 2
else:
status = -2
return level, status
def dfdt(t):
return 2 * (math.cos(t / 2) * math.cos(t - 2)) / \
(2 * math.sqrt(1 - math.sin(t / 2) ** 2)) - \
4 * math.asin(math.sin(t / 2)) * math.sin(t - 2)
if __name__ == '__main__':
# Start up the server to expose the metrics.
start_http_server(8000)
# Generate some requests.
while True:
with REQUEST_HIST.time():
tide_level, tide_status = measure_sea_level(time.time())
LEVEL.set(tide_level)
STATUS.set(tide_status)
if tide_level > 4 and tide_status == 2:
print('Extreme High Tide')
class TestHistogram(unittest.TestCase):
def setUp(self):
self.registry = CollectorRegistry()
self.histogram = Histogram('h', 'help', registry=self.registry)
self.labels = Histogram('hl', 'help', ['l'], registry=self.registry)
def test_histogram(self):
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(0, self.registry.get_sample_value('h_sum'))
self.histogram.observe(2)
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(2, self.registry.get_sample_value('h_sum'))
self.histogram.observe(2.5)
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(2, self.registry.get_sample_value('h_count'))
self.assertEqual(4.5, self.registry.get_sample_value('h_sum'))
self.histogram.observe(float("inf"))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(
3, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(3, self.registry.get_sample_value('h_count'))
self.assertEqual(float("inf"), self.registry.get_sample_value('h_sum'))
def test_setting_buckets(self):
h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2])
self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds)
h = Histogram('h',
'help',
registry=None,
buckets=[0, 1, 2, float("inf")])
self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds)
self.assertRaises(ValueError,
Histogram,
'h',
'help',
registry=None,
buckets=[])
self.assertRaises(ValueError,
Histogram,
'h',
'help',
registry=None,
buckets=[float("inf")])
self.assertRaises(ValueError,
Histogram,
'h',
'help',
registry=None,
buckets=[3, 1])
def test_labels(self):
self.labels.labels('a').observe(2)
self.assertEqual(
0,
self.registry.get_sample_value('hl_bucket', {
'le': '1.0',
'l': 'a'
}))
self.assertEqual(
1,
self.registry.get_sample_value('hl_bucket', {
'le': '2.5',
'l': 'a'
}))
self.assertEqual(
1,
self.registry.get_sample_value('hl_bucket', {
'le': '5.0',
'l': 'a'
}))
self.assertEqual(
1,
self.registry.get_sample_value('hl_bucket', {
'le': '+Inf',
'l': 'a'
}))
self.assertEqual(
1, self.registry.get_sample_value('hl_count', {'l': 'a'}))
self.assertEqual(2,
self.registry.get_sample_value('hl_sum', {'l': 'a'}))
def test_function_decorator(self):
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
@self.histogram.time()
def f():
pass
f()
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
def test_block_decorator(self):
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
with self.histogram.time():
pass
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
