def test_setting_buckets(self):
h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2])
self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds)
h = Histogram('h',
'help',
registry=None,
buckets=[0, 1, 2, float("inf")])
self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds)
self.assertRaises(ValueError,
Histogram,
'h',
'help',
registry=None,
buckets=[])
self.assertRaises(ValueError,
Histogram,
'h',
'help',
registry=None,
buckets=[float("inf")])
self.assertRaises(ValueError,
Histogram,
'h',
'help',
registry=None,
buckets=[3, 1])
def test_duplicate_metrics_raises(self):
registry = CollectorRegistry()
Counter('c_total', 'help', registry=registry)
self.assertRaises(ValueError, Counter, 'c_total', 'help', registry=registry)
self.assertRaises(ValueError, Gauge, 'c_total', 'help', registry=registry)
self.assertRaises(ValueError, Gauge, 'c_created', 'help', registry=registry)
Gauge('g_created', 'help', registry=registry)
self.assertRaises(ValueError, Gauge, 'g_created', 'help', registry=registry)
self.assertRaises(ValueError, Counter, 'g', 'help', registry=registry)
Summary('s', 'help', registry=registry)
self.assertRaises(ValueError, Summary, 's', 'help', registry=registry)
self.assertRaises(ValueError, Gauge, 's_created', 'help', registry=registry)
self.assertRaises(ValueError, Gauge, 's_sum', 'help', registry=registry)
self.assertRaises(ValueError, Gauge, 's_count', 'help', registry=registry)
# We don't currently expose quantiles, but let's prevent future
# clashes anyway.
self.assertRaises(ValueError, Gauge, 's', 'help', registry=registry)
Histogram('h', 'help', registry=registry)
self.assertRaises(ValueError, Histogram, 'h', 'help', registry=registry)
# Clashes aggaint various suffixes.
self.assertRaises(ValueError, Summary, 'h', 'help', registry=registry)
self.assertRaises(ValueError, Gauge, 'h_count', 'help', registry=registry)
self.assertRaises(ValueError, Gauge, 'h_sum', 'help', registry=registry)
self.assertRaises(ValueError, Gauge, 'h_bucket', 'help', registry=registry)
self.assertRaises(ValueError, Gauge, 'h_created', 'help', registry=registry)
# The name of the histogram itself is also taken.
self.assertRaises(ValueError, Gauge, 'h', 'help', registry=registry)
Info('i', 'help', registry=registry)
self.assertRaises(ValueError, Gauge, 'i_info', 'help', registry=registry)
def test_histogram_adds(self):
h1 = Histogram('h', 'help', registry=None)
h2 = Histogram('h', 'help', registry=None)
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(0, self.registry.get_sample_value('h_sum'))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
h1.observe(1)
h2.observe(2)
self.assertEqual(2, self.registry.get_sample_value('h_count'))
self.assertEqual(3, self.registry.get_sample_value('h_sum'))
self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
def test_histogram_adds(self):
h1 = Histogram('h', 'help', registry=None)
core._ValueClass = core._MultiProcessValue(lambda: 456)
h2 = Histogram('h', 'help', registry=None)
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(0, self.registry.get_sample_value('h_sum'))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
h1.observe(1)
h2.observe(2)
self.assertEqual(2, self.registry.get_sample_value('h_count'))
self.assertEqual(3, self.registry.get_sample_value('h_sum'))
self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
def test_reset_registry_with_labels(self):
registry = CollectorRegistry()
gauge = Gauge('g', 'help', ['l'], registry=registry)
gauge.labels('a').inc()
self.assertEqual(1, registry.get_sample_value('g', {'l': 'a'}))
counter = Counter('c_total', 'help', ['l'], registry=registry)
counter.labels('a').inc()
self.assertEqual(1, registry.get_sample_value('c_total', {'l': 'a'}))
summary = Summary('s', 'help', ['l'], registry=registry)
summary.labels('a').observe(10)
self.assertEqual(1, registry.get_sample_value('s_count', {'l': 'a'}))
self.assertEqual(10, registry.get_sample_value('s_sum', {'l': 'a'}))
histogram = Histogram('h', 'help', ['l'], registry=registry)
histogram.labels('a').observe(2)
self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '1.0', 'l': 'a'}))
self.assertEqual(1, registry.get_sample_value('h_bucket', {'le': '2.5', 'l': 'a'}))
self.assertEqual(1, registry.get_sample_value('h_bucket', {'le': '5.0', 'l': 'a'}))
self.assertEqual(1, registry.get_sample_value('h_bucket', {'le': '+Inf', 'l': 'a'}))
self.assertEqual(1, registry.get_sample_value('h_count', {'l': 'a'}))
self.assertEqual(2, registry.get_sample_value('h_sum', {'l': 'a'}))
registry.reset()
self.assertEqual(0, registry.get_sample_value('g', {'l': 'a'}))
self.assertEqual(0, registry.get_sample_value('c_total', {'l': 'a'}))
self.assertEqual(0, registry.get_sample_value('s_count', {'l': 'a'}))
self.assertEqual(0, registry.get_sample_value('s_sum', {'l': 'a'}))
self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '1.0', 'l': 'a'}))
self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '2.5', 'l': 'a'}))
self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '5.0', 'l': 'a'}))
self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '+Inf', 'l': 'a'}))
self.assertEqual(0, registry.get_sample_value('h_count', {'l': 'a'}))
self.assertEqual(0, registry.get_sample_value('h_sum', {'l': 'a'}))
def test_collect(self):
pid = 0
core._ValueClass = core._MultiProcessValue(lambda: pid)
labels = dict((i, i) for i in 'abcd')
def add_label(key, value):
l = labels.copy()
l[key] = value
return l
c = Counter('c', 'help', labelnames=labels.keys(), registry=None)
g = Gauge('g', 'help', labelnames=labels.keys(), registry=None)
h = Histogram('h', 'help', labelnames=labels.keys(), registry=None)
c.labels(**labels).inc(1)
g.labels(**labels).set(1)
h.labels(**labels).observe(1)
pid = 1
c.labels(**labels).inc(1)
g.labels(**labels).set(1)
h.labels(**labels).observe(5)
metrics = dict((m.name, m) for m in self.collector.collect())
self.assertEqual(
metrics['c'].samples, [Sample('c_total', labels, 2.0)]
)
metrics['g'].samples.sort(key=lambda x: x[1]['pid'])
self.assertEqual(metrics['g'].samples, [
Sample('g', add_label('pid', '0'), 1.0),
Sample('g', add_label('pid', '1'), 1.0),
])
metrics['h'].samples.sort(
key=lambda x: (x[0], float(x[1].get('le', 0)))
)
expected_histogram = [
Sample('h_bucket', add_label('le', '0.005'), 0.0),
Sample('h_bucket', add_label('le', '0.01'), 0.0),
Sample('h_bucket', add_label('le', '0.025'), 0.0),
Sample('h_bucket', add_label('le', '0.05'), 0.0),
Sample('h_bucket', add_label('le', '0.075'), 0.0),
Sample('h_bucket', add_label('le', '0.1'), 0.0),
Sample('h_bucket', add_label('le', '0.25'), 0.0),
Sample('h_bucket', add_label('le', '0.5'), 0.0),
Sample('h_bucket', add_label('le', '0.75'), 0.0),
Sample('h_bucket', add_label('le', '1.0'), 1.0),
Sample('h_bucket', add_label('le', '2.5'), 1.0),
Sample('h_bucket', add_label('le', '5.0'), 2.0),
Sample('h_bucket', add_label('le', '7.5'), 2.0),
Sample('h_bucket', add_label('le', '10.0'), 2.0),
Sample('h_bucket', add_label('le', '+Inf'), 2.0),
Sample('h_count', labels, 2.0),
Sample('h_sum', labels, 6.0),
]
self.assertEqual(metrics['h'].samples, expected_histogram)
def __init__(self, indexer, logger=getLogger(), metrics_registry=CollectorRegistry()):
self.__indexer = indexer
self.__logger = logger
self.__metrics_registry = metrics_registry
# metrics
self.__metrics_requests_total = Counter(
'{0}_indexer_grpc_requests_total'.format(NAME),
'The number of requests.',
[
'func'
],
registry=self.__metrics_registry
)
self.__metrics_requests_duration_seconds = Histogram(
'{0}_indexer_grpc_requests_duration_seconds'.format(NAME),
'The invocation duration in seconds.',
[
'func'
],
registry=self.__metrics_registry
)
def test_merge_no_accumulate(self):
pid = 0
core._ValueClass = core._MultiProcessValue(lambda: pid)
labels = dict((i, i) for i in 'abcd')
def add_label(key, value):
l = labels.copy()
l[key] = value
return l
h = Histogram('h', 'help', labelnames=labels.keys(), registry=None)
h.labels(**labels).observe(1)
pid = 1
h.labels(**labels).observe(5)
path = os.path.join(os.environ['prometheus_multiproc_dir'], '*.db')
files = glob.glob(path)
metrics = dict(
(m.name, m) for m in self.collector.merge(files, accumulate=False)
)
metrics['h'].samples.sort(
key=lambda x: (x[0], float(x[1].get('le', 0)))
)
expected_histogram = [
Sample('h_bucket', add_label('le', '0.005'), 0.0),
Sample('h_bucket', add_label('le', '0.01'), 0.0),
Sample('h_bucket', add_label('le', '0.025'), 0.0),
Sample('h_bucket', add_label('le', '0.05'), 0.0),
Sample('h_bucket', add_label('le', '0.075'), 0.0),
Sample('h_bucket', add_label('le', '0.1'), 0.0),
Sample('h_bucket', add_label('le', '0.25'), 0.0),
Sample('h_bucket', add_label('le', '0.5'), 0.0),
Sample('h_bucket', add_label('le', '0.75'), 0.0),
Sample('h_bucket', add_label('le', '1.0'), 1.0),
Sample('h_bucket', add_label('le', '2.5'), 0.0),
Sample('h_bucket', add_label('le', '5.0'), 1.0),
Sample('h_bucket', add_label('le', '7.5'), 0.0),
Sample('h_bucket', add_label('le', '10.0'), 0.0),
Sample('h_bucket', add_label('le', '+Inf'), 0.0),
Sample('h_sum', labels, 6.0),
]
self.assertEqual(metrics['h'].samples, expected_histogram)
def test_merge_no_accumulate(self):
self.pid = 0
labels = dict((i, i) for i in 'abcd')
def add_label(key, value):
l = labels.copy()
l[key] = value
return l
h = Histogram('hna', 'help', labelnames=labels.keys(), registry=None)
h.labels(**labels).observe(1)
self.pid = 1
h.labels(**labels).observe(5)
self.collector.accumulate = False
metrics = self.collector.collect()
self.collector.accumulate = True
metric = [x for x in metrics if x.name == 'hna'][0]
metric.samples.sort(
key=lambda x: (x[0], float(x[1].get('le', 0)))
)
expected_histogram = [
Sample('hna_bucket', add_label('le', '0.005'), 0.0),
Sample('hna_bucket', add_label('le', '0.01'), 0.0),
Sample('hna_bucket', add_label('le', '0.025'), 0.0),
Sample('hna_bucket', add_label('le', '0.05'), 0.0),
Sample('hna_bucket', add_label('le', '0.075'), 0.0),
Sample('hna_bucket', add_label('le', '0.1'), 0.0),
Sample('hna_bucket', add_label('le', '0.25'), 0.0),
Sample('hna_bucket', add_label('le', '0.5'), 0.0),
Sample('hna_bucket', add_label('le', '0.75'), 0.0),
Sample('hna_bucket', add_label('le', '1.0'), 1.0),
Sample('hna_bucket', add_label('le', '2.5'), 0.0),
Sample('hna_bucket', add_label('le', '5.0'), 1.0),
Sample('hna_bucket', add_label('le', '7.5'), 0.0),
Sample('hna_bucket', add_label('le', '10.0'), 0.0),
Sample('hna_bucket', add_label('le', '+Inf'), 0.0),
Sample('hna_sum', labels, 6.0),
]
self.assertEqual(metric.samples, expected_histogram)
class TestHistogram(unittest.TestCase):
def setUp(self):
self.registry = CollectorRegistry()
self.histogram = Histogram('h', 'help', registry=self.registry)
self.labels = Histogram('hl', 'help', ['l'], registry=self.registry)
def test_histogram(self):
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(0, self.registry.get_sample_value('h_sum'))
self.histogram.observe(2)
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(2, self.registry.get_sample_value('h_sum'))
self.histogram.observe(2.5)
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(2, self.registry.get_sample_value('h_count'))
self.assertEqual(4.5, self.registry.get_sample_value('h_sum'))
self.histogram.observe(float("inf"))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(
3, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(3, self.registry.get_sample_value('h_count'))
self.assertEqual(float("inf"), self.registry.get_sample_value('h_sum'))
def test_setting_buckets(self):
h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2])
self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds)
h = Histogram('h',
'help',
registry=None,
buckets=[0, 1, 2, float("inf")])
self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds)
self.assertRaises(ValueError,
Histogram,
'h',
'help',
registry=None,
buckets=[])
self.assertRaises(ValueError,
Histogram,
'h',
'help',
registry=None,
buckets=[float("inf")])
self.assertRaises(ValueError,
Histogram,
'h',
'help',
registry=None,
buckets=[3, 1])
def test_labels(self):
self.labels.labels('a').observe(2)
self.assertEqual(
0,
self.registry.get_sample_value('hl_bucket', {
'le': '1.0',
'l': 'a'
}))
self.assertEqual(
1,
self.registry.get_sample_value('hl_bucket', {
'le': '2.5',
'l': 'a'
}))
self.assertEqual(
1,
self.registry.get_sample_value('hl_bucket', {
'le': '5.0',
'l': 'a'
}))
self.assertEqual(
1,
self.registry.get_sample_value('hl_bucket', {
'le': '+Inf',
'l': 'a'
}))
self.assertEqual(
1, self.registry.get_sample_value('hl_count', {'l': 'a'}))
self.assertEqual(2,
self.registry.get_sample_value('hl_sum', {'l': 'a'}))
def test_function_decorator(self):
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
@self.histogram.time()
def f():
pass
self.assertEqual(([], None, None, None), inspect.getargspec(f))
f()
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
def test_block_decorator(self):
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
with self.histogram.time():
pass
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
generate_latest,
CONTENT_TYPE_LATEST,
)
from prometheus_client.core import GaugeMetricFamily, Histogram, Counter
from prometheus_client.multiprocess import MultiProcessCollector
from sqlalchemy import cast, String
from models import count_groups
from models.payment import Payment
from models.product import Product
from models.purchase import Purchase, AdmissionTicket
from models.cfp import Proposal
metrics = Blueprint("metric", __name__)
request_duration = Histogram("emf_request_duration_seconds",
"Request duration", ["endpoint", "method"])
request_total = Counter("emf_request_total", "Total request count",
["endpoint", "method", "http_status"])
def gauge_groups(gauge, query, *entities):
for count, *key in count_groups(query, *entities):
gauge.add_metric(key, count)
class ExternalMetrics:
def __init__(self, registry=None):
if registry is not None:
registry.register(self)
def collect(self):
class TestHistogram(unittest.TestCase):
def setUp(self):
self.registry = CollectorRegistry()
self.histogram = Histogram('h', 'help', registry=self.registry)
self.labels = Histogram('hl', 'help', ['l'], registry=self.registry)
def test_repr(self):
self.assertEqual(repr(self.histogram), "prometheus_client.metrics.Histogram(h)")
self.assertEqual(repr(self.labels), "prometheus_client.metrics.Histogram(hl)")
def test_histogram(self):
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(0, self.registry.get_sample_value('h_sum'))
self.histogram.observe(2)
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(2, self.registry.get_sample_value('h_sum'))
self.histogram.observe(2.5)
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(2, self.registry.get_sample_value('h_count'))
self.assertEqual(4.5, self.registry.get_sample_value('h_sum'))
self.histogram.observe(float("inf"))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(3, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(3, self.registry.get_sample_value('h_count'))
self.assertEqual(float("inf"), self.registry.get_sample_value('h_sum'))
def test_histogram_not_observable(self):
""".observe() must fail if the Summary is not observable."""
assert_not_observable(self.labels.observe, 1)
def test_setting_buckets(self):
h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2])
self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds)
h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2, float("inf")])
self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds)
self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[])
self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[float("inf")])
self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[3, 1])
def test_labels(self):
self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, labelnames=['le'])
self.labels.labels('a').observe(2)
self.assertEqual(0, self.registry.get_sample_value('hl_bucket', {'le': '1.0', 'l': 'a'}))
self.assertEqual(1, self.registry.get_sample_value('hl_bucket', {'le': '2.5', 'l': 'a'}))
self.assertEqual(1, self.registry.get_sample_value('hl_bucket', {'le': '5.0', 'l': 'a'}))
self.assertEqual(1, self.registry.get_sample_value('hl_bucket', {'le': '+Inf', 'l': 'a'}))
self.assertEqual(1, self.registry.get_sample_value('hl_count', {'l': 'a'}))
self.assertEqual(2, self.registry.get_sample_value('hl_sum', {'l': 'a'}))
def test_function_decorator(self):
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
@self.histogram.time()
def f():
pass
self.assertEqual(([], None, None, None), getargspec(f))
f()
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
def test_function_decorator_multithread(self):
self.assertEqual(0, self.registry.get_sample_value('h_count'))
workers = 3
duration = 0.1
pool = ThreadPoolExecutor(max_workers=workers)
@self.histogram.time()
def f():
time.sleep(duration)
jobs = workers * 3
for i in range(jobs):
pool.submit(f)
pool.shutdown(wait=True)
self.assertEqual(jobs, self.registry.get_sample_value('h_count'))
rounding_coefficient = 0.9
total_expected_duration = jobs * duration * rounding_coefficient
self.assertLess(total_expected_duration, self.registry.get_sample_value('h_sum'))
def test_block_decorator(self):
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
with self.histogram.time():
pass
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
def test_exemplar_invalid_label_name(self):
self.assertRaises(ValueError, self.histogram.observe, 3.0, exemplar={':o)': 'smile'})
self.assertRaises(ValueError, self.histogram.observe, 3.0, exemplar={'1': 'number'})
def test_exemplar_too_long(self):
# 129 characters in total should fail.
self.assertRaises(ValueError, self.histogram.observe, 1.0, exemplar={
'abcdefghijklmnopqrstuvwxyz': '26+16 characters',
'x1234567': '8+15 characters',
'zyxwvutsrqponmlkjihgfedcba': '26+16 characters',
'y123456': '7+15 characters',
})
def __init__(self,
host='localhost',
port=7070,
seed_addr=None,
conf=SyncObjConf(),
data_dir='/tmp/cockatrice/index',
grpc_port=5050,
grpc_max_workers=10,
http_port=8080,
logger=getLogger(),
http_logger=getLogger(),
metrics_registry=CollectorRegistry()):
self.__host = host
self.__port = port
self.__seed_addr = seed_addr
self.__conf = conf
self.__data_dir = data_dir
self.__grpc_port = grpc_port
self.__grpc_max_workers = grpc_max_workers
self.__http_port = http_port
self.__logger = logger
self.__http_logger = http_logger
self.__metrics_registry = metrics_registry
# metrics
self.__metrics_core_documents = Gauge(
'{0}_indexer_index_documents'.format(NAME),
'The number of documents.', [
'index_name',
],
registry=self.__metrics_registry)
self.__metrics_requests_total = Counter(
'{0}_indexer_requests_total'.format(NAME),
'The number of requests.', ['func'],
registry=self.__metrics_registry)
self.__metrics_requests_duration_seconds = Histogram(
'{0}_indexer_requests_duration_seconds'.format(NAME),
'The invocation duration in seconds.', ['func'],
registry=self.__metrics_registry)
self.__self_addr = '{0}:{1}'.format(self.__host, self.__port)
self.__peer_addrs = [] if self.__seed_addr is None else get_peers(
bind_addr=self.__seed_addr, timeout=10)
self.__other_addrs = [
peer_addr for peer_addr in self.__peer_addrs
if peer_addr != self.__self_addr
]
self.__conf.serializer = self.__serialize
self.__conf.deserializer = self.__deserialize
self.__conf.validate()
self.__indices = {}
self.__index_configs = {}
self.__writers = {}
self.__auto_commit_timers = {}
self.__lock = RLock()
# create data dir
os.makedirs(self.__data_dir, exist_ok=True)
self.__file_storage = FileStorage(self.__data_dir,
supports_mmap=True,
readonly=False,
debug=False)
self.__ram_storage = RamStorage()
# if seed addr specified and self node does not exist in the cluster, add self node to the cluster
if self.__seed_addr is not None and self.__self_addr not in self.__peer_addrs:
Thread(target=add_node,
kwargs={
'node_name': self.__self_addr,
'bind_addr': self.__seed_addr,
'timeout': 10
}).start()
# copy snapshot from the leader node
if self.__seed_addr is not None:
try:
metadata = get_metadata(bind_addr=get_leader(
bind_addr=self.__seed_addr, timeout=10),
timeout=10)
response = requests.get('http://{0}/snapshot'.format(
metadata['http_addr']))
if response.status_code == HTTPStatus.OK:
with open(self.__conf.fullDumpFile, 'wb') as f:
f.write(response.content)
except Exception as ex:
self.__logger.error('failed to copy snapshot: {0}'.format(ex))
# start node
metadata = {
'grpc_addr': '{0}:{1}'.format(self.__host, self.__grpc_port),
'http_addr': '{0}:{1}'.format(self.__host, self.__http_port)
}
self.__logger.info('starting raft state machine')
super(Indexer, self).__init__(self.__self_addr,
self.__peer_addrs,
conf=self.__conf,
metadata=metadata)
self.__logger.info('raft state machine has started')
if os.path.exists(self.__conf.fullDumpFile):
self.__logger.debug('snapshot exists: {0}'.format(
self.__conf.fullDumpFile))
else:
pass
while not self.isReady():
# recovering data
self.__logger.debug('waiting for cluster ready')
self.__logger.debug(self.getStatus())
time.sleep(1)
self.__logger.info('cluster ready')
self.__logger.debug(self.getStatus())
# open existing indices on startup
for index_name in self.get_index_names():
self.__open_index(index_name, index_config=None)
# record index metrics timer
self.metrics_timer = Timer(10, self.__record_index_metrics)
self.metrics_timer.start()
# start gRPC
self.__grpc_server = grpc.server(
futures.ThreadPoolExecutor(max_workers=self.__grpc_max_workers))
add_IndexServicer_to_server(
IndexGRPCServicer(self,
logger=self.__logger,
metrics_registry=self.__metrics_registry),
self.__grpc_server)
self.__grpc_server.add_insecure_port('{0}:{1}'.format(
self.__host, self.__grpc_port))
self.__grpc_server.start()
self.__logger.info('gRPC server has started')
# start HTTP server
self.__http_servicer = IndexHTTPServicer(self, self.__logger,
self.__http_logger,
self.__metrics_registry)
self.__http_server = HTTPServer(self.__host, self.__http_port,
self.__http_servicer)
self.__http_server.start()
self.__logger.info('HTTP server has started')
self.__logger.info('indexer has started')
def setUp(self):
self.registry = CollectorRegistry()
self.histogram = Histogram('h', 'help', registry=self.registry)
self.labels = Histogram('hl', 'help', ['l'], registry=self.registry)
def process_datapoint(self, datapoint):
if (datapoint['feed'] != 'metrics'):
log.debug(
"'feed' field is not 'metrics' in datapoint, skipping: {}".
format(datapoint))
return
daemon = str(datapoint['service']).replace('druid/', '').lower()
if (daemon not in self.supported_metrics):
log.debug("daemon '{}' is not supported, skipping: {}".format(
daemon, datapoint))
return
metric_name = str(datapoint['metric'])
if (metric_name not in self.supported_metrics[daemon]):
log.debug("metric '{}' is not supported, skipping: {}".format(
datapoint['metric'], datapoint))
return
config = self.supported_metrics[daemon][metric_name]
config.setdefault('labels', [])
config.setdefault('type', 'gauge')
config.setdefault('suffix', '_count')
metric_type = config['type']
if metric_type == 'skip':
return
metric_name = self._get_metric_name(daemon, metric_name, config)
metric_value = float(datapoint['value'])
metric_labels = tuple(sorted(config['labels'] + ['host']))
label_values = tuple(
[datapoint[label_name] for label_name in metric_labels])
if '_metric_' not in config:
if metric_type == 'counter':
config['_metric_'] = Counter(metric_name, metric_name,
metric_labels)
if metric_type == 'gauge':
config['_metric_'] = Gauge(metric_name, metric_name,
metric_labels)
elif metric_type == 'summary':
config['_metric_'] = Summary(metric_name, metric_name,
metric_labels)
elif metric_type == 'histogram':
config['_metric_'] = Histogram(metric_name,
metric_name,
metric_labels,
buckets=config['buckets'])
metric = config['_metric_']
if len(metric_labels) > 0:
metric = metric.labels(*label_values)
if metric_type == 'counter':
metric.inc(metric_value)
if metric_type == 'gauge':
metric.set(metric_value)
elif metric_type == 'summary':
metric.observe(metric_value)
elif metric_type == 'histogram':
metric.observe(metric_value)
self.datapoints_processed.inc()
logger = logging.getLogger(__name__)
# total number of responses served, split by method/servlet/tag
response_count = Counter("synapse_http_server_response_count", "",
["method", "servlet", "tag"])
requests_counter = Counter("synapse_http_server_requests_received", "",
["method", "servlet"])
outgoing_responses_counter = Counter("synapse_http_server_responses", "",
["method", "code"])
response_timer = Histogram(
"synapse_http_server_response_time_seconds",
"sec",
["method", "servlet", "tag", "code"],
)
response_ru_utime = Counter("synapse_http_server_response_ru_utime_seconds",
"sec", ["method", "servlet", "tag"])
response_ru_stime = Counter("synapse_http_server_response_ru_stime_seconds",
"sec", ["method", "servlet", "tag"])
response_db_txn_count = Counter("synapse_http_server_response_db_txn_count",
"", ["method", "servlet", "tag"])
# seconds spent waiting for db txns, excluding scheduling time, when processing
# this request
response_db_txn_duration = Counter(
def process_datapoint(self, datapoint):
global sep_config
if (datapoint['feed'] != 'metrics'):
log.debug(
"'feed' field is not 'metrics' in datapoint, skipping: {}".
format(datapoint))
return
daemon = str(datapoint['service']).replace('druid/', '').lower()
if (daemon not in self.supported_metrics):
log.warn("daemon '{}' is not supported, skipping: {}".format(
daemon, datapoint))
return
metric_name = str(datapoint['metric'])
if (metric_name not in self.supported_metrics[daemon]):
log.warn("metric '{}' is not supported, skipping: {}".format(
datapoint['metric'], datapoint))
return
# if 'sep_config' not in locals():
# sep_config = {}
if daemon not in sep_config:
sep_config[daemon] = {}
log.debug("Reverse Metric: {}".format(sep_config))
if metric_name not in sep_config[daemon]:
sep_config[daemon][metric_name] = copy.copy(
self.supported_metrics[daemon][metric_name])
log.debug("Reverse IFtrue: {}")
else:
sep_config[daemon][metric_name] = sep_config[daemon][metric_name]
log.debug("Reverse IFelse: {}")
#config = self.supported_metrics[daemon][metric_name]
log.debug("Reverse Metric: {}".format(sep_config))
sep_config[daemon][metric_name].setdefault('labels', [])
sep_config[daemon][metric_name].setdefault('type', 'gauge')
sep_config[daemon][metric_name].setdefault('suffix', '_count')
metric_type = sep_config[daemon][metric_name]['type']
if metric_type == 'skip':
return
metric_name_full = self._get_metric_name(
daemon, metric_name, sep_config[daemon][metric_name])
metric_value = float(datapoint['value'])
metric_labels = tuple(
sorted(sep_config[daemon][metric_name]['labels'] + ['host']))
log.debug("Labels: {}".format(metric_labels))
label_values = tuple([
datapoint[label_name.replace('_', ' ')]
for label_name in metric_labels
])
log.debug("Labels value: {}".format(label_values))
if '_metric_' not in sep_config[daemon][metric_name]:
if metric_type == 'counter':
sep_config[daemon][metric_name]['_metric_'] = Counter(
metric_name_full, metric_name_full, metric_labels)
if metric_type == 'gauge':
sep_config[daemon][metric_name]['_metric_'] = Gauge(
metric_name_full, metric_name_full, metric_labels)
elif metric_type == 'summary':
sep_config[daemon][metric_name]['_metric_'] = Summary(
metric_name_full, metric_name_full, metric_labels)
elif metric_type == 'histogram':
sep_config[daemon][metric_name]['_metric_'] = Histogram(
metric_name_full,
metric_name_full,
metric_labels,
buckets=sep_config[daemon][metric_name]['buckets'])
log.debug("final metric_name: {}".format(metric_name))
log.debug("sep config : {}".format(sep_config[daemon]))
metric = sep_config[daemon][metric_name]['_metric_']
if len(metric_labels) > 0:
metric = metric.labels(*label_values)
if metric_type == 'counter':
metric.inc(metric_value)
if metric_type == 'gauge':
metric.set(metric_value)
elif metric_type == 'summary':
metric.observe(metric_value)
elif metric_type == 'histogram':
metric.observe(metric_value)
self.datapoints_processed.inc()
up.set(0)
return []
up.set(1)
return [
meeting_room_occupancy,
meeting_room_will_be_occupied,
meeting_room_will_be_free,
today_meetings_left,
]
app = Sanic()
request_histogram = Histogram(
"request_duration_seconds",
"Histogram or request duration.",
labelnames=["type", "status_code", "endpoint", "method"])
def observe_latency():
def decorator(f):
@wraps(f)
async def decorated_function(request, *args, **kwargs):
start = datetime.now()
r = await f(request, *args, **kwargs)
request_histogram.labels("REST", r.status, request.uri_template,
request.method).observe(
(datetime.now() - start).seconds)
return r
return decorated_function
def __init__(self,
host='localhost',
port=7070,
seed_addr=None,
conf=SyncObjConf(),
data_dir='/tmp/cockatrice/management',
grpc_port=5050,
grpc_max_workers=10,
http_port=8080,
logger=getLogger(),
http_logger=getLogger(),
metrics_registry=CollectorRegistry()):
self.__host = host
self.__port = port
self.__seed_addr = seed_addr
self.__conf = conf
self.__data_dir = data_dir
self.__grpc_port = grpc_port
self.__grpc_max_workers = grpc_max_workers
self.__http_port = http_port
self.__logger = logger
self.__http_logger = http_logger
self.__metrics_registry = metrics_registry
# metrics
self.__metrics_requests_total = Counter(
'{0}_manager_requests_total'.format(NAME),
'The number of requests.', ['func'],
registry=self.__metrics_registry)
self.__metrics_requests_duration_seconds = Histogram(
'{0}_manager_requests_duration_seconds'.format(NAME),
'The invocation duration in seconds.', ['func'],
registry=self.__metrics_registry)
self.__self_addr = '{0}:{1}'.format(self.__host, self.__port)
self.__peer_addrs = [] if self.__seed_addr is None else get_peers(
self.__seed_addr)
self.__other_addrs = [
peer_addr for peer_addr in self.__peer_addrs
if peer_addr != self.__self_addr
]
self.__conf.serializer = self.__serialize
self.__conf.deserializer = self.__deserialize
self.__conf.validate()
self.__data = LockedDict()
self.__lock = RLock()
# add this node to the cluster
if self.__self_addr not in self.__peer_addrs and self.__seed_addr is not None:
Thread(target=add_node,
kwargs={
'node_name': self.__self_addr,
'bind_addr': self.__seed_addr,
'timeout': 0.5
}).start()
# create data dir
os.makedirs(self.__data_dir, exist_ok=True)
# copy snapshot from the leader node
if self.__seed_addr is not None:
try:
leader = get_status(bind_addr=self.__seed_addr,
timeout=0.5)['leader']
self.__logger.info('copying snapshot from {0}'.format(leader))
snapshot = get_snapshot(bind_addr=leader, timeout=0.5)
if snapshot is not None:
with open(self.__conf.fullDumpFile, 'wb') as f:
f.write(snapshot)
self.__logger.info(
'snapshot copied from {0}'.format(leader))
except Exception as ex:
self.__logger.error(
'failed to copy snapshot from {0}: {1}'.format(leader, ex))
# start node
super(Manager, self).__init__(self.__self_addr,
self.__other_addrs,
conf=self.__conf)
self.__logger.info('state machine has started')
while not self.isReady():
# recovering data
self.__logger.debug('waiting for cluster ready')
time.sleep(1)
self.__logger.info('cluster ready')
# start gRPC
self.__grpc_server = grpc.server(
futures.ThreadPoolExecutor(max_workers=self.__grpc_max_workers))
add_ManagementServicer_to_server(
ManagementGRPCServicer(self,
logger=self.__logger,
metrics_registry=self.__metrics_registry),
self.__grpc_server)
self.__grpc_server.add_insecure_port('{0}:{1}'.format(
self.__host, self.__grpc_port))
self.__grpc_server.start()
self.__logger.info('gRPC server has started')
# start HTTP server
self.__http_servicer = ManagementHTTPServicer(self, self.__logger,
self.__http_logger,
self.__metrics_registry)
self.__http_server = HTTPServer(self.__host, self.__http_port,
self.__http_servicer)
self.__http_server.start()
self.__logger.info('HTTP server has started')
self.__logger.info('manager has started')
class Manager(RaftNode):
def __init__(self,
host='localhost',
port=7070,
seed_addr=None,
conf=SyncObjConf(),
data_dir='/tmp/cockatrice/management',
grpc_port=5050,
grpc_max_workers=10,
http_port=8080,
logger=getLogger(),
http_logger=getLogger(),
metrics_registry=CollectorRegistry()):
self.__host = host
self.__port = port
self.__seed_addr = seed_addr
self.__conf = conf
self.__data_dir = data_dir
self.__grpc_port = grpc_port
self.__grpc_max_workers = grpc_max_workers
self.__http_port = http_port
self.__logger = logger
self.__http_logger = http_logger
self.__metrics_registry = metrics_registry
# metrics
self.__metrics_requests_total = Counter(
'{0}_manager_requests_total'.format(NAME),
'The number of requests.', ['func'],
registry=self.__metrics_registry)
self.__metrics_requests_duration_seconds = Histogram(
'{0}_manager_requests_duration_seconds'.format(NAME),
'The invocation duration in seconds.', ['func'],
registry=self.__metrics_registry)
self.__self_addr = '{0}:{1}'.format(self.__host, self.__port)
self.__peer_addrs = [] if self.__seed_addr is None else get_peers(
self.__seed_addr)
self.__other_addrs = [
peer_addr for peer_addr in self.__peer_addrs
if peer_addr != self.__self_addr
]
self.__conf.serializer = self.__serialize
self.__conf.deserializer = self.__deserialize
self.__conf.validate()
self.__data = LockedDict()
self.__lock = RLock()
# add this node to the cluster
if self.__self_addr not in self.__peer_addrs and self.__seed_addr is not None:
Thread(target=add_node,
kwargs={
'node_name': self.__self_addr,
'bind_addr': self.__seed_addr,
'timeout': 0.5
}).start()
# create data dir
os.makedirs(self.__data_dir, exist_ok=True)
# copy snapshot from the leader node
if self.__seed_addr is not None:
try:
leader = get_status(bind_addr=self.__seed_addr,
timeout=0.5)['leader']
self.__logger.info('copying snapshot from {0}'.format(leader))
snapshot = get_snapshot(bind_addr=leader, timeout=0.5)
if snapshot is not None:
with open(self.__conf.fullDumpFile, 'wb') as f:
f.write(snapshot)
self.__logger.info(
'snapshot copied from {0}'.format(leader))
except Exception as ex:
self.__logger.error(
'failed to copy snapshot from {0}: {1}'.format(leader, ex))
# start node
super(Manager, self).__init__(self.__self_addr,
self.__other_addrs,
conf=self.__conf)
self.__logger.info('state machine has started')
while not self.isReady():
# recovering data
self.__logger.debug('waiting for cluster ready')
time.sleep(1)
self.__logger.info('cluster ready')
# start gRPC
self.__grpc_server = grpc.server(
futures.ThreadPoolExecutor(max_workers=self.__grpc_max_workers))
add_ManagementServicer_to_server(
ManagementGRPCServicer(self,
logger=self.__logger,
metrics_registry=self.__metrics_registry),
self.__grpc_server)
self.__grpc_server.add_insecure_port('{0}:{1}'.format(
self.__host, self.__grpc_port))
self.__grpc_server.start()
self.__logger.info('gRPC server has started')
# start HTTP server
self.__http_servicer = ManagementHTTPServicer(self, self.__logger,
self.__http_logger,
self.__metrics_registry)
self.__http_server = HTTPServer(self.__host, self.__http_port,
self.__http_servicer)
self.__http_server.start()
self.__logger.info('HTTP server has started')
self.__logger.info('manager has started')
def stop(self):
# stop HTTP server
self.__http_server.stop()
self.__logger.info('HTTP server has stopped')
# stop gRPC server
self.__grpc_server.stop(grace=0.0)
self.__logger.info('gRPC server has stopped')
# stop node
self.destroy()
self.__logger.info('state machine has stopped')
self.__logger.info('manager has stopped')
def __record_metrics(self, start_time, func_name):
self.__metrics_requests_total.labels(func=func_name).inc()
self.__metrics_requests_duration_seconds.labels(
func=func_name).observe(time.time() - start_time)
# serializer
def __serialize(self, filename, raft_data):
with self.__lock:
try:
self.__logger.info('serializer has started')
with zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED) as f:
# store the federation data
f.writestr('federation.bin', pickle.dumps(self.__data))
self.__logger.debug(
'federation data has stored in {0}'.format(filename))
# store the raft data
f.writestr(RAFT_DATA_FILE, pickle.dumps(raft_data))
self.__logger.info(
'{0} has restored'.format(RAFT_DATA_FILE))
self.__logger.info('snapshot has created')
except Exception as ex:
self.__logger.error(
'failed to create snapshot: {0}'.format(ex))
finally:
self.__logger.info('serializer has stopped')
# deserializer
def __deserialize(self, filename):
raft_data = None
with self.__lock:
try:
self.__logger.info('deserializer has started')
with zipfile.ZipFile(filename, 'r') as zf:
# extract the federation data
zf.extract('federation.bin', path=self.__data_dir)
self.__data = pickle.loads(zf.read('federation.bin'))
self.__logger.info('federation.bin has restored')
# restore the raft data
raft_data = pickle.loads(zf.read(RAFT_DATA_FILE))
self.__logger.info(
'raft.{0} has restored'.format(RAFT_DATA_FILE))
self.__logger.info('snapshot has restored')
except Exception as ex:
self.__logger.error(
'failed to restore indices: {0}'.format(ex))
finally:
self.__logger.info('deserializer has stopped')
return raft_data
def is_healthy(self):
return self.isHealthy()
def is_alive(self):
return self.isAlive()
def is_ready(self):
return self.isReady()
def __key_value_to_dict(self, key, value):
keys = [k for k in key.split('/') if k != '']
if len(keys) > 1:
value = self.__key_value_to_dict('/'.join(keys[1:]), value)
return {keys[0]: value}
def __put(self, key, value):
start_time = time.time()
try:
if key == '/':
self.__data.update(value)
else:
self.__data.update(self.__key_value_to_dict(key, value))
finally:
self.__record_metrics(start_time, 'put')
@replicated
def put(self, key, value):
self.__put(key, value)
def get(self, key):
start_time = time.time()
try:
value = self.__data
keys = [k for k in key.split('/') if k != '']
for k in keys:
value = value.get(k, None)
if value is None:
return None
finally:
self.__record_metrics(start_time, 'get')
return value
def __delete(self, key):
start_time = time.time()
try:
if key == '/':
value = dict(self.__data)
self.__clear()
else:
keys = [k for k in key.split('/') if k != '']
value = self.__data
i = 0
while i < len(keys):
if len(keys[i:]) == 1:
return value.pop(keys[i], None)
value = value.get(keys[i], None)
if value is None:
return None
i += 1
finally:
self.__record_metrics(start_time, 'delete')
return value
@replicated
def delete(self, key):
return self.__delete(key)
def __clear(self):
start_time = time.time()
try:
self.__data.clear()
finally:
self.__record_metrics(start_time, 'clear')
@replicated
def clear(self):
self.__clear()
def __init__(self,
manager,
logger=getLogger(),
http_logger=getLogger(),
metrics_registry=CollectorRegistry()):
self.__manager = manager
self.__logger = logger
self.__http_logger = http_logger
self.__metrics_registry = metrics_registry
# metrics
self.__metrics_requests_total = Counter(
'{0}_manager_http_requests_total'.format(NAME),
'The number of requests.', ['method', 'endpoint', 'status_code'],
registry=self.__metrics_registry)
self.__metrics_requests_duration_seconds = Histogram(
'{0}_manager_http_requests_duration_seconds'.format(NAME),
'The invocation duration in seconds.', ['method', 'endpoint'],
registry=self.__metrics_registry)
self.__metrics_requests_bytes_total = Counter(
'{0}_manager_http_requests_bytes_total'.format(NAME),
'A summary of the invocation requests bytes.',
['method', 'endpoint'],
registry=self.__metrics_registry)
self.__metrics_responses_bytes_total = Counter(
'{0}_manager_http_responses_bytes_total'.format(NAME),
'A summary of the invocation responses bytes.',
['method', 'endpoint'],
registry=self.__metrics_registry)
self.app = Flask('supervise_http_server')
self.app.add_url_rule('/data',
endpoint='put',
view_func=self.__put,
methods=['PUT'],
strict_slashes=False)
self.app.add_url_rule('/data',
endpoint='get',
view_func=self.__get,
methods=['GET'],
strict_slashes=False)
self.app.add_url_rule('/data',
endpoint='delete',
view_func=self.__delete,
methods=['DELETE'],
strict_slashes=False)
self.app.add_url_rule('/data/<path:key>',
endpoint='put',
view_func=self.__put,
methods=['PUT'])
self.app.add_url_rule('/data/<path:key>',
endpoint='get',
view_func=self.__get,
methods=['GET'])
self.app.add_url_rule('/data/<path:key>',
endpoint='delete',
view_func=self.__delete,
methods=['DELETE'])
self.app.add_url_rule('/metrics',
endpoint='metrics',
view_func=self.__metrics,
methods=['GET'])
self.app.add_url_rule('/healthiness',
endpoint='healthiness',
view_func=self.__healthiness,
methods=['GET'])
self.app.add_url_rule('/liveness',
endpoint='liveness',
view_func=self.__liveness,
methods=['GET'])
self.app.add_url_rule('/readiness',
endpoint='readiness',
view_func=self.__readiness,
methods=['GET'])
self.app.add_url_rule('/status',
endpoint='status',
view_func=self.__get_status,
methods=['GET'])
# disable Flask default logger
self.app.logger.disabled = True
getLogger('werkzeug').disabled = True
GaugeMetricFamily,
Histogram,
Counter,
)
from prometheus_client.multiprocess import MultiProcessCollector
from sqlalchemy import cast, String
from models import count_groups
from models.payment import Payment
from models.product import Product
from models.purchase import Purchase, AdmissionTicket
from models.cfp import Proposal
metrics = Blueprint('metric', __name__)
request_duration = Histogram('emf_request_duration_seconds',
"Request duration", ['endpoint', 'method'])
request_total = Counter('emf_request_total', "Total request count",
['endpoint', 'method', 'http_status'])
def gauge_groups(gauge, query, *entities):
for count, *key in count_groups(query, *entities):
gauge.add_metric(key, count)
class ExternalMetrics:
def __init__(self, registry=None):
if registry is not None:
registry.register(self)
def collect(self):
class ManagementHTTPServicer:
def __init__(self,
manager,
logger=getLogger(),
http_logger=getLogger(),
metrics_registry=CollectorRegistry()):
self.__manager = manager
self.__logger = logger
self.__http_logger = http_logger
self.__metrics_registry = metrics_registry
# metrics
self.__metrics_requests_total = Counter(
'{0}_manager_http_requests_total'.format(NAME),
'The number of requests.', ['method', 'endpoint', 'status_code'],
registry=self.__metrics_registry)
self.__metrics_requests_duration_seconds = Histogram(
'{0}_manager_http_requests_duration_seconds'.format(NAME),
'The invocation duration in seconds.', ['method', 'endpoint'],
registry=self.__metrics_registry)
self.__metrics_requests_bytes_total = Counter(
'{0}_manager_http_requests_bytes_total'.format(NAME),
'A summary of the invocation requests bytes.',
['method', 'endpoint'],
registry=self.__metrics_registry)
self.__metrics_responses_bytes_total = Counter(
'{0}_manager_http_responses_bytes_total'.format(NAME),
'A summary of the invocation responses bytes.',
['method', 'endpoint'],
registry=self.__metrics_registry)
self.app = Flask('supervise_http_server')
self.app.add_url_rule('/data',
endpoint='put',
view_func=self.__put,
methods=['PUT'],
strict_slashes=False)
self.app.add_url_rule('/data',
endpoint='get',
view_func=self.__get,
methods=['GET'],
strict_slashes=False)
self.app.add_url_rule('/data',
endpoint='delete',
view_func=self.__delete,
methods=['DELETE'],
strict_slashes=False)
self.app.add_url_rule('/data/<path:key>',
endpoint='put',
view_func=self.__put,
methods=['PUT'])
self.app.add_url_rule('/data/<path:key>',
endpoint='get',
view_func=self.__get,
methods=['GET'])
self.app.add_url_rule('/data/<path:key>',
endpoint='delete',
view_func=self.__delete,
methods=['DELETE'])
self.app.add_url_rule('/metrics',
endpoint='metrics',
view_func=self.__metrics,
methods=['GET'])
self.app.add_url_rule('/healthiness',
endpoint='healthiness',
view_func=self.__healthiness,
methods=['GET'])
self.app.add_url_rule('/liveness',
endpoint='liveness',
view_func=self.__liveness,
methods=['GET'])
self.app.add_url_rule('/readiness',
endpoint='readiness',
view_func=self.__readiness,
methods=['GET'])
self.app.add_url_rule('/status',
endpoint='status',
view_func=self.__get_status,
methods=['GET'])
# disable Flask default logger
self.app.logger.disabled = True
getLogger('werkzeug').disabled = True
def __record_metrics(self, start_time, req, resp):
self.__metrics_requests_total.labels(
method=req.method,
endpoint=req.path +
('?{0}'.format(req.query_string.decode('utf-8'))
if len(req.query_string) > 0 else ''),
status_code=resp.status_code.value).inc()
self.__metrics_requests_bytes_total.labels(
method=req.method,
endpoint=req.path +
('?{0}'.format(req.query_string.decode('utf-8'))
if len(req.query_string) > 0 else '')).inc(
req.content_length if req.content_length is not None else 0)
self.__metrics_responses_bytes_total.labels(
method=req.method,
endpoint=req.path +
('?{0}'.format(req.query_string.decode('utf-8'))
if len(req.query_string) > 0 else '')).inc(
resp.content_length if resp.content_length is not None else 0)
self.__metrics_requests_duration_seconds.labels(
method=req.method,
endpoint=req.path +
('?{0}'.format(req.query_string.decode('utf-8'))
if len(req.query_string) > 0 else '')).observe(time.time() -
start_time)
return
def __put(self, key=''):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
mime = mimeparse.parse_mime_type(
request.headers.get('Content-Type'))
charset = 'utf-8' if mime[2].get(
'charset') is None else mime[2].get('charset')
if mime[1] == 'yaml':
value = yaml.safe_load(request.data.decode(charset))
elif mime[1] == 'json':
value = json.loads(request.data.decode(charset))
else:
# handle as a string
value = request.data.decode(charset)
sync = False
if request.args.get('sync', default='',
type=str).lower() in TRUE_STRINGS:
sync = True
self.__manager.put(key if key.startswith('/') else '/' + key,
value,
sync=sync)
if sync:
status_code = HTTPStatus.CREATED
else:
status_code = HTTPStatus.ACCEPTED
except (ConstructorError, JSONDecodeError, ValueError) as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.BAD_REQUEST
self.__logger.error(ex)
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {
'code': status_code.value,
'phrase': status_code.phrase,
'description': status_code.description
}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __get(self, key=''):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
value = self.__manager.get(key if key.startswith('/') else '/' +
key)
if value is None:
status_code = HTTPStatus.NOT_FOUND
else:
data['value'] = value
status_code = HTTPStatus.OK
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {
'code': status_code.value,
'phrase': status_code.phrase,
'description': status_code.description
}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __delete(self, key=''):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
sync = False
if request.args.get('sync', default='',
type=str).lower() in TRUE_STRINGS:
sync = True
value = self.__manager.delete(key if key.startswith('/') else '/' +
key,
sync=sync)
if value is None:
status_code = HTTPStatus.NOT_FOUND
else:
status_code = HTTPStatus.OK
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {
'code': status_code.value,
'phrase': status_code.phrase,
'description': status_code.description
}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __get_status(self):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, resp)
return response
data = {}
status_code = None
try:
data['node_status'] = self.__manager.getStatus()
status_code = HTTPStatus.OK
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {
'code': status_code.value,
'phrase': status_code.phrase,
'description': status_code.description
}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __healthiness(self):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
healthy = self.__manager.is_healthy()
data['healthy'] = healthy
if healthy:
status_code = HTTPStatus.OK
else:
status_code = HTTPStatus.SERVICE_UNAVAILABLE
data['error'] = 'node is not healthy'
except Exception as ex:
data['healthy'] = False
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {
'code': status_code.value,
'phrase': status_code.phrase,
'description': status_code.description
}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __liveness(self):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
alive = self.__manager.is_alive()
data['liveness'] = alive
if alive:
status_code = HTTPStatus.OK
else:
status_code = HTTPStatus.SERVICE_UNAVAILABLE
data['error'] = 'node is not alive'
except Exception as ex:
data['liveness'] = False
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {
'code': status_code.value,
'phrase': status_code.phrase,
'description': status_code.description
}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __readiness(self):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
ready = self.__manager.is_ready()
data['readiness'] = ready
if ready:
status_code = HTTPStatus.OK
else:
status_code = HTTPStatus.SERVICE_UNAVAILABLE
data['error'] = 'node is not ready'
except Exception as ex:
data['readiness'] = False
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {
'code': status_code.value,
'phrase': status_code.phrase,
'description': status_code.description
}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __metrics(self):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
resp = Response()
try:
resp.status_code = HTTPStatus.OK
resp.content_type = CONTENT_TYPE_LATEST
resp.data = generate_latest(self.__metrics_registry)
except Exception as ex:
resp.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
resp.content_type = 'text/plain; charset="UTF-8"'
resp.data = '{0}\n{1}'.format(resp.status_code.phrase,
resp.status_code.description)
self.__logger.error(ex)
return resp
def test_empty_labels_list(self):
Histogram('h', 'help', [], registry=self.registry)
self.assertEqual(0, self.registry.get_sample_value('h_sum'))
class TestHistogram(unittest.TestCase):
def setUp(self):
self.registry = CollectorRegistry()
self.histogram = Histogram('h', 'help', registry=self.registry)
self.labels = Histogram('hl', 'help', ['l'], registry=self.registry)
def test_histogram(self):
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(0, self.registry.get_sample_value('h_sum'))
self.histogram.observe(2)
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(2, self.registry.get_sample_value('h_sum'))
self.histogram.observe(2.5)
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(2, self.registry.get_sample_value('h_count'))
self.assertEqual(4.5, self.registry.get_sample_value('h_sum'))
self.histogram.observe(float("inf"))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '1.0'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '2.5'}))
self.assertEqual(
2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
self.assertEqual(
3, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
self.assertEqual(3, self.registry.get_sample_value('h_count'))
self.assertEqual(float("inf"), self.registry.get_sample_value('h_sum'))
def test_setting_buckets(self):
h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2])
self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds)
h = Histogram('h',
'help',
registry=None,
buckets=[0, 1, 2, float("inf")])
self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds)
self.assertRaises(ValueError,
Histogram,
'h',
'help',
registry=None,
buckets=[])
self.assertRaises(ValueError,
Histogram,
'h',
'help',
registry=None,
buckets=[float("inf")])
self.assertRaises(ValueError,
Histogram,
'h',
'help',
registry=None,
buckets=[3, 1])
def test_labels(self):
self.assertRaises(ValueError,
Histogram,
'h',
'help',
registry=None,
labelnames=['le'])
self.labels.labels('a').observe(2)
self.assertEqual(
0,
self.registry.get_sample_value('hl_bucket', {
'le': '1.0',
'l': 'a'
}))
self.assertEqual(
1,
self.registry.get_sample_value('hl_bucket', {
'le': '2.5',
'l': 'a'
}))
self.assertEqual(
1,
self.registry.get_sample_value('hl_bucket', {
'le': '5.0',
'l': 'a'
}))
self.assertEqual(
1,
self.registry.get_sample_value('hl_bucket', {
'le': '+Inf',
'l': 'a'
}))
self.assertEqual(
1, self.registry.get_sample_value('hl_count', {'l': 'a'}))
self.assertEqual(2,
self.registry.get_sample_value('hl_sum', {'l': 'a'}))
def test_function_decorator(self):
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
@self.histogram.time()
def f():
pass
self.assertEqual(([], None, None, None), inspect.getargspec(f))
f()
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
def test_function_decorator_multithread(self):
self.assertEqual(0, self.registry.get_sample_value('h_count'))
workers = 3
duration = 0.1
pool = ThreadPoolExecutor(max_workers=workers)
@self.histogram.time()
def f():
time.sleep(duration)
jobs = workers * 3
for i in range(jobs):
pool.submit(f)
pool.shutdown(wait=True)
self.assertEqual(jobs, self.registry.get_sample_value('h_count'))
rounding_coefficient = 0.9
total_expected_duration = jobs * duration * rounding_coefficient
self.assertLess(total_expected_duration,
self.registry.get_sample_value('h_sum'))
def test_block_decorator(self):
self.assertEqual(0, self.registry.get_sample_value('h_count'))
self.assertEqual(
0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
with self.histogram.time():
pass
self.assertEqual(1, self.registry.get_sample_value('h_count'))
self.assertEqual(
1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
#
gc_unreachable = Gauge("python_gc_unreachable_total", "Unreachable GC objects",
["gen"])
gc_time = Histogram(
"python_gc_time",
"Time taken to GC (sec)",
["gen"],
buckets=[
0.0025,
0.005,
0.01,
0.025,
0.05,
0.10,
0.25,
0.50,
1.00,
2.50,
5.00,
7.50,
15.00,
30.00,
45.00,
60.00,
],
)
class GCCounts(Collector):
def collect(self) -> Iterable[Metric]:
class IndexHTTPServicer:
def __init__(self, indexer, logger=getLogger(), http_logger=getLogger(),
metrics_registry=CollectorRegistry()):
self.__indexer = indexer
self.__logger = logger
self.__http_logger = http_logger
self.__metrics_registry = metrics_registry
# metrics
self.__metrics_requests_total = Counter(
'{0}_indexer_http_requests_total'.format(NAME),
'The number of requests.',
[
'method',
'endpoint',
'status_code'
],
registry=self.__metrics_registry
)
self.__metrics_requests_duration_seconds = Histogram(
'{0}_indexer_http_requests_duration_seconds'.format(NAME),
'The invocation duration in seconds.',
[
'method',
'endpoint'
],
registry=self.__metrics_registry
)
self.__metrics_requests_bytes_total = Counter(
'{0}_indexer_http_requests_bytes_total'.format(NAME),
'A summary of the invocation requests bytes.',
[
'method',
'endpoint'
],
registry=self.__metrics_registry
)
self.__metrics_responses_bytes_total = Counter(
'{0}_indexer_http_responses_bytes_total'.format(NAME),
'A summary of the invocation responses bytes.',
[
'method',
'endpoint'
],
registry=self.__metrics_registry
)
self.app = Flask('indexer_http')
self.app.add_url_rule('/', endpoint='root', view_func=self.__root, methods=['GET'])
self.app.add_url_rule('/indices/<index_name>', endpoint='get_index', view_func=self.__get_index,
methods=['GET'])
self.app.add_url_rule('/indices/<index_name>', endpoint='create_index', view_func=self.__create_index,
methods=['PUT'])
self.app.add_url_rule('/indices/<index_name>', endpoint='delete_index', view_func=self.__delete_index,
methods=['DELETE'])
self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='get_document',
view_func=self.__get_document, methods=['GET'])
self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='put_document',
view_func=self.__put_document, methods=['PUT'])
self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='delete_document',
view_func=self.__delete_document, methods=['DELETE'])
self.app.add_url_rule('/indices/<index_name>/documents', endpoint='put_documents',
view_func=self.__put_documents, methods=['PUT'])
self.app.add_url_rule('/indices/<index_name>/documents', endpoint='delete_documents',
view_func=self.__delete_documents, methods=['DELETE'])
self.app.add_url_rule('/indices/<index_name>/search', endpoint='search_documents',
view_func=self.__search_documents, methods=['GET', 'POST'])
self.app.add_url_rule('/indices/<index_name>/optimize', endpoint='optimize_index',
view_func=self.__optimize_index, methods=['GET'])
self.app.add_url_rule('/indices/<index_name>/commit', endpoint='commit',
view_func=self.__commit_index, methods=['GET'])
self.app.add_url_rule('/indices/<index_name>/rollback', endpoint='rollback',
view_func=self.__rollback_index, methods=['GET'])
self.app.add_url_rule('/nodes/<node_name>', endpoint='put_node', view_func=self.__put_node, methods=['PUT'])
self.app.add_url_rule('/nodes/<node_name>', endpoint='delete_node', view_func=self.__delete_node,
methods=['DELETE'])
self.app.add_url_rule('/snapshot', endpoint='get_snapshot', view_func=self.__get_snapshot, methods=['GET'])
self.app.add_url_rule('/snapshot', endpoint='put_snapshot', view_func=self.__put_snapshot, methods=['PUT'])
self.app.add_url_rule('/metrics', endpoint='metrics', view_func=self.__metrics, methods=['GET'])
self.app.add_url_rule('/healthiness', endpoint='healthiness', view_func=self.__healthiness, methods=['GET'])
self.app.add_url_rule('/liveness', endpoint='liveness', view_func=self.__liveness, methods=['GET'])
self.app.add_url_rule('/readiness', endpoint='readiness', view_func=self.__readiness, methods=['GET'])
self.app.add_url_rule('/status', endpoint='status', view_func=self.__get_status, methods=['GET'])
# disable Flask default logger
self.app.logger.disabled = True
getLogger('werkzeug').disabled = True
def __record_metrics(self, start_time, req, resp):
self.__metrics_requests_total.labels(
method=req.method,
endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else ''),
status_code=resp.status_code.value
).inc()
self.__metrics_requests_bytes_total.labels(
method=req.method,
endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '')
).inc(req.content_length if req.content_length is not None else 0)
self.__metrics_responses_bytes_total.labels(
method=req.method,
endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '')
).inc(resp.content_length if resp.content_length is not None else 0)
self.__metrics_requests_duration_seconds.labels(
method=req.method,
endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '')
).observe(time.time() - start_time)
return
def __root(self):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
resp = Response()
try:
resp.status_code = HTTPStatus.OK
resp.content_type = 'text/plain; charset="UTF-8"'
resp.data = NAME + ' ' + VERSION + ' is running.\n'
except Exception as ex:
resp.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
resp.content_type = 'text/plain; charset="UTF-8"'
resp.data = '{0}\n{1}'.format(resp.status_code.phrase, resp.status_code.description)
self.__logger.error(ex)
return resp
def __create_index(self, index_name):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
mime = mimeparse.parse_mime_type(request.headers.get('Content-Type'))
charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset')
if mime[1] == 'yaml':
index_config_dict = yaml.safe_load(request.data.decode(charset))
elif mime[1] == 'json':
index_config_dict = json.loads(request.data.decode(charset))
else:
raise ValueError('unsupported format')
if index_config_dict is None:
raise ValueError('index config is None')
sync = False
if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS:
sync = True
index_config = IndexConfig(index_config_dict)
self.__indexer.create_index(index_name, index_config, sync=sync)
if sync:
status_code = HTTPStatus.CREATED
else:
status_code = HTTPStatus.ACCEPTED
except (ConstructorError, JSONDecodeError, ValueError) as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.BAD_REQUEST
self.__logger.error(ex)
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __get_index(self, index_name):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
index = self.__indexer.get_index(index_name)
if index is None:
status_code = HTTPStatus.NOT_FOUND
else:
data['index'] = {
'name': index.indexname,
'doc_count': index.doc_count(),
'doc_count_all': index.doc_count_all(),
'last_modified': index.latest_generation(),
'latest_generation': index.last_modified(),
'version': index.version,
'storage': {
'folder': index.storage.folder,
'supports_mmap': index.storage.supports_mmap,
'readonly': index.storage.readonly,
'files': list(index.storage.list())
}
}
status_code = HTTPStatus.OK
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __delete_index(self, index_name):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
sync = False
if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS:
sync = True
self.__indexer.delete_index(index_name, sync=sync)
if sync:
status_code = HTTPStatus.OK
else:
status_code = HTTPStatus.ACCEPTED
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __commit_index(self, index_name):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
sync = False
if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS:
sync = True
self.__indexer.commit_index(index_name, sync=sync)
if sync:
status_code = HTTPStatus.OK
else:
status_code = HTTPStatus.ACCEPTED
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __rollback_index(self, index_name):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
sync = False
if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS:
sync = True
self.__indexer.rollback_index(index_name, sync=sync)
if sync:
status_code = HTTPStatus.OK
else:
status_code = HTTPStatus.ACCEPTED
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __optimize_index(self, index_name):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
sync = False
if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS:
sync = True
self.__indexer.optimize_index(index_name, sync=sync)
if sync:
status_code = HTTPStatus.OK
else:
status_code = HTTPStatus.ACCEPTED
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __put_document(self, index_name, doc_id):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
mime = mimeparse.parse_mime_type(request.headers.get('Content-Type'))
charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset')
if mime[1] == 'yaml':
fields_dict = yaml.safe_load(request.data.decode(charset))
elif mime[1] == 'json':
fields_dict = json.loads(request.data.decode(charset))
else:
raise ValueError('unsupported format')
if fields_dict is None:
raise ValueError('fields are None')
sync = False
if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS:
sync = True
count = self.__indexer.put_document(index_name, doc_id, fields_dict, sync=sync)
if sync:
if count > 0:
data['count'] = count
status_code = HTTPStatus.CREATED
else:
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
else:
status_code = HTTPStatus.ACCEPTED
except (ConstructorError, JSONDecodeError, ValueError) as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.BAD_REQUEST
self.__logger.error(ex)
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __get_document(self, index_name, doc_id):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
results_page = self.__indexer.get_document(index_name, doc_id)
if results_page.total > 0:
fields = {}
for i in results_page.results[0].iteritems():
fields[i[0]] = i[1]
data['fields'] = fields
status_code = HTTPStatus.OK
else:
status_code = HTTPStatus.NOT_FOUND
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __delete_document(self, index_name, doc_id):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
sync = False
if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS:
sync = True
count = self.__indexer.delete_document(index_name, doc_id, sync=sync)
if sync:
if count > 0:
status_code = HTTPStatus.OK
elif count == 0:
status_code = HTTPStatus.NOT_FOUND
else:
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
else:
status_code = HTTPStatus.ACCEPTED
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __put_documents(self, index_name):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
mime = mimeparse.parse_mime_type(request.headers.get('Content-Type'))
charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset')
if mime[1] == 'yaml':
docs_dict = yaml.safe_load(request.data.decode(charset))
elif mime[1] == 'json':
docs_dict = json.loads(request.data.decode(charset))
else:
raise ValueError('unsupported format')
sync = False
if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS:
sync = True
count = self.__indexer.put_documents(index_name, docs_dict, sync=sync)
if sync:
if count > 0:
data['count'] = count
status_code = HTTPStatus.CREATED
else:
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
else:
status_code = HTTPStatus.ACCEPTED
except (ConstructorError, JSONDecodeError, ValueError) as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.BAD_REQUEST
self.__logger.error(ex)
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __delete_documents(self, index_name):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
mime = mimeparse.parse_mime_type(request.headers.get('Content-Type'))
charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset')
if mime[1] == 'yaml':
doc_ids_list = yaml.safe_load(request.data.decode(charset))
elif mime[1] == 'json':
doc_ids_list = json.loads(request.data.decode(charset))
else:
raise ValueError('unsupported format')
sync = False
if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS:
sync = True
count = self.__indexer.delete_documents(index_name, doc_ids_list, sync=sync)
if sync:
if count > 0:
data['count'] = count
status_code = HTTPStatus.OK
else:
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
else:
status_code = HTTPStatus.ACCEPTED
except (ConstructorError, JSONDecodeError, ValueError) as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.BAD_REQUEST
self.__logger.error(ex)
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __search_documents(self, index_name):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, resp)
return response
data = {}
status_code = None
try:
query = request.args.get('query', default='', type=str)
search_field = request.args.get('search_field', default='', type=str)
page_num = request.args.get('page_num', default=1, type=int)
page_len = request.args.get('page_len', default=10, type=int)
weighting = BM25F
if len(request.data) > 0:
mime = mimeparse.parse_mime_type(request.headers.get('Content-Type'))
charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset')
if mime[1] == 'yaml':
weighting = get_multi_weighting(yaml.safe_load(request.data.decode(charset)))
elif mime[1] == 'json':
weighting = get_multi_weighting(json.loads(request.data.decode(charset)))
else:
raise ValueError('unsupported format')
results_page = self.__indexer.search_documents(index_name, query, search_field, page_num,
page_len=page_len, weighting=weighting)
if results_page.pagecount >= page_num or results_page.total <= 0:
results = {
'is_last_page': results_page.is_last_page(),
'page_count': results_page.pagecount,
'page_len': results_page.pagelen,
'page_num': results_page.pagenum,
'total': results_page.total,
'offset': results_page.offset
}
hits = []
for result in results_page.results[results_page.offset:]:
fields = {}
for item in result.iteritems():
fields[item[0]] = item[1]
hit = {
'fields': fields,
'doc_num': result.docnum,
'score': result.score,
'rank': result.rank,
'pos': result.pos
}
hits.append(hit)
results['hits'] = hits
data['results'] = results
status_code = HTTPStatus.OK
else:
data['error'] = 'page_num must be <= {0}'.format(results_page.pagecount)
status_code = HTTPStatus.BAD_REQUEST
except (ConstructorError, JSONDecodeError, ValueError) as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.BAD_REQUEST
self.__logger.error(ex)
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __put_node(self, node_name):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, resp)
return response
data = {}
status_code = None
try:
self.__indexer.addNodeToCluster(node_name)
status_code = HTTPStatus.OK
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __get_status(self):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, resp)
return response
data = {}
status_code = None
try:
data['node_status'] = self.__indexer.getStatus()
status_code = HTTPStatus.OK
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __delete_node(self, node_name):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, resp)
return response
data = {}
status_code = None
try:
self.__indexer.removeNodeFromCluster(node_name)
status_code = HTTPStatus.OK
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __put_snapshot(self):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, resp)
return response
data = {}
status_code = None
try:
sync = False
if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS:
sync = True
self.__indexer.create_snapshot(sync=sync)
status_code = HTTPStatus.ACCEPTED
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __get_snapshot(self):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
try:
if self.__indexer.is_snapshot_exist():
def generate():
with self.__indexer.open_snapshot_file() as f:
chunk = f.read(1024)
yield chunk
resp = Response(generate(), status=HTTPStatus.OK, mimetype='application/zip', headers={
'Content-Disposition': 'attachment; filename=snapshot.zip'
})
else:
resp = Response(status=HTTPStatus.NOT_FOUND)
except Exception as ex:
resp = Response(status=HTTPStatus.INTERNAL_SERVER_ERROR)
self.__logger.error(ex)
return resp
def __healthiness(self):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
healthy = self.__indexer.is_healthy()
data['healthy'] = healthy
if healthy:
status_code = HTTPStatus.OK
else:
status_code = HTTPStatus.SERVICE_UNAVAILABLE
data['error'] = 'node is not healthy'
except Exception as ex:
data['healthy'] = False
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __liveness(self):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
alive = self.__indexer.is_alive()
data['liveness'] = alive
if alive:
status_code = HTTPStatus.OK
else:
status_code = HTTPStatus.SERVICE_UNAVAILABLE
data['error'] = 'node is not alive'
except Exception as ex:
data['liveness'] = False
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __readiness(self):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
ready = self.__indexer.is_ready()
data['readiness'] = ready
if ready:
status_code = HTTPStatus.OK
else:
status_code = HTTPStatus.SERVICE_UNAVAILABLE
data['error'] = 'node is not ready'
except Exception as ex:
data['readiness'] = False
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def __metrics(self):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
resp = Response()
try:
resp.status_code = HTTPStatus.OK
resp.content_type = CONTENT_TYPE_LATEST
resp.data = generate_latest(self.__metrics_registry)
except Exception as ex:
resp.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
resp.content_type = 'text/plain; charset="UTF-8"'
resp.data = '{0}\n{1}'.format(resp.status_code.phrase, resp.status_code.description)
self.__logger.error(ex)
return resp
class Indexer(RaftNode):
def __init__(self,
host='localhost',
port=7070,
seed_addr=None,
conf=SyncObjConf(),
data_dir='/tmp/cockatrice/index',
grpc_port=5050,
grpc_max_workers=10,
http_port=8080,
logger=getLogger(),
http_logger=getLogger(),
metrics_registry=CollectorRegistry()):
self.__host = host
self.__port = port
self.__seed_addr = seed_addr
self.__conf = conf
self.__data_dir = data_dir
self.__grpc_port = grpc_port
self.__grpc_max_workers = grpc_max_workers
self.__http_port = http_port
self.__logger = logger
self.__http_logger = http_logger
self.__metrics_registry = metrics_registry
# metrics
self.__metrics_core_documents = Gauge(
'{0}_indexer_index_documents'.format(NAME),
'The number of documents.', [
'index_name',
],
registry=self.__metrics_registry)
self.__metrics_requests_total = Counter(
'{0}_indexer_requests_total'.format(NAME),
'The number of requests.', ['func'],
registry=self.__metrics_registry)
self.__metrics_requests_duration_seconds = Histogram(
'{0}_indexer_requests_duration_seconds'.format(NAME),
'The invocation duration in seconds.', ['func'],
registry=self.__metrics_registry)
self.__self_addr = '{0}:{1}'.format(self.__host, self.__port)
self.__peer_addrs = [] if self.__seed_addr is None else get_peers(
bind_addr=self.__seed_addr, timeout=10)
self.__other_addrs = [
peer_addr for peer_addr in self.__peer_addrs
if peer_addr != self.__self_addr
]
self.__conf.serializer = self.__serialize
self.__conf.deserializer = self.__deserialize
self.__conf.validate()
self.__indices = {}
self.__index_configs = {}
self.__writers = {}
self.__auto_commit_timers = {}
self.__lock = RLock()
# create data dir
os.makedirs(self.__data_dir, exist_ok=True)
self.__file_storage = FileStorage(self.__data_dir,
supports_mmap=True,
readonly=False,
debug=False)
self.__ram_storage = RamStorage()
# if seed addr specified and self node does not exist in the cluster, add self node to the cluster
if self.__seed_addr is not None and self.__self_addr not in self.__peer_addrs:
Thread(target=add_node,
kwargs={
'node_name': self.__self_addr,
'bind_addr': self.__seed_addr,
'timeout': 10
}).start()
# copy snapshot from the leader node
if self.__seed_addr is not None:
try:
metadata = get_metadata(bind_addr=get_leader(
bind_addr=self.__seed_addr, timeout=10),
timeout=10)
response = requests.get('http://{0}/snapshot'.format(
metadata['http_addr']))
if response.status_code == HTTPStatus.OK:
with open(self.__conf.fullDumpFile, 'wb') as f:
f.write(response.content)
except Exception as ex:
self.__logger.error('failed to copy snapshot: {0}'.format(ex))
# start node
metadata = {
'grpc_addr': '{0}:{1}'.format(self.__host, self.__grpc_port),
'http_addr': '{0}:{1}'.format(self.__host, self.__http_port)
}
self.__logger.info('starting raft state machine')
super(Indexer, self).__init__(self.__self_addr,
self.__peer_addrs,
conf=self.__conf,
metadata=metadata)
self.__logger.info('raft state machine has started')
if os.path.exists(self.__conf.fullDumpFile):
self.__logger.debug('snapshot exists: {0}'.format(
self.__conf.fullDumpFile))
else:
pass
while not self.isReady():
# recovering data
self.__logger.debug('waiting for cluster ready')
self.__logger.debug(self.getStatus())
time.sleep(1)
self.__logger.info('cluster ready')
self.__logger.debug(self.getStatus())
# open existing indices on startup
for index_name in self.get_index_names():
self.__open_index(index_name, index_config=None)
# record index metrics timer
self.metrics_timer = Timer(10, self.__record_index_metrics)
self.metrics_timer.start()
# start gRPC
self.__grpc_server = grpc.server(
futures.ThreadPoolExecutor(max_workers=self.__grpc_max_workers))
add_IndexServicer_to_server(
IndexGRPCServicer(self,
logger=self.__logger,
metrics_registry=self.__metrics_registry),
self.__grpc_server)
self.__grpc_server.add_insecure_port('{0}:{1}'.format(
self.__host, self.__grpc_port))
self.__grpc_server.start()
self.__logger.info('gRPC server has started')
# start HTTP server
self.__http_servicer = IndexHTTPServicer(self, self.__logger,
self.__http_logger,
self.__metrics_registry)
self.__http_server = HTTPServer(self.__host, self.__http_port,
self.__http_servicer)
self.__http_server.start()
self.__logger.info('HTTP server has started')
self.__logger.info('indexer has started')
def stop(self):
# stop HTTP server
self.__http_server.stop()
self.__logger.info('HTTP server has stopped')
# stop gRPC server
self.__grpc_server.stop(grace=0.0)
self.__logger.info('gRPC server has stopped')
self.metrics_timer.cancel()
# close indices
for index_name in list(self.__indices.keys()):
self.__close_index(index_name)
self.destroy()
self.__logger.info('index core has stopped')
def __record_index_metrics(self):
for index_name in list(self.__indices.keys()):
try:
self.__metrics_core_documents.labels(
index_name=index_name).set(self.get_doc_count(index_name))
except Exception as ex:
self.__logger.error(ex)
def __record_metrics(self, start_time, func_name):
self.__metrics_requests_total.labels(func=func_name).inc()
self.__metrics_requests_duration_seconds.labels(
func=func_name).observe(time.time() - start_time)
# def __serialize_indices(self, filename):
# with self.__lock:
# try:
# self.__logger.info('starting serialize indices')
#
# except Exception as ex:
# self.__logger.error('failed to create snapshot: {0}'.format(ex))
# finally:
# self.__logger.info('serialize indices has finished')
# def __serialize_raft_data(self, filename, raft_data):
# with self.__lock:
# pass
# index serializer
def __serialize(self, filename, raft_data):
with self.__lock:
try:
self.__logger.debug('serializer has started')
# store the index files and raft logs to the snapshot file
with zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED) as f:
for index_name in self.get_index_names():
self.__commit_index(index_name)
# with self.__get_writer(index_name).writelock:
# with self.__indices[index_name].lock('WRITELOCK'):
# index files
for index_filename in self.get_index_files(index_name):
if self.__index_configs.get(
index_name).get_storage_type() == "ram":
with self.__ram_storage.open_file(
index_filename) as r:
f.writestr(index_filename, r.read())
else:
f.write(
os.path.join(self.__file_storage.folder,
index_filename),
index_filename)
self.__logger.debug('{0} has stored in {1}'.format(
index_filename, filename))
# index config file
f.write(
os.path.join(
self.__file_storage.folder,
self.get_index_config_file(index_name)),
self.get_index_config_file(index_name))
self.__logger.debug('{0} has stored in {1}'.format(
self.get_index_config_file(index_name), filename))
# store the raft data
f.writestr(RAFT_DATA_FILE, pickle.dumps(raft_data))
self.__logger.debug(
'{0} has restored'.format(RAFT_DATA_FILE))
self.__logger.debug('snapshot has created')
except Exception as ex:
self.__logger.error(
'failed to create snapshot: {0}'.format(ex))
finally:
self.__logger.debug('serializer has stopped')
# index deserializer
def __deserialize(self, filename):
with self.__lock:
try:
self.__logger.debug('deserializer has started')
with zipfile.ZipFile(filename, 'r') as zf:
# get file names in snapshot file
filenames = list(zf.namelist())
# get index names in snapshot file
index_names = []
pattern_toc = re.compile(r'^_(.+)_\d+\.toc$')
for f in filenames:
match = pattern_toc.search(f)
if match and match.group(1) not in index_names:
index_names.append(match.group(1))
for index_name in index_names:
# extract the index config first
zf.extract(self.get_index_config_file(index_name),
path=self.__file_storage.folder)
index_config = pickle.loads(
zf.read(self.get_index_config_file(index_name)))
# get index files
pattern_toc = re.compile(r'^_{0}_(\d+)\..+$'.format(
index_name)) # ex) _myindex_0.toc
pattern_seg = re.compile(
r'^{0}_([a-z0-9]+)\..+$'.format(index_name)
) # ex) myindex_zseabukc2nbpvh0u.seg
pattern_lock = re.compile(r'^{0}_WRITELOCK$'.format(
index_name)) # ex) myindex_WRITELOCK
index_files = []
for file_name in filenames:
if re.match(pattern_toc, file_name):
index_files.append(file_name)
elif re.match(pattern_seg, file_name):
index_files.append(file_name)
elif re.match(pattern_lock, file_name):
index_files.append(file_name)
# extract the index files
for index_file in index_files:
if index_config.get_storage_type() == 'ram':
with self.__ram_storage.create_file(
index_file) as r:
r.write(zf.read(index_file))
else:
zf.extract(index_file,
path=self.__file_storage.folder)
self.__logger.debug(
'{0} has restored from {1}'.format(
index_file, filename))
self.__logger.debug(
'{0} has restored'.format(index_name))
# extract the raft data
raft_data = pickle.loads(zf.read(RAFT_DATA_FILE))
self.__logger.debug(
'{0} has restored'.format(RAFT_DATA_FILE))
return raft_data
except Exception as ex:
self.__logger.error(
'failed to restore indices: {0}'.format(ex))
finally:
self.__logger.debug('deserializer has stopped')
def is_healthy(self):
return self.isHealthy()
def is_alive(self):
return self.isAlive()
def is_ready(self):
return self.isReady()
def get_addr(self):
return self.__self_addr
def get_index_files(self, index_name):
index_files = []
pattern_toc = re.compile(
r'^_{0}_(\d+)\..+$'.format(index_name)) # ex) _myindex_0.toc
pattern_seg = re.compile(r'^{0}_([a-z0-9]+)\..+$'.format(
index_name)) # ex) myindex_zseabukc2nbpvh0u.seg
pattern_lock = re.compile(
r'^{0}_WRITELOCK$'.format(index_name)) # ex) myindex_WRITELOCK
if self.__index_configs.get(index_name).get_storage_type() == "ram":
storage = self.__ram_storage
else:
storage = self.__file_storage
for file_name in list(storage.list()):
if re.match(pattern_toc, file_name):
index_files.append(file_name)
elif re.match(pattern_seg, file_name):
index_files.append(file_name)
elif re.match(pattern_lock, file_name):
index_files.append(file_name)
return index_files
@staticmethod
def get_index_config_file(index_name):
return '{0}_CONFIG'.format(index_name)
def get_index_names(self):
index_names = []
pattern_toc = re.compile(r'^_(.+)_\d+\.toc$')
for filename in list(self.__file_storage.list()):
match = pattern_toc.search(filename)
if match and match.group(1) not in index_names:
index_names.append(match.group(1))
for filename in list(self.__ram_storage.list()):
match = pattern_toc.search(filename)
if match and match.group(1) not in index_names:
index_names.append(match.group(1))
return index_names
def is_index_exist(self, index_name):
return self.__file_storage.index_exists(
indexname=index_name) or self.__ram_storage.index_exists(
indexname=index_name)
def is_index_open(self, index_name):
return index_name in self.__indices
@replicated
def open_index(self, index_name, index_config=None):
return self.__open_index(index_name, index_config=index_config)
def __open_index(self, index_name, index_config=None):
start_time = time.time()
index = None
try:
# open the index
index = self.__indices.get(index_name)
if index is None:
self.__logger.debug('opening {0}'.format(index_name))
if index_config is None:
# set saved index config
with open(
os.path.join(
self.__file_storage.folder,
self.get_index_config_file(index_name)),
'rb') as f:
self.__index_configs[index_name] = pickle.loads(
f.read())
else:
# set given index config
self.__index_configs[index_name] = index_config
if self.__index_configs[index_name].get_storage_type(
) == 'ram':
index = self.__ram_storage.open_index(
indexname=index_name,
schema=self.__index_configs[index_name].get_schema())
else:
index = self.__file_storage.open_index(
indexname=index_name,
schema=self.__index_configs[index_name].get_schema())
self.__indices[index_name] = index
self.__logger.info('{0} has opened'.format(index_name))
# open the index writer
self.__open_writer(index_name)
except Exception as ex:
self.__logger.error('failed to open {0}: {1}'.format(
index_name, ex))
finally:
self.__record_metrics(start_time, 'open_index')
return index
@replicated
def close_index(self, index_name):
return self.__close_index(index_name)
def __close_index(self, index_name):
start_time = time.time()
index = None
try:
# close the index writer
self.__close_writer(index_name)
# close the index
index = self.__indices.pop(index_name)
if index is not None:
self.__logger.debug('closing {0}'.format(index_name))
index.close()
self.__logger.info('{0} has closed'.format(index_name))
except Exception as ex:
self.__logger.error('failed to close {0}: {1}'.format(
index_name, ex))
finally:
self.__record_metrics(start_time, 'close_index')
return index
@replicated
def create_index(self, index_name, index_config):
return self.__create_index(index_name, index_config)
def __create_index(self, index_name, index_config):
if self.is_index_exist(index_name):
# open the index
return self.__open_index(index_name, index_config=index_config)
start_time = time.time()
index = None
with self.__lock:
try:
self.__logger.debug('creating {0}'.format(index_name))
# set index config
self.__index_configs[index_name] = index_config
self.__logger.debug(
self.__index_configs[index_name].get_storage_type())
# create the index
if self.__index_configs[index_name].get_storage_type(
) == 'ram':
index = self.__ram_storage.create_index(
self.__index_configs[index_name].get_schema(),
indexname=index_name)
else:
index = self.__file_storage.create_index(
self.__index_configs[index_name].get_schema(),
indexname=index_name)
self.__indices[index_name] = index
self.__logger.info('{0} has created'.format(index_name))
# save the index config
with open(
os.path.join(self.__file_storage.folder,
self.get_index_config_file(index_name)),
'wb') as f:
f.write(pickle.dumps(index_config))
# open the index writer
self.__open_writer(index_name)
except Exception as ex:
self.__logger.error('failed to create {0}: {1}'.format(
index_name, ex))
finally:
self.__record_metrics(start_time, 'create_index')
return index
@replicated
def delete_index(self, index_name):
return self.__delete_index(index_name)
def __delete_index(self, index_name):
# close index
index = self.__close_index(index_name)
start_time = time.time()
with self.__lock:
try:
self.__logger.debug('deleting {0}'.format(index_name))
# delete index files
for filename in self.get_index_files(index_name):
self.__file_storage.delete_file(filename)
self.__logger.debug('{0} was deleted'.format(filename))
self.__logger.info('{0} has deleted'.format(index_name))
# delete the index config
self.__index_configs.pop(index_name, None)
os.remove(
os.path.join(self.__file_storage.folder,
self.get_index_config_file(index_name)))
except Exception as ex:
self.__logger.error('failed to delete {0}: {1}'.format(
index_name, ex))
finally:
self.__record_metrics(start_time, 'delete_index')
return index
def get_index(self, index_name):
return self.__get_index(index_name)
def __get_index(self, index_name):
start_time = time.time()
try:
index = self.__indices.get(index_name)
except Exception as ex:
raise ex
finally:
self.__record_metrics(start_time, 'get_index')
return index
def __start_auto_commit_timer(self, index_name, period):
timer = self.__auto_commit_timers.get(index_name, None)
if timer is None:
self.__auto_commit_timers[index_name] = threading.Timer(
period,
self.__auto_commit_index,
kwargs={
'index_name': index_name,
'period': period
})
self.__auto_commit_timers[index_name].start()
self.__logger.debug(
'auto commit timer for {0} were started'.format(index_name))
def __stop_auto_commit_timer(self, index_name):
timer = self.__auto_commit_timers.pop(index_name, None)
if timer is not None:
timer.cancel()
self.__logger.debug(
'auto commit timer for {0} were stopped'.format(index_name))
def __auto_commit_index(self, index_name, period):
self.__stop_auto_commit_timer(index_name)
self.__commit_index(index_name)
self.__start_auto_commit_timer(index_name, period=period)
def __open_writer(self, index_name):
writer = None
try:
writer = self.__writers.get(index_name, None)
if writer is None or writer.is_closed:
self.__logger.debug(
'opening writer for {0}'.format(index_name))
writer = self.__indices.get(index_name).writer()
self.__writers[index_name] = writer
self.__logger.debug(
'writer for {0} has opened'.format(index_name))
self.__start_auto_commit_timer(
index_name,
period=self.__index_configs.get(
index_name).get_writer_auto_commit_period())
except Exception as ex:
self.__logger.error('failed to open writer for {0}: {1}'.format(
index_name, ex))
return writer
def __close_writer(self, index_name):
writer = None
try:
self.__stop_auto_commit_timer(index_name)
# close the index
writer = self.__writers.pop(index_name, None)
if writer is not None:
self.__logger.debug(
'closing writer for {0}'.format(index_name))
writer.commit()
self.__logger.debug(
'writer for {0} has closed'.format(index_name))
except Exception as ex:
self.__logger.error('failed to close writer for {0}: {1}'.format(
index_name, ex))
return writer
def __get_writer(self, index_name):
return self.__writers.get(index_name, None)
def __get_searcher(self, index_name, weighting=None):
try:
if weighting is None:
searcher = self.__indices.get(index_name).searcher()
else:
searcher = self.__indices.get(index_name).searcher(
weighting=weighting)
except Exception as ex:
raise ex
return searcher
@replicated
def commit_index(self, index_name):
return self.__commit_index(index_name)
def __commit_index(self, index_name):
start_time = time.time()
success = False
with self.__lock:
try:
self.__logger.debug('committing {0}'.format(index_name))
self.__get_writer(index_name).commit()
self.__open_writer(index_name) # reopen writer
self.__logger.info('{0} has committed'.format(index_name))
success = True
except Exception as ex:
self.__logger.error('failed to commit index {0}: {1}'.format(
index_name, ex))
finally:
self.__record_metrics(start_time, 'commit_index')
return success
@replicated
def rollback_index(self, index_name):
return self.__rollback_index(index_name)
def __rollback_index(self, index_name):
start_time = time.time()
success = False
with self.__lock:
try:
self.__logger.debug('rolling back {0}'.format(index_name))
self.__get_writer(index_name).cancel()
self.__open_writer(index_name) # reopen writer
self.__logger.info('{0} has rolled back'.format(index_name))
success = True
except Exception as ex:
self.__logger.error('failed to rollback index {0}: {1}'.format(
index_name, ex))
finally:
self.__record_metrics(start_time, 'rollback_index')
return success
@replicated
def optimize_index(self, index_name):
return self.__optimize_index(index_name)
def __optimize_index(self, index_name):
start_time = time.time()
success = False
with self.__lock:
try:
self.__logger.debug('optimizing {0}'.format(index_name))
self.__get_writer(index_name).commit(optimize=True,
merge=False)
self.__open_writer(index_name) # reopen writer
self.__logger.info('{0} has optimized'.format(index_name))
success = True
except Exception as ex:
self.__logger.error('failed to optimize {0}: {1}'.format(
index_name, ex))
finally:
self.__record_metrics(start_time, 'optimize_index')
return success
def get_doc_count(self, index_name):
try:
cnt = self.__indices.get(index_name).doc_count()
except Exception as ex:
raise ex
return cnt
def get_schema(self, index_name):
try:
schema = self.__indices.get(index_name).schema
except Exception as ex:
raise ex
return schema
@replicated
def put_document(self, index_name, doc_id, fields):
return self.__put_document(index_name, doc_id, fields)
def __put_document(self, index_name, doc_id, fields):
doc = copy.deepcopy(fields)
doc[self.__index_configs.get(index_name).get_doc_id_field()] = doc_id
return self.__put_documents(index_name, [doc])
@replicated
def put_documents(self, index_name, docs):
return self.__put_documents(index_name, docs)
def __put_documents(self, index_name, docs):
start_time = time.time()
with self.__lock:
try:
self.__logger.debug(
'putting documents to {0}'.format(index_name))
# count = self.__get_writer(index_name).update_documents(docs)
count = 0
for doc in docs:
self.__get_writer(index_name).update_document(**doc)
count += 1
self.__logger.info('{0} documents has put to {1}'.format(
count, index_name))
except Exception as ex:
self.__logger.error(
'failed to put documents to {0}: {1}'.format(
index_name, ex))
count = -1
finally:
self.__record_metrics(start_time, 'put_documents')
return count
def get_document(self, index_name, doc_id):
try:
results_page = self.search_documents(
index_name,
doc_id,
self.__index_configs.get(index_name).get_doc_id_field(),
1,
page_len=1)
if results_page.total > 0:
self.__logger.debug('{0} was got from {1}'.format(
doc_id, index_name))
else:
self.__logger.debug('{0} did not exist in {1}'.format(
doc_id, index_name))
except Exception as ex:
raise ex
return results_page
@replicated
def delete_document(self, index_name, doc_id):
return self.__delete_document(index_name, doc_id)
def __delete_document(self, index_name, doc_id):
return self.__delete_documents(index_name, [doc_id])
@replicated
def delete_documents(self, index_name, doc_ids):
return self.__delete_documents(index_name, doc_ids)
def __delete_documents(self, index_name, doc_ids):
start_time = time.time()
with self.__lock:
try:
self.__logger.debug(
'deleting documents from {0}'.format(index_name))
# count = self.__get_writer(index_name).delete_documents(doc_ids, doc_id_field=self.__index_configs.get(
# index_name).get_doc_id_field())
count = 0
for doc_id in doc_ids:
count += self.__get_writer(index_name).delete_by_term(
self.__index_configs.get(
index_name).get_doc_id_field(), doc_id)
self.__logger.info('{0} documents has deleted from {1}'.format(
count, index_name))
except Exception as ex:
self.__logger.error(
'failed to delete documents in bulk to {0}: {1}'.format(
index_name, ex))
count = -1
finally:
self.__record_metrics(start_time, 'delete_documents')
return count
def search_documents(self,
index_name,
query,
search_field,
page_num,
page_len=10,
weighting=None,
**kwargs):
start_time = time.time()
try:
searcher = self.__get_searcher(index_name, weighting=weighting)
query_parser = QueryParser(search_field,
self.get_schema(index_name))
query_obj = query_parser.parse(query)
results_page = searcher.search_page(query_obj,
page_num,
pagelen=page_len,
**kwargs)
self.__logger.info('{0} documents ware searched from {1}'.format(
results_page.total, index_name))
except Exception as ex:
raise ex
finally:
self.__record_metrics(start_time, 'search_documents')
return results_page
@replicated
def create_snapshot(self):
self.__create_snapshot()
def __create_snapshot(self):
self.forceLogCompaction()
def get_snapshot_file_name(self):
return self.__conf.fullDumpFile
def is_snapshot_exist(self):
return os.path.exists(self.get_snapshot_file_name())
def open_snapshot_file(self):
with self.__lock:
try:
file = open(self.get_snapshot_file_name(), mode='rb')
except Exception as ex:
raise ex
return file
def test_unit_notappended(self):
Histogram('h_seconds',
'help', [],
registry=self.registry,
unit="seconds")
self.assertEqual(0, self.registry.get_sample_value('h_seconds_sum'))
def __init__(self, indexer, logger=getLogger(), http_logger=getLogger(),
metrics_registry=CollectorRegistry()):
self.__indexer = indexer
self.__logger = logger
self.__http_logger = http_logger
self.__metrics_registry = metrics_registry
# metrics
self.__metrics_requests_total = Counter(
'{0}_indexer_http_requests_total'.format(NAME),
'The number of requests.',
[
'method',
'endpoint',
'status_code'
],
registry=self.__metrics_registry
)
self.__metrics_requests_duration_seconds = Histogram(
'{0}_indexer_http_requests_duration_seconds'.format(NAME),
'The invocation duration in seconds.',
[
'method',
'endpoint'
],
registry=self.__metrics_registry
)
self.__metrics_requests_bytes_total = Counter(
'{0}_indexer_http_requests_bytes_total'.format(NAME),
'A summary of the invocation requests bytes.',
[
'method',
'endpoint'
],
registry=self.__metrics_registry
)
self.__metrics_responses_bytes_total = Counter(
'{0}_indexer_http_responses_bytes_total'.format(NAME),
'A summary of the invocation responses bytes.',
[
'method',
'endpoint'
],
registry=self.__metrics_registry
)
self.app = Flask('indexer_http')
self.app.add_url_rule('/', endpoint='root', view_func=self.__root, methods=['GET'])
self.app.add_url_rule('/indices/<index_name>', endpoint='get_index', view_func=self.__get_index,
methods=['GET'])
self.app.add_url_rule('/indices/<index_name>', endpoint='create_index', view_func=self.__create_index,
methods=['PUT'])
self.app.add_url_rule('/indices/<index_name>', endpoint='delete_index', view_func=self.__delete_index,
methods=['DELETE'])
self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='get_document',
view_func=self.__get_document, methods=['GET'])
self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='put_document',
view_func=self.__put_document, methods=['PUT'])
self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='delete_document',
view_func=self.__delete_document, methods=['DELETE'])
self.app.add_url_rule('/indices/<index_name>/documents', endpoint='put_documents',
view_func=self.__put_documents, methods=['PUT'])
self.app.add_url_rule('/indices/<index_name>/documents', endpoint='delete_documents',
view_func=self.__delete_documents, methods=['DELETE'])
self.app.add_url_rule('/indices/<index_name>/search', endpoint='search_documents',
view_func=self.__search_documents, methods=['GET', 'POST'])
self.app.add_url_rule('/indices/<index_name>/optimize', endpoint='optimize_index',
view_func=self.__optimize_index, methods=['GET'])
self.app.add_url_rule('/indices/<index_name>/commit', endpoint='commit',
view_func=self.__commit_index, methods=['GET'])
self.app.add_url_rule('/indices/<index_name>/rollback', endpoint='rollback',
view_func=self.__rollback_index, methods=['GET'])
self.app.add_url_rule('/nodes/<node_name>', endpoint='put_node', view_func=self.__put_node, methods=['PUT'])
self.app.add_url_rule('/nodes/<node_name>', endpoint='delete_node', view_func=self.__delete_node,
methods=['DELETE'])
self.app.add_url_rule('/snapshot', endpoint='get_snapshot', view_func=self.__get_snapshot, methods=['GET'])
self.app.add_url_rule('/snapshot', endpoint='put_snapshot', view_func=self.__put_snapshot, methods=['PUT'])
self.app.add_url_rule('/metrics', endpoint='metrics', view_func=self.__metrics, methods=['GET'])
self.app.add_url_rule('/healthiness', endpoint='healthiness', view_func=self.__healthiness, methods=['GET'])
self.app.add_url_rule('/liveness', endpoint='liveness', view_func=self.__liveness, methods=['GET'])
self.app.add_url_rule('/readiness', endpoint='readiness', view_func=self.__readiness, methods=['GET'])
self.app.add_url_rule('/status', endpoint='status', view_func=self.__get_status, methods=['GET'])
# disable Flask default logger
self.app.logger.disabled = True
getLogger('werkzeug').disabled = True
