def test_setting_buckets(self): h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2, float("inf")]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[float("inf")]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[3, 1])
def test_duplicate_metrics_raises(self): registry = CollectorRegistry() Counter('c_total', 'help', registry=registry) self.assertRaises(ValueError, Counter, 'c_total', 'help', registry=registry) self.assertRaises(ValueError, Gauge, 'c_total', 'help', registry=registry) self.assertRaises(ValueError, Gauge, 'c_created', 'help', registry=registry) Gauge('g_created', 'help', registry=registry) self.assertRaises(ValueError, Gauge, 'g_created', 'help', registry=registry) self.assertRaises(ValueError, Counter, 'g', 'help', registry=registry) Summary('s', 'help', registry=registry) self.assertRaises(ValueError, Summary, 's', 'help', registry=registry) self.assertRaises(ValueError, Gauge, 's_created', 'help', registry=registry) self.assertRaises(ValueError, Gauge, 's_sum', 'help', registry=registry) self.assertRaises(ValueError, Gauge, 's_count', 'help', registry=registry) # We don't currently expose quantiles, but let's prevent future # clashes anyway. self.assertRaises(ValueError, Gauge, 's', 'help', registry=registry) Histogram('h', 'help', registry=registry) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=registry) # Clashes aggaint various suffixes. self.assertRaises(ValueError, Summary, 'h', 'help', registry=registry) self.assertRaises(ValueError, Gauge, 'h_count', 'help', registry=registry) self.assertRaises(ValueError, Gauge, 'h_sum', 'help', registry=registry) self.assertRaises(ValueError, Gauge, 'h_bucket', 'help', registry=registry) self.assertRaises(ValueError, Gauge, 'h_created', 'help', registry=registry) # The name of the histogram itself is also taken. self.assertRaises(ValueError, Gauge, 'h', 'help', registry=registry) Info('i', 'help', registry=registry) self.assertRaises(ValueError, Gauge, 'i_info', 'help', registry=registry)
def test_histogram_adds(self): h1 = Histogram('h', 'help', registry=None) h2 = Histogram('h', 'help', registry=None) self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual(0, self.registry.get_sample_value('h_sum')) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) h1.observe(1) h2.observe(2) self.assertEqual(2, self.registry.get_sample_value('h_count')) self.assertEqual(3, self.registry.get_sample_value('h_sum')) self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
def test_histogram_adds(self): h1 = Histogram('h', 'help', registry=None) core._ValueClass = core._MultiProcessValue(lambda: 456) h2 = Histogram('h', 'help', registry=None) self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual(0, self.registry.get_sample_value('h_sum')) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) h1.observe(1) h2.observe(2) self.assertEqual(2, self.registry.get_sample_value('h_count')) self.assertEqual(3, self.registry.get_sample_value('h_sum')) self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '5.0'}))
def test_reset_registry_with_labels(self): registry = CollectorRegistry() gauge = Gauge('g', 'help', ['l'], registry=registry) gauge.labels('a').inc() self.assertEqual(1, registry.get_sample_value('g', {'l': 'a'})) counter = Counter('c_total', 'help', ['l'], registry=registry) counter.labels('a').inc() self.assertEqual(1, registry.get_sample_value('c_total', {'l': 'a'})) summary = Summary('s', 'help', ['l'], registry=registry) summary.labels('a').observe(10) self.assertEqual(1, registry.get_sample_value('s_count', {'l': 'a'})) self.assertEqual(10, registry.get_sample_value('s_sum', {'l': 'a'})) histogram = Histogram('h', 'help', ['l'], registry=registry) histogram.labels('a').observe(2) self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '1.0', 'l': 'a'})) self.assertEqual(1, registry.get_sample_value('h_bucket', {'le': '2.5', 'l': 'a'})) self.assertEqual(1, registry.get_sample_value('h_bucket', {'le': '5.0', 'l': 'a'})) self.assertEqual(1, registry.get_sample_value('h_bucket', {'le': '+Inf', 'l': 'a'})) self.assertEqual(1, registry.get_sample_value('h_count', {'l': 'a'})) self.assertEqual(2, registry.get_sample_value('h_sum', {'l': 'a'})) registry.reset() self.assertEqual(0, registry.get_sample_value('g', {'l': 'a'})) self.assertEqual(0, registry.get_sample_value('c_total', {'l': 'a'})) self.assertEqual(0, registry.get_sample_value('s_count', {'l': 'a'})) self.assertEqual(0, registry.get_sample_value('s_sum', {'l': 'a'})) self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '1.0', 'l': 'a'})) self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '2.5', 'l': 'a'})) self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '5.0', 'l': 'a'})) self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '+Inf', 'l': 'a'})) self.assertEqual(0, registry.get_sample_value('h_count', {'l': 'a'})) self.assertEqual(0, registry.get_sample_value('h_sum', {'l': 'a'}))
def test_collect(self): pid = 0 core._ValueClass = core._MultiProcessValue(lambda: pid) labels = dict((i, i) for i in 'abcd') def add_label(key, value): l = labels.copy() l[key] = value return l c = Counter('c', 'help', labelnames=labels.keys(), registry=None) g = Gauge('g', 'help', labelnames=labels.keys(), registry=None) h = Histogram('h', 'help', labelnames=labels.keys(), registry=None) c.labels(**labels).inc(1) g.labels(**labels).set(1) h.labels(**labels).observe(1) pid = 1 c.labels(**labels).inc(1) g.labels(**labels).set(1) h.labels(**labels).observe(5) metrics = dict((m.name, m) for m in self.collector.collect()) self.assertEqual( metrics['c'].samples, [Sample('c_total', labels, 2.0)] ) metrics['g'].samples.sort(key=lambda x: x[1]['pid']) self.assertEqual(metrics['g'].samples, [ Sample('g', add_label('pid', '0'), 1.0), Sample('g', add_label('pid', '1'), 1.0), ]) metrics['h'].samples.sort( key=lambda x: (x[0], float(x[1].get('le', 0))) ) expected_histogram = [ Sample('h_bucket', add_label('le', '0.005'), 0.0), Sample('h_bucket', add_label('le', '0.01'), 0.0), Sample('h_bucket', add_label('le', '0.025'), 0.0), Sample('h_bucket', add_label('le', '0.05'), 0.0), Sample('h_bucket', add_label('le', '0.075'), 0.0), Sample('h_bucket', add_label('le', '0.1'), 0.0), Sample('h_bucket', add_label('le', '0.25'), 0.0), Sample('h_bucket', add_label('le', '0.5'), 0.0), Sample('h_bucket', add_label('le', '0.75'), 0.0), Sample('h_bucket', add_label('le', '1.0'), 1.0), Sample('h_bucket', add_label('le', '2.5'), 1.0), Sample('h_bucket', add_label('le', '5.0'), 2.0), Sample('h_bucket', add_label('le', '7.5'), 2.0), Sample('h_bucket', add_label('le', '10.0'), 2.0), Sample('h_bucket', add_label('le', '+Inf'), 2.0), Sample('h_count', labels, 2.0), Sample('h_sum', labels, 6.0), ] self.assertEqual(metrics['h'].samples, expected_histogram)
def __init__(self, indexer, logger=getLogger(), metrics_registry=CollectorRegistry()): self.__indexer = indexer self.__logger = logger self.__metrics_registry = metrics_registry # metrics self.__metrics_requests_total = Counter( '{0}_indexer_grpc_requests_total'.format(NAME), 'The number of requests.', [ 'func' ], registry=self.__metrics_registry ) self.__metrics_requests_duration_seconds = Histogram( '{0}_indexer_grpc_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', [ 'func' ], registry=self.__metrics_registry )
def test_merge_no_accumulate(self): pid = 0 core._ValueClass = core._MultiProcessValue(lambda: pid) labels = dict((i, i) for i in 'abcd') def add_label(key, value): l = labels.copy() l[key] = value return l h = Histogram('h', 'help', labelnames=labels.keys(), registry=None) h.labels(**labels).observe(1) pid = 1 h.labels(**labels).observe(5) path = os.path.join(os.environ['prometheus_multiproc_dir'], '*.db') files = glob.glob(path) metrics = dict( (m.name, m) for m in self.collector.merge(files, accumulate=False) ) metrics['h'].samples.sort( key=lambda x: (x[0], float(x[1].get('le', 0))) ) expected_histogram = [ Sample('h_bucket', add_label('le', '0.005'), 0.0), Sample('h_bucket', add_label('le', '0.01'), 0.0), Sample('h_bucket', add_label('le', '0.025'), 0.0), Sample('h_bucket', add_label('le', '0.05'), 0.0), Sample('h_bucket', add_label('le', '0.075'), 0.0), Sample('h_bucket', add_label('le', '0.1'), 0.0), Sample('h_bucket', add_label('le', '0.25'), 0.0), Sample('h_bucket', add_label('le', '0.5'), 0.0), Sample('h_bucket', add_label('le', '0.75'), 0.0), Sample('h_bucket', add_label('le', '1.0'), 1.0), Sample('h_bucket', add_label('le', '2.5'), 0.0), Sample('h_bucket', add_label('le', '5.0'), 1.0), Sample('h_bucket', add_label('le', '7.5'), 0.0), Sample('h_bucket', add_label('le', '10.0'), 0.0), Sample('h_bucket', add_label('le', '+Inf'), 0.0), Sample('h_sum', labels, 6.0), ] self.assertEqual(metrics['h'].samples, expected_histogram)
def test_merge_no_accumulate(self): self.pid = 0 labels = dict((i, i) for i in 'abcd') def add_label(key, value): l = labels.copy() l[key] = value return l h = Histogram('hna', 'help', labelnames=labels.keys(), registry=None) h.labels(**labels).observe(1) self.pid = 1 h.labels(**labels).observe(5) self.collector.accumulate = False metrics = self.collector.collect() self.collector.accumulate = True metric = [x for x in metrics if x.name == 'hna'][0] metric.samples.sort( key=lambda x: (x[0], float(x[1].get('le', 0))) ) expected_histogram = [ Sample('hna_bucket', add_label('le', '0.005'), 0.0), Sample('hna_bucket', add_label('le', '0.01'), 0.0), Sample('hna_bucket', add_label('le', '0.025'), 0.0), Sample('hna_bucket', add_label('le', '0.05'), 0.0), Sample('hna_bucket', add_label('le', '0.075'), 0.0), Sample('hna_bucket', add_label('le', '0.1'), 0.0), Sample('hna_bucket', add_label('le', '0.25'), 0.0), Sample('hna_bucket', add_label('le', '0.5'), 0.0), Sample('hna_bucket', add_label('le', '0.75'), 0.0), Sample('hna_bucket', add_label('le', '1.0'), 1.0), Sample('hna_bucket', add_label('le', '2.5'), 0.0), Sample('hna_bucket', add_label('le', '5.0'), 1.0), Sample('hna_bucket', add_label('le', '7.5'), 0.0), Sample('hna_bucket', add_label('le', '10.0'), 0.0), Sample('hna_bucket', add_label('le', '+Inf'), 0.0), Sample('hna_sum', labels, 6.0), ] self.assertEqual(metric.samples, expected_histogram)
class TestHistogram(unittest.TestCase): def setUp(self): self.registry = CollectorRegistry() self.histogram = Histogram('h', 'help', registry=self.registry) self.labels = Histogram('hl', 'help', ['l'], registry=self.registry) def test_histogram(self): self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual(0, self.registry.get_sample_value('h_sum')) self.histogram.observe(2) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual(2, self.registry.get_sample_value('h_sum')) self.histogram.observe(2.5) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(2, self.registry.get_sample_value('h_count')) self.assertEqual(4.5, self.registry.get_sample_value('h_sum')) self.histogram.observe(float("inf")) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 3, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(3, self.registry.get_sample_value('h_count')) self.assertEqual(float("inf"), self.registry.get_sample_value('h_sum')) def test_setting_buckets(self): h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2, float("inf")]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[float("inf")]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[3, 1]) def test_labels(self): self.labels.labels('a').observe(2) self.assertEqual( 0, self.registry.get_sample_value('hl_bucket', { 'le': '1.0', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_bucket', { 'le': '2.5', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_bucket', { 'le': '5.0', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_bucket', { 'le': '+Inf', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_count', {'l': 'a'})) self.assertEqual(2, self.registry.get_sample_value('hl_sum', {'l': 'a'})) def test_function_decorator(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) @self.histogram.time() def f(): pass self.assertEqual(([], None, None, None), inspect.getargspec(f)) f() self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) def test_block_decorator(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) with self.histogram.time(): pass self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
generate_latest, CONTENT_TYPE_LATEST, ) from prometheus_client.core import GaugeMetricFamily, Histogram, Counter from prometheus_client.multiprocess import MultiProcessCollector from sqlalchemy import cast, String from models import count_groups from models.payment import Payment from models.product import Product from models.purchase import Purchase, AdmissionTicket from models.cfp import Proposal metrics = Blueprint("metric", __name__) request_duration = Histogram("emf_request_duration_seconds", "Request duration", ["endpoint", "method"]) request_total = Counter("emf_request_total", "Total request count", ["endpoint", "method", "http_status"]) def gauge_groups(gauge, query, *entities): for count, *key in count_groups(query, *entities): gauge.add_metric(key, count) class ExternalMetrics: def __init__(self, registry=None): if registry is not None: registry.register(self) def collect(self):
class TestHistogram(unittest.TestCase): def setUp(self): self.registry = CollectorRegistry() self.histogram = Histogram('h', 'help', registry=self.registry) self.labels = Histogram('hl', 'help', ['l'], registry=self.registry) def test_repr(self): self.assertEqual(repr(self.histogram), "prometheus_client.metrics.Histogram(h)") self.assertEqual(repr(self.labels), "prometheus_client.metrics.Histogram(hl)") def test_histogram(self): self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual(0, self.registry.get_sample_value('h_sum')) self.histogram.observe(2) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual(2, self.registry.get_sample_value('h_sum')) self.histogram.observe(2.5) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(2, self.registry.get_sample_value('h_count')) self.assertEqual(4.5, self.registry.get_sample_value('h_sum')) self.histogram.observe(float("inf")) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual(3, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(3, self.registry.get_sample_value('h_count')) self.assertEqual(float("inf"), self.registry.get_sample_value('h_sum')) def test_histogram_not_observable(self): """.observe() must fail if the Summary is not observable.""" assert_not_observable(self.labels.observe, 1) def test_setting_buckets(self): h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2, float("inf")]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[float("inf")]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[3, 1]) def test_labels(self): self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, labelnames=['le']) self.labels.labels('a').observe(2) self.assertEqual(0, self.registry.get_sample_value('hl_bucket', {'le': '1.0', 'l': 'a'})) self.assertEqual(1, self.registry.get_sample_value('hl_bucket', {'le': '2.5', 'l': 'a'})) self.assertEqual(1, self.registry.get_sample_value('hl_bucket', {'le': '5.0', 'l': 'a'})) self.assertEqual(1, self.registry.get_sample_value('hl_bucket', {'le': '+Inf', 'l': 'a'})) self.assertEqual(1, self.registry.get_sample_value('hl_count', {'l': 'a'})) self.assertEqual(2, self.registry.get_sample_value('hl_sum', {'l': 'a'})) def test_function_decorator(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) @self.histogram.time() def f(): pass self.assertEqual(([], None, None, None), getargspec(f)) f() self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) def test_function_decorator_multithread(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) workers = 3 duration = 0.1 pool = ThreadPoolExecutor(max_workers=workers) @self.histogram.time() def f(): time.sleep(duration) jobs = workers * 3 for i in range(jobs): pool.submit(f) pool.shutdown(wait=True) self.assertEqual(jobs, self.registry.get_sample_value('h_count')) rounding_coefficient = 0.9 total_expected_duration = jobs * duration * rounding_coefficient self.assertLess(total_expected_duration, self.registry.get_sample_value('h_sum')) def test_block_decorator(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) with self.histogram.time(): pass self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) def test_exemplar_invalid_label_name(self): self.assertRaises(ValueError, self.histogram.observe, 3.0, exemplar={':o)': 'smile'}) self.assertRaises(ValueError, self.histogram.observe, 3.0, exemplar={'1': 'number'}) def test_exemplar_too_long(self): # 129 characters in total should fail. self.assertRaises(ValueError, self.histogram.observe, 1.0, exemplar={ 'abcdefghijklmnopqrstuvwxyz': '26+16 characters', 'x1234567': '8+15 characters', 'zyxwvutsrqponmlkjihgfedcba': '26+16 characters', 'y123456': '7+15 characters', })
def __init__(self, host='localhost', port=7070, seed_addr=None, conf=SyncObjConf(), data_dir='/tmp/cockatrice/index', grpc_port=5050, grpc_max_workers=10, http_port=8080, logger=getLogger(), http_logger=getLogger(), metrics_registry=CollectorRegistry()): self.__host = host self.__port = port self.__seed_addr = seed_addr self.__conf = conf self.__data_dir = data_dir self.__grpc_port = grpc_port self.__grpc_max_workers = grpc_max_workers self.__http_port = http_port self.__logger = logger self.__http_logger = http_logger self.__metrics_registry = metrics_registry # metrics self.__metrics_core_documents = Gauge( '{0}_indexer_index_documents'.format(NAME), 'The number of documents.', [ 'index_name', ], registry=self.__metrics_registry) self.__metrics_requests_total = Counter( '{0}_indexer_requests_total'.format(NAME), 'The number of requests.', ['func'], registry=self.__metrics_registry) self.__metrics_requests_duration_seconds = Histogram( '{0}_indexer_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', ['func'], registry=self.__metrics_registry) self.__self_addr = '{0}:{1}'.format(self.__host, self.__port) self.__peer_addrs = [] if self.__seed_addr is None else get_peers( bind_addr=self.__seed_addr, timeout=10) self.__other_addrs = [ peer_addr for peer_addr in self.__peer_addrs if peer_addr != self.__self_addr ] self.__conf.serializer = self.__serialize self.__conf.deserializer = self.__deserialize self.__conf.validate() self.__indices = {} self.__index_configs = {} self.__writers = {} self.__auto_commit_timers = {} self.__lock = RLock() # create data dir os.makedirs(self.__data_dir, exist_ok=True) self.__file_storage = FileStorage(self.__data_dir, supports_mmap=True, readonly=False, debug=False) self.__ram_storage = RamStorage() # if seed addr specified and self node does not exist in the cluster, add self node to the cluster if self.__seed_addr is not None and self.__self_addr not in self.__peer_addrs: Thread(target=add_node, kwargs={ 'node_name': self.__self_addr, 'bind_addr': self.__seed_addr, 'timeout': 10 }).start() # copy snapshot from the leader node if self.__seed_addr is not None: try: metadata = get_metadata(bind_addr=get_leader( bind_addr=self.__seed_addr, timeout=10), timeout=10) response = requests.get('http://{0}/snapshot'.format( metadata['http_addr'])) if response.status_code == HTTPStatus.OK: with open(self.__conf.fullDumpFile, 'wb') as f: f.write(response.content) except Exception as ex: self.__logger.error('failed to copy snapshot: {0}'.format(ex)) # start node metadata = { 'grpc_addr': '{0}:{1}'.format(self.__host, self.__grpc_port), 'http_addr': '{0}:{1}'.format(self.__host, self.__http_port) } self.__logger.info('starting raft state machine') super(Indexer, self).__init__(self.__self_addr, self.__peer_addrs, conf=self.__conf, metadata=metadata) self.__logger.info('raft state machine has started') if os.path.exists(self.__conf.fullDumpFile): self.__logger.debug('snapshot exists: {0}'.format( self.__conf.fullDumpFile)) else: pass while not self.isReady(): # recovering data self.__logger.debug('waiting for cluster ready') self.__logger.debug(self.getStatus()) time.sleep(1) self.__logger.info('cluster ready') self.__logger.debug(self.getStatus()) # open existing indices on startup for index_name in self.get_index_names(): self.__open_index(index_name, index_config=None) # record index metrics timer self.metrics_timer = Timer(10, self.__record_index_metrics) self.metrics_timer.start() # start gRPC self.__grpc_server = grpc.server( futures.ThreadPoolExecutor(max_workers=self.__grpc_max_workers)) add_IndexServicer_to_server( IndexGRPCServicer(self, logger=self.__logger, metrics_registry=self.__metrics_registry), self.__grpc_server) self.__grpc_server.add_insecure_port('{0}:{1}'.format( self.__host, self.__grpc_port)) self.__grpc_server.start() self.__logger.info('gRPC server has started') # start HTTP server self.__http_servicer = IndexHTTPServicer(self, self.__logger, self.__http_logger, self.__metrics_registry) self.__http_server = HTTPServer(self.__host, self.__http_port, self.__http_servicer) self.__http_server.start() self.__logger.info('HTTP server has started') self.__logger.info('indexer has started')
def setUp(self): self.registry = CollectorRegistry() self.histogram = Histogram('h', 'help', registry=self.registry) self.labels = Histogram('hl', 'help', ['l'], registry=self.registry)
def process_datapoint(self, datapoint): if (datapoint['feed'] != 'metrics'): log.debug( "'feed' field is not 'metrics' in datapoint, skipping: {}". format(datapoint)) return daemon = str(datapoint['service']).replace('druid/', '').lower() if (daemon not in self.supported_metrics): log.debug("daemon '{}' is not supported, skipping: {}".format( daemon, datapoint)) return metric_name = str(datapoint['metric']) if (metric_name not in self.supported_metrics[daemon]): log.debug("metric '{}' is not supported, skipping: {}".format( datapoint['metric'], datapoint)) return config = self.supported_metrics[daemon][metric_name] config.setdefault('labels', []) config.setdefault('type', 'gauge') config.setdefault('suffix', '_count') metric_type = config['type'] if metric_type == 'skip': return metric_name = self._get_metric_name(daemon, metric_name, config) metric_value = float(datapoint['value']) metric_labels = tuple(sorted(config['labels'] + ['host'])) label_values = tuple( [datapoint[label_name] for label_name in metric_labels]) if '_metric_' not in config: if metric_type == 'counter': config['_metric_'] = Counter(metric_name, metric_name, metric_labels) if metric_type == 'gauge': config['_metric_'] = Gauge(metric_name, metric_name, metric_labels) elif metric_type == 'summary': config['_metric_'] = Summary(metric_name, metric_name, metric_labels) elif metric_type == 'histogram': config['_metric_'] = Histogram(metric_name, metric_name, metric_labels, buckets=config['buckets']) metric = config['_metric_'] if len(metric_labels) > 0: metric = metric.labels(*label_values) if metric_type == 'counter': metric.inc(metric_value) if metric_type == 'gauge': metric.set(metric_value) elif metric_type == 'summary': metric.observe(metric_value) elif metric_type == 'histogram': metric.observe(metric_value) self.datapoints_processed.inc()
logger = logging.getLogger(__name__) # total number of responses served, split by method/servlet/tag response_count = Counter("synapse_http_server_response_count", "", ["method", "servlet", "tag"]) requests_counter = Counter("synapse_http_server_requests_received", "", ["method", "servlet"]) outgoing_responses_counter = Counter("synapse_http_server_responses", "", ["method", "code"]) response_timer = Histogram( "synapse_http_server_response_time_seconds", "sec", ["method", "servlet", "tag", "code"], ) response_ru_utime = Counter("synapse_http_server_response_ru_utime_seconds", "sec", ["method", "servlet", "tag"]) response_ru_stime = Counter("synapse_http_server_response_ru_stime_seconds", "sec", ["method", "servlet", "tag"]) response_db_txn_count = Counter("synapse_http_server_response_db_txn_count", "", ["method", "servlet", "tag"]) # seconds spent waiting for db txns, excluding scheduling time, when processing # this request response_db_txn_duration = Counter(
def process_datapoint(self, datapoint): global sep_config if (datapoint['feed'] != 'metrics'): log.debug( "'feed' field is not 'metrics' in datapoint, skipping: {}". format(datapoint)) return daemon = str(datapoint['service']).replace('druid/', '').lower() if (daemon not in self.supported_metrics): log.warn("daemon '{}' is not supported, skipping: {}".format( daemon, datapoint)) return metric_name = str(datapoint['metric']) if (metric_name not in self.supported_metrics[daemon]): log.warn("metric '{}' is not supported, skipping: {}".format( datapoint['metric'], datapoint)) return # if 'sep_config' not in locals(): # sep_config = {} if daemon not in sep_config: sep_config[daemon] = {} log.debug("Reverse Metric: {}".format(sep_config)) if metric_name not in sep_config[daemon]: sep_config[daemon][metric_name] = copy.copy( self.supported_metrics[daemon][metric_name]) log.debug("Reverse IFtrue: {}") else: sep_config[daemon][metric_name] = sep_config[daemon][metric_name] log.debug("Reverse IFelse: {}") #config = self.supported_metrics[daemon][metric_name] log.debug("Reverse Metric: {}".format(sep_config)) sep_config[daemon][metric_name].setdefault('labels', []) sep_config[daemon][metric_name].setdefault('type', 'gauge') sep_config[daemon][metric_name].setdefault('suffix', '_count') metric_type = sep_config[daemon][metric_name]['type'] if metric_type == 'skip': return metric_name_full = self._get_metric_name( daemon, metric_name, sep_config[daemon][metric_name]) metric_value = float(datapoint['value']) metric_labels = tuple( sorted(sep_config[daemon][metric_name]['labels'] + ['host'])) log.debug("Labels: {}".format(metric_labels)) label_values = tuple([ datapoint[label_name.replace('_', ' ')] for label_name in metric_labels ]) log.debug("Labels value: {}".format(label_values)) if '_metric_' not in sep_config[daemon][metric_name]: if metric_type == 'counter': sep_config[daemon][metric_name]['_metric_'] = Counter( metric_name_full, metric_name_full, metric_labels) if metric_type == 'gauge': sep_config[daemon][metric_name]['_metric_'] = Gauge( metric_name_full, metric_name_full, metric_labels) elif metric_type == 'summary': sep_config[daemon][metric_name]['_metric_'] = Summary( metric_name_full, metric_name_full, metric_labels) elif metric_type == 'histogram': sep_config[daemon][metric_name]['_metric_'] = Histogram( metric_name_full, metric_name_full, metric_labels, buckets=sep_config[daemon][metric_name]['buckets']) log.debug("final metric_name: {}".format(metric_name)) log.debug("sep config : {}".format(sep_config[daemon])) metric = sep_config[daemon][metric_name]['_metric_'] if len(metric_labels) > 0: metric = metric.labels(*label_values) if metric_type == 'counter': metric.inc(metric_value) if metric_type == 'gauge': metric.set(metric_value) elif metric_type == 'summary': metric.observe(metric_value) elif metric_type == 'histogram': metric.observe(metric_value) self.datapoints_processed.inc()
up.set(0) return [] up.set(1) return [ meeting_room_occupancy, meeting_room_will_be_occupied, meeting_room_will_be_free, today_meetings_left, ] app = Sanic() request_histogram = Histogram( "request_duration_seconds", "Histogram or request duration.", labelnames=["type", "status_code", "endpoint", "method"]) def observe_latency(): def decorator(f): @wraps(f) async def decorated_function(request, *args, **kwargs): start = datetime.now() r = await f(request, *args, **kwargs) request_histogram.labels("REST", r.status, request.uri_template, request.method).observe( (datetime.now() - start).seconds) return r return decorated_function
def __init__(self, host='localhost', port=7070, seed_addr=None, conf=SyncObjConf(), data_dir='/tmp/cockatrice/management', grpc_port=5050, grpc_max_workers=10, http_port=8080, logger=getLogger(), http_logger=getLogger(), metrics_registry=CollectorRegistry()): self.__host = host self.__port = port self.__seed_addr = seed_addr self.__conf = conf self.__data_dir = data_dir self.__grpc_port = grpc_port self.__grpc_max_workers = grpc_max_workers self.__http_port = http_port self.__logger = logger self.__http_logger = http_logger self.__metrics_registry = metrics_registry # metrics self.__metrics_requests_total = Counter( '{0}_manager_requests_total'.format(NAME), 'The number of requests.', ['func'], registry=self.__metrics_registry) self.__metrics_requests_duration_seconds = Histogram( '{0}_manager_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', ['func'], registry=self.__metrics_registry) self.__self_addr = '{0}:{1}'.format(self.__host, self.__port) self.__peer_addrs = [] if self.__seed_addr is None else get_peers( self.__seed_addr) self.__other_addrs = [ peer_addr for peer_addr in self.__peer_addrs if peer_addr != self.__self_addr ] self.__conf.serializer = self.__serialize self.__conf.deserializer = self.__deserialize self.__conf.validate() self.__data = LockedDict() self.__lock = RLock() # add this node to the cluster if self.__self_addr not in self.__peer_addrs and self.__seed_addr is not None: Thread(target=add_node, kwargs={ 'node_name': self.__self_addr, 'bind_addr': self.__seed_addr, 'timeout': 0.5 }).start() # create data dir os.makedirs(self.__data_dir, exist_ok=True) # copy snapshot from the leader node if self.__seed_addr is not None: try: leader = get_status(bind_addr=self.__seed_addr, timeout=0.5)['leader'] self.__logger.info('copying snapshot from {0}'.format(leader)) snapshot = get_snapshot(bind_addr=leader, timeout=0.5) if snapshot is not None: with open(self.__conf.fullDumpFile, 'wb') as f: f.write(snapshot) self.__logger.info( 'snapshot copied from {0}'.format(leader)) except Exception as ex: self.__logger.error( 'failed to copy snapshot from {0}: {1}'.format(leader, ex)) # start node super(Manager, self).__init__(self.__self_addr, self.__other_addrs, conf=self.__conf) self.__logger.info('state machine has started') while not self.isReady(): # recovering data self.__logger.debug('waiting for cluster ready') time.sleep(1) self.__logger.info('cluster ready') # start gRPC self.__grpc_server = grpc.server( futures.ThreadPoolExecutor(max_workers=self.__grpc_max_workers)) add_ManagementServicer_to_server( ManagementGRPCServicer(self, logger=self.__logger, metrics_registry=self.__metrics_registry), self.__grpc_server) self.__grpc_server.add_insecure_port('{0}:{1}'.format( self.__host, self.__grpc_port)) self.__grpc_server.start() self.__logger.info('gRPC server has started') # start HTTP server self.__http_servicer = ManagementHTTPServicer(self, self.__logger, self.__http_logger, self.__metrics_registry) self.__http_server = HTTPServer(self.__host, self.__http_port, self.__http_servicer) self.__http_server.start() self.__logger.info('HTTP server has started') self.__logger.info('manager has started')
class Manager(RaftNode): def __init__(self, host='localhost', port=7070, seed_addr=None, conf=SyncObjConf(), data_dir='/tmp/cockatrice/management', grpc_port=5050, grpc_max_workers=10, http_port=8080, logger=getLogger(), http_logger=getLogger(), metrics_registry=CollectorRegistry()): self.__host = host self.__port = port self.__seed_addr = seed_addr self.__conf = conf self.__data_dir = data_dir self.__grpc_port = grpc_port self.__grpc_max_workers = grpc_max_workers self.__http_port = http_port self.__logger = logger self.__http_logger = http_logger self.__metrics_registry = metrics_registry # metrics self.__metrics_requests_total = Counter( '{0}_manager_requests_total'.format(NAME), 'The number of requests.', ['func'], registry=self.__metrics_registry) self.__metrics_requests_duration_seconds = Histogram( '{0}_manager_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', ['func'], registry=self.__metrics_registry) self.__self_addr = '{0}:{1}'.format(self.__host, self.__port) self.__peer_addrs = [] if self.__seed_addr is None else get_peers( self.__seed_addr) self.__other_addrs = [ peer_addr for peer_addr in self.__peer_addrs if peer_addr != self.__self_addr ] self.__conf.serializer = self.__serialize self.__conf.deserializer = self.__deserialize self.__conf.validate() self.__data = LockedDict() self.__lock = RLock() # add this node to the cluster if self.__self_addr not in self.__peer_addrs and self.__seed_addr is not None: Thread(target=add_node, kwargs={ 'node_name': self.__self_addr, 'bind_addr': self.__seed_addr, 'timeout': 0.5 }).start() # create data dir os.makedirs(self.__data_dir, exist_ok=True) # copy snapshot from the leader node if self.__seed_addr is not None: try: leader = get_status(bind_addr=self.__seed_addr, timeout=0.5)['leader'] self.__logger.info('copying snapshot from {0}'.format(leader)) snapshot = get_snapshot(bind_addr=leader, timeout=0.5) if snapshot is not None: with open(self.__conf.fullDumpFile, 'wb') as f: f.write(snapshot) self.__logger.info( 'snapshot copied from {0}'.format(leader)) except Exception as ex: self.__logger.error( 'failed to copy snapshot from {0}: {1}'.format(leader, ex)) # start node super(Manager, self).__init__(self.__self_addr, self.__other_addrs, conf=self.__conf) self.__logger.info('state machine has started') while not self.isReady(): # recovering data self.__logger.debug('waiting for cluster ready') time.sleep(1) self.__logger.info('cluster ready') # start gRPC self.__grpc_server = grpc.server( futures.ThreadPoolExecutor(max_workers=self.__grpc_max_workers)) add_ManagementServicer_to_server( ManagementGRPCServicer(self, logger=self.__logger, metrics_registry=self.__metrics_registry), self.__grpc_server) self.__grpc_server.add_insecure_port('{0}:{1}'.format( self.__host, self.__grpc_port)) self.__grpc_server.start() self.__logger.info('gRPC server has started') # start HTTP server self.__http_servicer = ManagementHTTPServicer(self, self.__logger, self.__http_logger, self.__metrics_registry) self.__http_server = HTTPServer(self.__host, self.__http_port, self.__http_servicer) self.__http_server.start() self.__logger.info('HTTP server has started') self.__logger.info('manager has started') def stop(self): # stop HTTP server self.__http_server.stop() self.__logger.info('HTTP server has stopped') # stop gRPC server self.__grpc_server.stop(grace=0.0) self.__logger.info('gRPC server has stopped') # stop node self.destroy() self.__logger.info('state machine has stopped') self.__logger.info('manager has stopped') def __record_metrics(self, start_time, func_name): self.__metrics_requests_total.labels(func=func_name).inc() self.__metrics_requests_duration_seconds.labels( func=func_name).observe(time.time() - start_time) # serializer def __serialize(self, filename, raft_data): with self.__lock: try: self.__logger.info('serializer has started') with zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED) as f: # store the federation data f.writestr('federation.bin', pickle.dumps(self.__data)) self.__logger.debug( 'federation data has stored in {0}'.format(filename)) # store the raft data f.writestr(RAFT_DATA_FILE, pickle.dumps(raft_data)) self.__logger.info( '{0} has restored'.format(RAFT_DATA_FILE)) self.__logger.info('snapshot has created') except Exception as ex: self.__logger.error( 'failed to create snapshot: {0}'.format(ex)) finally: self.__logger.info('serializer has stopped') # deserializer def __deserialize(self, filename): raft_data = None with self.__lock: try: self.__logger.info('deserializer has started') with zipfile.ZipFile(filename, 'r') as zf: # extract the federation data zf.extract('federation.bin', path=self.__data_dir) self.__data = pickle.loads(zf.read('federation.bin')) self.__logger.info('federation.bin has restored') # restore the raft data raft_data = pickle.loads(zf.read(RAFT_DATA_FILE)) self.__logger.info( 'raft.{0} has restored'.format(RAFT_DATA_FILE)) self.__logger.info('snapshot has restored') except Exception as ex: self.__logger.error( 'failed to restore indices: {0}'.format(ex)) finally: self.__logger.info('deserializer has stopped') return raft_data def is_healthy(self): return self.isHealthy() def is_alive(self): return self.isAlive() def is_ready(self): return self.isReady() def __key_value_to_dict(self, key, value): keys = [k for k in key.split('/') if k != ''] if len(keys) > 1: value = self.__key_value_to_dict('/'.join(keys[1:]), value) return {keys[0]: value} def __put(self, key, value): start_time = time.time() try: if key == '/': self.__data.update(value) else: self.__data.update(self.__key_value_to_dict(key, value)) finally: self.__record_metrics(start_time, 'put') @replicated def put(self, key, value): self.__put(key, value) def get(self, key): start_time = time.time() try: value = self.__data keys = [k for k in key.split('/') if k != ''] for k in keys: value = value.get(k, None) if value is None: return None finally: self.__record_metrics(start_time, 'get') return value def __delete(self, key): start_time = time.time() try: if key == '/': value = dict(self.__data) self.__clear() else: keys = [k for k in key.split('/') if k != ''] value = self.__data i = 0 while i < len(keys): if len(keys[i:]) == 1: return value.pop(keys[i], None) value = value.get(keys[i], None) if value is None: return None i += 1 finally: self.__record_metrics(start_time, 'delete') return value @replicated def delete(self, key): return self.__delete(key) def __clear(self): start_time = time.time() try: self.__data.clear() finally: self.__record_metrics(start_time, 'clear') @replicated def clear(self): self.__clear()
def __init__(self, manager, logger=getLogger(), http_logger=getLogger(), metrics_registry=CollectorRegistry()): self.__manager = manager self.__logger = logger self.__http_logger = http_logger self.__metrics_registry = metrics_registry # metrics self.__metrics_requests_total = Counter( '{0}_manager_http_requests_total'.format(NAME), 'The number of requests.', ['method', 'endpoint', 'status_code'], registry=self.__metrics_registry) self.__metrics_requests_duration_seconds = Histogram( '{0}_manager_http_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', ['method', 'endpoint'], registry=self.__metrics_registry) self.__metrics_requests_bytes_total = Counter( '{0}_manager_http_requests_bytes_total'.format(NAME), 'A summary of the invocation requests bytes.', ['method', 'endpoint'], registry=self.__metrics_registry) self.__metrics_responses_bytes_total = Counter( '{0}_manager_http_responses_bytes_total'.format(NAME), 'A summary of the invocation responses bytes.', ['method', 'endpoint'], registry=self.__metrics_registry) self.app = Flask('supervise_http_server') self.app.add_url_rule('/data', endpoint='put', view_func=self.__put, methods=['PUT'], strict_slashes=False) self.app.add_url_rule('/data', endpoint='get', view_func=self.__get, methods=['GET'], strict_slashes=False) self.app.add_url_rule('/data', endpoint='delete', view_func=self.__delete, methods=['DELETE'], strict_slashes=False) self.app.add_url_rule('/data/<path:key>', endpoint='put', view_func=self.__put, methods=['PUT']) self.app.add_url_rule('/data/<path:key>', endpoint='get', view_func=self.__get, methods=['GET']) self.app.add_url_rule('/data/<path:key>', endpoint='delete', view_func=self.__delete, methods=['DELETE']) self.app.add_url_rule('/metrics', endpoint='metrics', view_func=self.__metrics, methods=['GET']) self.app.add_url_rule('/healthiness', endpoint='healthiness', view_func=self.__healthiness, methods=['GET']) self.app.add_url_rule('/liveness', endpoint='liveness', view_func=self.__liveness, methods=['GET']) self.app.add_url_rule('/readiness', endpoint='readiness', view_func=self.__readiness, methods=['GET']) self.app.add_url_rule('/status', endpoint='status', view_func=self.__get_status, methods=['GET']) # disable Flask default logger self.app.logger.disabled = True getLogger('werkzeug').disabled = True
GaugeMetricFamily, Histogram, Counter, ) from prometheus_client.multiprocess import MultiProcessCollector from sqlalchemy import cast, String from models import count_groups from models.payment import Payment from models.product import Product from models.purchase import Purchase, AdmissionTicket from models.cfp import Proposal metrics = Blueprint('metric', __name__) request_duration = Histogram('emf_request_duration_seconds', "Request duration", ['endpoint', 'method']) request_total = Counter('emf_request_total', "Total request count", ['endpoint', 'method', 'http_status']) def gauge_groups(gauge, query, *entities): for count, *key in count_groups(query, *entities): gauge.add_metric(key, count) class ExternalMetrics: def __init__(self, registry=None): if registry is not None: registry.register(self) def collect(self):
class ManagementHTTPServicer: def __init__(self, manager, logger=getLogger(), http_logger=getLogger(), metrics_registry=CollectorRegistry()): self.__manager = manager self.__logger = logger self.__http_logger = http_logger self.__metrics_registry = metrics_registry # metrics self.__metrics_requests_total = Counter( '{0}_manager_http_requests_total'.format(NAME), 'The number of requests.', ['method', 'endpoint', 'status_code'], registry=self.__metrics_registry) self.__metrics_requests_duration_seconds = Histogram( '{0}_manager_http_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', ['method', 'endpoint'], registry=self.__metrics_registry) self.__metrics_requests_bytes_total = Counter( '{0}_manager_http_requests_bytes_total'.format(NAME), 'A summary of the invocation requests bytes.', ['method', 'endpoint'], registry=self.__metrics_registry) self.__metrics_responses_bytes_total = Counter( '{0}_manager_http_responses_bytes_total'.format(NAME), 'A summary of the invocation responses bytes.', ['method', 'endpoint'], registry=self.__metrics_registry) self.app = Flask('supervise_http_server') self.app.add_url_rule('/data', endpoint='put', view_func=self.__put, methods=['PUT'], strict_slashes=False) self.app.add_url_rule('/data', endpoint='get', view_func=self.__get, methods=['GET'], strict_slashes=False) self.app.add_url_rule('/data', endpoint='delete', view_func=self.__delete, methods=['DELETE'], strict_slashes=False) self.app.add_url_rule('/data/<path:key>', endpoint='put', view_func=self.__put, methods=['PUT']) self.app.add_url_rule('/data/<path:key>', endpoint='get', view_func=self.__get, methods=['GET']) self.app.add_url_rule('/data/<path:key>', endpoint='delete', view_func=self.__delete, methods=['DELETE']) self.app.add_url_rule('/metrics', endpoint='metrics', view_func=self.__metrics, methods=['GET']) self.app.add_url_rule('/healthiness', endpoint='healthiness', view_func=self.__healthiness, methods=['GET']) self.app.add_url_rule('/liveness', endpoint='liveness', view_func=self.__liveness, methods=['GET']) self.app.add_url_rule('/readiness', endpoint='readiness', view_func=self.__readiness, methods=['GET']) self.app.add_url_rule('/status', endpoint='status', view_func=self.__get_status, methods=['GET']) # disable Flask default logger self.app.logger.disabled = True getLogger('werkzeug').disabled = True def __record_metrics(self, start_time, req, resp): self.__metrics_requests_total.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else ''), status_code=resp.status_code.value).inc() self.__metrics_requests_bytes_total.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '')).inc( req.content_length if req.content_length is not None else 0) self.__metrics_responses_bytes_total.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '')).inc( resp.content_length if resp.content_length is not None else 0) self.__metrics_requests_duration_seconds.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '')).observe(time.time() - start_time) return def __put(self, key=''): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: mime = mimeparse.parse_mime_type( request.headers.get('Content-Type')) charset = 'utf-8' if mime[2].get( 'charset') is None else mime[2].get('charset') if mime[1] == 'yaml': value = yaml.safe_load(request.data.decode(charset)) elif mime[1] == 'json': value = json.loads(request.data.decode(charset)) else: # handle as a string value = request.data.decode(charset) sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True self.__manager.put(key if key.startswith('/') else '/' + key, value, sync=sync) if sync: status_code = HTTPStatus.CREATED else: status_code = HTTPStatus.ACCEPTED except (ConstructorError, JSONDecodeError, ValueError) as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.BAD_REQUEST self.__logger.error(ex) except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __get(self, key=''): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: value = self.__manager.get(key if key.startswith('/') else '/' + key) if value is None: status_code = HTTPStatus.NOT_FOUND else: data['value'] = value status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __delete(self, key=''): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True value = self.__manager.delete(key if key.startswith('/') else '/' + key, sync=sync) if value is None: status_code = HTTPStatus.NOT_FOUND else: status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __get_status(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, resp) return response data = {} status_code = None try: data['node_status'] = self.__manager.getStatus() status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __healthiness(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: healthy = self.__manager.is_healthy() data['healthy'] = healthy if healthy: status_code = HTTPStatus.OK else: status_code = HTTPStatus.SERVICE_UNAVAILABLE data['error'] = 'node is not healthy' except Exception as ex: data['healthy'] = False data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __liveness(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: alive = self.__manager.is_alive() data['liveness'] = alive if alive: status_code = HTTPStatus.OK else: status_code = HTTPStatus.SERVICE_UNAVAILABLE data['error'] = 'node is not alive' except Exception as ex: data['liveness'] = False data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __readiness(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: ready = self.__manager.is_ready() data['readiness'] = ready if ready: status_code = HTTPStatus.OK else: status_code = HTTPStatus.SERVICE_UNAVAILABLE data['error'] = 'node is not ready' except Exception as ex: data['readiness'] = False data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __metrics(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response resp = Response() try: resp.status_code = HTTPStatus.OK resp.content_type = CONTENT_TYPE_LATEST resp.data = generate_latest(self.__metrics_registry) except Exception as ex: resp.status_code = HTTPStatus.INTERNAL_SERVER_ERROR resp.content_type = 'text/plain; charset="UTF-8"' resp.data = '{0}\n{1}'.format(resp.status_code.phrase, resp.status_code.description) self.__logger.error(ex) return resp
def test_empty_labels_list(self): Histogram('h', 'help', [], registry=self.registry) self.assertEqual(0, self.registry.get_sample_value('h_sum'))
class TestHistogram(unittest.TestCase): def setUp(self): self.registry = CollectorRegistry() self.histogram = Histogram('h', 'help', registry=self.registry) self.labels = Histogram('hl', 'help', ['l'], registry=self.registry) def test_histogram(self): self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual(0, self.registry.get_sample_value('h_sum')) self.histogram.observe(2) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual(2, self.registry.get_sample_value('h_sum')) self.histogram.observe(2.5) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(2, self.registry.get_sample_value('h_count')) self.assertEqual(4.5, self.registry.get_sample_value('h_sum')) self.histogram.observe(float("inf")) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 3, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(3, self.registry.get_sample_value('h_count')) self.assertEqual(float("inf"), self.registry.get_sample_value('h_sum')) def test_setting_buckets(self): h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2, float("inf")]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[float("inf")]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[3, 1]) def test_labels(self): self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, labelnames=['le']) self.labels.labels('a').observe(2) self.assertEqual( 0, self.registry.get_sample_value('hl_bucket', { 'le': '1.0', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_bucket', { 'le': '2.5', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_bucket', { 'le': '5.0', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_bucket', { 'le': '+Inf', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_count', {'l': 'a'})) self.assertEqual(2, self.registry.get_sample_value('hl_sum', {'l': 'a'})) def test_function_decorator(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) @self.histogram.time() def f(): pass self.assertEqual(([], None, None, None), inspect.getargspec(f)) f() self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) def test_function_decorator_multithread(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) workers = 3 duration = 0.1 pool = ThreadPoolExecutor(max_workers=workers) @self.histogram.time() def f(): time.sleep(duration) jobs = workers * 3 for i in range(jobs): pool.submit(f) pool.shutdown(wait=True) self.assertEqual(jobs, self.registry.get_sample_value('h_count')) rounding_coefficient = 0.9 total_expected_duration = jobs * duration * rounding_coefficient self.assertLess(total_expected_duration, self.registry.get_sample_value('h_sum')) def test_block_decorator(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) with self.histogram.time(): pass self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
# gc_unreachable = Gauge("python_gc_unreachable_total", "Unreachable GC objects", ["gen"]) gc_time = Histogram( "python_gc_time", "Time taken to GC (sec)", ["gen"], buckets=[ 0.0025, 0.005, 0.01, 0.025, 0.05, 0.10, 0.25, 0.50, 1.00, 2.50, 5.00, 7.50, 15.00, 30.00, 45.00, 60.00, ], ) class GCCounts(Collector): def collect(self) -> Iterable[Metric]:
class IndexHTTPServicer: def __init__(self, indexer, logger=getLogger(), http_logger=getLogger(), metrics_registry=CollectorRegistry()): self.__indexer = indexer self.__logger = logger self.__http_logger = http_logger self.__metrics_registry = metrics_registry # metrics self.__metrics_requests_total = Counter( '{0}_indexer_http_requests_total'.format(NAME), 'The number of requests.', [ 'method', 'endpoint', 'status_code' ], registry=self.__metrics_registry ) self.__metrics_requests_duration_seconds = Histogram( '{0}_indexer_http_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', [ 'method', 'endpoint' ], registry=self.__metrics_registry ) self.__metrics_requests_bytes_total = Counter( '{0}_indexer_http_requests_bytes_total'.format(NAME), 'A summary of the invocation requests bytes.', [ 'method', 'endpoint' ], registry=self.__metrics_registry ) self.__metrics_responses_bytes_total = Counter( '{0}_indexer_http_responses_bytes_total'.format(NAME), 'A summary of the invocation responses bytes.', [ 'method', 'endpoint' ], registry=self.__metrics_registry ) self.app = Flask('indexer_http') self.app.add_url_rule('/', endpoint='root', view_func=self.__root, methods=['GET']) self.app.add_url_rule('/indices/<index_name>', endpoint='get_index', view_func=self.__get_index, methods=['GET']) self.app.add_url_rule('/indices/<index_name>', endpoint='create_index', view_func=self.__create_index, methods=['PUT']) self.app.add_url_rule('/indices/<index_name>', endpoint='delete_index', view_func=self.__delete_index, methods=['DELETE']) self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='get_document', view_func=self.__get_document, methods=['GET']) self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='put_document', view_func=self.__put_document, methods=['PUT']) self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='delete_document', view_func=self.__delete_document, methods=['DELETE']) self.app.add_url_rule('/indices/<index_name>/documents', endpoint='put_documents', view_func=self.__put_documents, methods=['PUT']) self.app.add_url_rule('/indices/<index_name>/documents', endpoint='delete_documents', view_func=self.__delete_documents, methods=['DELETE']) self.app.add_url_rule('/indices/<index_name>/search', endpoint='search_documents', view_func=self.__search_documents, methods=['GET', 'POST']) self.app.add_url_rule('/indices/<index_name>/optimize', endpoint='optimize_index', view_func=self.__optimize_index, methods=['GET']) self.app.add_url_rule('/indices/<index_name>/commit', endpoint='commit', view_func=self.__commit_index, methods=['GET']) self.app.add_url_rule('/indices/<index_name>/rollback', endpoint='rollback', view_func=self.__rollback_index, methods=['GET']) self.app.add_url_rule('/nodes/<node_name>', endpoint='put_node', view_func=self.__put_node, methods=['PUT']) self.app.add_url_rule('/nodes/<node_name>', endpoint='delete_node', view_func=self.__delete_node, methods=['DELETE']) self.app.add_url_rule('/snapshot', endpoint='get_snapshot', view_func=self.__get_snapshot, methods=['GET']) self.app.add_url_rule('/snapshot', endpoint='put_snapshot', view_func=self.__put_snapshot, methods=['PUT']) self.app.add_url_rule('/metrics', endpoint='metrics', view_func=self.__metrics, methods=['GET']) self.app.add_url_rule('/healthiness', endpoint='healthiness', view_func=self.__healthiness, methods=['GET']) self.app.add_url_rule('/liveness', endpoint='liveness', view_func=self.__liveness, methods=['GET']) self.app.add_url_rule('/readiness', endpoint='readiness', view_func=self.__readiness, methods=['GET']) self.app.add_url_rule('/status', endpoint='status', view_func=self.__get_status, methods=['GET']) # disable Flask default logger self.app.logger.disabled = True getLogger('werkzeug').disabled = True def __record_metrics(self, start_time, req, resp): self.__metrics_requests_total.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else ''), status_code=resp.status_code.value ).inc() self.__metrics_requests_bytes_total.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '') ).inc(req.content_length if req.content_length is not None else 0) self.__metrics_responses_bytes_total.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '') ).inc(resp.content_length if resp.content_length is not None else 0) self.__metrics_requests_duration_seconds.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '') ).observe(time.time() - start_time) return def __root(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response resp = Response() try: resp.status_code = HTTPStatus.OK resp.content_type = 'text/plain; charset="UTF-8"' resp.data = NAME + ' ' + VERSION + ' is running.\n' except Exception as ex: resp.status_code = HTTPStatus.INTERNAL_SERVER_ERROR resp.content_type = 'text/plain; charset="UTF-8"' resp.data = '{0}\n{1}'.format(resp.status_code.phrase, resp.status_code.description) self.__logger.error(ex) return resp def __create_index(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: mime = mimeparse.parse_mime_type(request.headers.get('Content-Type')) charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset') if mime[1] == 'yaml': index_config_dict = yaml.safe_load(request.data.decode(charset)) elif mime[1] == 'json': index_config_dict = json.loads(request.data.decode(charset)) else: raise ValueError('unsupported format') if index_config_dict is None: raise ValueError('index config is None') sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True index_config = IndexConfig(index_config_dict) self.__indexer.create_index(index_name, index_config, sync=sync) if sync: status_code = HTTPStatus.CREATED else: status_code = HTTPStatus.ACCEPTED except (ConstructorError, JSONDecodeError, ValueError) as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.BAD_REQUEST self.__logger.error(ex) except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __get_index(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: index = self.__indexer.get_index(index_name) if index is None: status_code = HTTPStatus.NOT_FOUND else: data['index'] = { 'name': index.indexname, 'doc_count': index.doc_count(), 'doc_count_all': index.doc_count_all(), 'last_modified': index.latest_generation(), 'latest_generation': index.last_modified(), 'version': index.version, 'storage': { 'folder': index.storage.folder, 'supports_mmap': index.storage.supports_mmap, 'readonly': index.storage.readonly, 'files': list(index.storage.list()) } } status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __delete_index(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True self.__indexer.delete_index(index_name, sync=sync) if sync: status_code = HTTPStatus.OK else: status_code = HTTPStatus.ACCEPTED except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __commit_index(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True self.__indexer.commit_index(index_name, sync=sync) if sync: status_code = HTTPStatus.OK else: status_code = HTTPStatus.ACCEPTED except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __rollback_index(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True self.__indexer.rollback_index(index_name, sync=sync) if sync: status_code = HTTPStatus.OK else: status_code = HTTPStatus.ACCEPTED except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __optimize_index(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True self.__indexer.optimize_index(index_name, sync=sync) if sync: status_code = HTTPStatus.OK else: status_code = HTTPStatus.ACCEPTED except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __put_document(self, index_name, doc_id): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: mime = mimeparse.parse_mime_type(request.headers.get('Content-Type')) charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset') if mime[1] == 'yaml': fields_dict = yaml.safe_load(request.data.decode(charset)) elif mime[1] == 'json': fields_dict = json.loads(request.data.decode(charset)) else: raise ValueError('unsupported format') if fields_dict is None: raise ValueError('fields are None') sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True count = self.__indexer.put_document(index_name, doc_id, fields_dict, sync=sync) if sync: if count > 0: data['count'] = count status_code = HTTPStatus.CREATED else: status_code = HTTPStatus.INTERNAL_SERVER_ERROR else: status_code = HTTPStatus.ACCEPTED except (ConstructorError, JSONDecodeError, ValueError) as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.BAD_REQUEST self.__logger.error(ex) except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __get_document(self, index_name, doc_id): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: results_page = self.__indexer.get_document(index_name, doc_id) if results_page.total > 0: fields = {} for i in results_page.results[0].iteritems(): fields[i[0]] = i[1] data['fields'] = fields status_code = HTTPStatus.OK else: status_code = HTTPStatus.NOT_FOUND except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __delete_document(self, index_name, doc_id): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True count = self.__indexer.delete_document(index_name, doc_id, sync=sync) if sync: if count > 0: status_code = HTTPStatus.OK elif count == 0: status_code = HTTPStatus.NOT_FOUND else: status_code = HTTPStatus.INTERNAL_SERVER_ERROR else: status_code = HTTPStatus.ACCEPTED except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __put_documents(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: mime = mimeparse.parse_mime_type(request.headers.get('Content-Type')) charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset') if mime[1] == 'yaml': docs_dict = yaml.safe_load(request.data.decode(charset)) elif mime[1] == 'json': docs_dict = json.loads(request.data.decode(charset)) else: raise ValueError('unsupported format') sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True count = self.__indexer.put_documents(index_name, docs_dict, sync=sync) if sync: if count > 0: data['count'] = count status_code = HTTPStatus.CREATED else: status_code = HTTPStatus.INTERNAL_SERVER_ERROR else: status_code = HTTPStatus.ACCEPTED except (ConstructorError, JSONDecodeError, ValueError) as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.BAD_REQUEST self.__logger.error(ex) except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __delete_documents(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: mime = mimeparse.parse_mime_type(request.headers.get('Content-Type')) charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset') if mime[1] == 'yaml': doc_ids_list = yaml.safe_load(request.data.decode(charset)) elif mime[1] == 'json': doc_ids_list = json.loads(request.data.decode(charset)) else: raise ValueError('unsupported format') sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True count = self.__indexer.delete_documents(index_name, doc_ids_list, sync=sync) if sync: if count > 0: data['count'] = count status_code = HTTPStatus.OK else: status_code = HTTPStatus.INTERNAL_SERVER_ERROR else: status_code = HTTPStatus.ACCEPTED except (ConstructorError, JSONDecodeError, ValueError) as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.BAD_REQUEST self.__logger.error(ex) except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __search_documents(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, resp) return response data = {} status_code = None try: query = request.args.get('query', default='', type=str) search_field = request.args.get('search_field', default='', type=str) page_num = request.args.get('page_num', default=1, type=int) page_len = request.args.get('page_len', default=10, type=int) weighting = BM25F if len(request.data) > 0: mime = mimeparse.parse_mime_type(request.headers.get('Content-Type')) charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset') if mime[1] == 'yaml': weighting = get_multi_weighting(yaml.safe_load(request.data.decode(charset))) elif mime[1] == 'json': weighting = get_multi_weighting(json.loads(request.data.decode(charset))) else: raise ValueError('unsupported format') results_page = self.__indexer.search_documents(index_name, query, search_field, page_num, page_len=page_len, weighting=weighting) if results_page.pagecount >= page_num or results_page.total <= 0: results = { 'is_last_page': results_page.is_last_page(), 'page_count': results_page.pagecount, 'page_len': results_page.pagelen, 'page_num': results_page.pagenum, 'total': results_page.total, 'offset': results_page.offset } hits = [] for result in results_page.results[results_page.offset:]: fields = {} for item in result.iteritems(): fields[item[0]] = item[1] hit = { 'fields': fields, 'doc_num': result.docnum, 'score': result.score, 'rank': result.rank, 'pos': result.pos } hits.append(hit) results['hits'] = hits data['results'] = results status_code = HTTPStatus.OK else: data['error'] = 'page_num must be <= {0}'.format(results_page.pagecount) status_code = HTTPStatus.BAD_REQUEST except (ConstructorError, JSONDecodeError, ValueError) as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.BAD_REQUEST self.__logger.error(ex) except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __put_node(self, node_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, resp) return response data = {} status_code = None try: self.__indexer.addNodeToCluster(node_name) status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __get_status(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, resp) return response data = {} status_code = None try: data['node_status'] = self.__indexer.getStatus() status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __delete_node(self, node_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, resp) return response data = {} status_code = None try: self.__indexer.removeNodeFromCluster(node_name) status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __put_snapshot(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, resp) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True self.__indexer.create_snapshot(sync=sync) status_code = HTTPStatus.ACCEPTED except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __get_snapshot(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response try: if self.__indexer.is_snapshot_exist(): def generate(): with self.__indexer.open_snapshot_file() as f: chunk = f.read(1024) yield chunk resp = Response(generate(), status=HTTPStatus.OK, mimetype='application/zip', headers={ 'Content-Disposition': 'attachment; filename=snapshot.zip' }) else: resp = Response(status=HTTPStatus.NOT_FOUND) except Exception as ex: resp = Response(status=HTTPStatus.INTERNAL_SERVER_ERROR) self.__logger.error(ex) return resp def __healthiness(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: healthy = self.__indexer.is_healthy() data['healthy'] = healthy if healthy: status_code = HTTPStatus.OK else: status_code = HTTPStatus.SERVICE_UNAVAILABLE data['error'] = 'node is not healthy' except Exception as ex: data['healthy'] = False data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __liveness(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: alive = self.__indexer.is_alive() data['liveness'] = alive if alive: status_code = HTTPStatus.OK else: status_code = HTTPStatus.SERVICE_UNAVAILABLE data['error'] = 'node is not alive' except Exception as ex: data['liveness'] = False data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __readiness(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: ready = self.__indexer.is_ready() data['readiness'] = ready if ready: status_code = HTTPStatus.OK else: status_code = HTTPStatus.SERVICE_UNAVAILABLE data['error'] = 'node is not ready' except Exception as ex: data['readiness'] = False data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __metrics(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response resp = Response() try: resp.status_code = HTTPStatus.OK resp.content_type = CONTENT_TYPE_LATEST resp.data = generate_latest(self.__metrics_registry) except Exception as ex: resp.status_code = HTTPStatus.INTERNAL_SERVER_ERROR resp.content_type = 'text/plain; charset="UTF-8"' resp.data = '{0}\n{1}'.format(resp.status_code.phrase, resp.status_code.description) self.__logger.error(ex) return resp
class Indexer(RaftNode): def __init__(self, host='localhost', port=7070, seed_addr=None, conf=SyncObjConf(), data_dir='/tmp/cockatrice/index', grpc_port=5050, grpc_max_workers=10, http_port=8080, logger=getLogger(), http_logger=getLogger(), metrics_registry=CollectorRegistry()): self.__host = host self.__port = port self.__seed_addr = seed_addr self.__conf = conf self.__data_dir = data_dir self.__grpc_port = grpc_port self.__grpc_max_workers = grpc_max_workers self.__http_port = http_port self.__logger = logger self.__http_logger = http_logger self.__metrics_registry = metrics_registry # metrics self.__metrics_core_documents = Gauge( '{0}_indexer_index_documents'.format(NAME), 'The number of documents.', [ 'index_name', ], registry=self.__metrics_registry) self.__metrics_requests_total = Counter( '{0}_indexer_requests_total'.format(NAME), 'The number of requests.', ['func'], registry=self.__metrics_registry) self.__metrics_requests_duration_seconds = Histogram( '{0}_indexer_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', ['func'], registry=self.__metrics_registry) self.__self_addr = '{0}:{1}'.format(self.__host, self.__port) self.__peer_addrs = [] if self.__seed_addr is None else get_peers( bind_addr=self.__seed_addr, timeout=10) self.__other_addrs = [ peer_addr for peer_addr in self.__peer_addrs if peer_addr != self.__self_addr ] self.__conf.serializer = self.__serialize self.__conf.deserializer = self.__deserialize self.__conf.validate() self.__indices = {} self.__index_configs = {} self.__writers = {} self.__auto_commit_timers = {} self.__lock = RLock() # create data dir os.makedirs(self.__data_dir, exist_ok=True) self.__file_storage = FileStorage(self.__data_dir, supports_mmap=True, readonly=False, debug=False) self.__ram_storage = RamStorage() # if seed addr specified and self node does not exist in the cluster, add self node to the cluster if self.__seed_addr is not None and self.__self_addr not in self.__peer_addrs: Thread(target=add_node, kwargs={ 'node_name': self.__self_addr, 'bind_addr': self.__seed_addr, 'timeout': 10 }).start() # copy snapshot from the leader node if self.__seed_addr is not None: try: metadata = get_metadata(bind_addr=get_leader( bind_addr=self.__seed_addr, timeout=10), timeout=10) response = requests.get('http://{0}/snapshot'.format( metadata['http_addr'])) if response.status_code == HTTPStatus.OK: with open(self.__conf.fullDumpFile, 'wb') as f: f.write(response.content) except Exception as ex: self.__logger.error('failed to copy snapshot: {0}'.format(ex)) # start node metadata = { 'grpc_addr': '{0}:{1}'.format(self.__host, self.__grpc_port), 'http_addr': '{0}:{1}'.format(self.__host, self.__http_port) } self.__logger.info('starting raft state machine') super(Indexer, self).__init__(self.__self_addr, self.__peer_addrs, conf=self.__conf, metadata=metadata) self.__logger.info('raft state machine has started') if os.path.exists(self.__conf.fullDumpFile): self.__logger.debug('snapshot exists: {0}'.format( self.__conf.fullDumpFile)) else: pass while not self.isReady(): # recovering data self.__logger.debug('waiting for cluster ready') self.__logger.debug(self.getStatus()) time.sleep(1) self.__logger.info('cluster ready') self.__logger.debug(self.getStatus()) # open existing indices on startup for index_name in self.get_index_names(): self.__open_index(index_name, index_config=None) # record index metrics timer self.metrics_timer = Timer(10, self.__record_index_metrics) self.metrics_timer.start() # start gRPC self.__grpc_server = grpc.server( futures.ThreadPoolExecutor(max_workers=self.__grpc_max_workers)) add_IndexServicer_to_server( IndexGRPCServicer(self, logger=self.__logger, metrics_registry=self.__metrics_registry), self.__grpc_server) self.__grpc_server.add_insecure_port('{0}:{1}'.format( self.__host, self.__grpc_port)) self.__grpc_server.start() self.__logger.info('gRPC server has started') # start HTTP server self.__http_servicer = IndexHTTPServicer(self, self.__logger, self.__http_logger, self.__metrics_registry) self.__http_server = HTTPServer(self.__host, self.__http_port, self.__http_servicer) self.__http_server.start() self.__logger.info('HTTP server has started') self.__logger.info('indexer has started') def stop(self): # stop HTTP server self.__http_server.stop() self.__logger.info('HTTP server has stopped') # stop gRPC server self.__grpc_server.stop(grace=0.0) self.__logger.info('gRPC server has stopped') self.metrics_timer.cancel() # close indices for index_name in list(self.__indices.keys()): self.__close_index(index_name) self.destroy() self.__logger.info('index core has stopped') def __record_index_metrics(self): for index_name in list(self.__indices.keys()): try: self.__metrics_core_documents.labels( index_name=index_name).set(self.get_doc_count(index_name)) except Exception as ex: self.__logger.error(ex) def __record_metrics(self, start_time, func_name): self.__metrics_requests_total.labels(func=func_name).inc() self.__metrics_requests_duration_seconds.labels( func=func_name).observe(time.time() - start_time) # def __serialize_indices(self, filename): # with self.__lock: # try: # self.__logger.info('starting serialize indices') # # except Exception as ex: # self.__logger.error('failed to create snapshot: {0}'.format(ex)) # finally: # self.__logger.info('serialize indices has finished') # def __serialize_raft_data(self, filename, raft_data): # with self.__lock: # pass # index serializer def __serialize(self, filename, raft_data): with self.__lock: try: self.__logger.debug('serializer has started') # store the index files and raft logs to the snapshot file with zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED) as f: for index_name in self.get_index_names(): self.__commit_index(index_name) # with self.__get_writer(index_name).writelock: # with self.__indices[index_name].lock('WRITELOCK'): # index files for index_filename in self.get_index_files(index_name): if self.__index_configs.get( index_name).get_storage_type() == "ram": with self.__ram_storage.open_file( index_filename) as r: f.writestr(index_filename, r.read()) else: f.write( os.path.join(self.__file_storage.folder, index_filename), index_filename) self.__logger.debug('{0} has stored in {1}'.format( index_filename, filename)) # index config file f.write( os.path.join( self.__file_storage.folder, self.get_index_config_file(index_name)), self.get_index_config_file(index_name)) self.__logger.debug('{0} has stored in {1}'.format( self.get_index_config_file(index_name), filename)) # store the raft data f.writestr(RAFT_DATA_FILE, pickle.dumps(raft_data)) self.__logger.debug( '{0} has restored'.format(RAFT_DATA_FILE)) self.__logger.debug('snapshot has created') except Exception as ex: self.__logger.error( 'failed to create snapshot: {0}'.format(ex)) finally: self.__logger.debug('serializer has stopped') # index deserializer def __deserialize(self, filename): with self.__lock: try: self.__logger.debug('deserializer has started') with zipfile.ZipFile(filename, 'r') as zf: # get file names in snapshot file filenames = list(zf.namelist()) # get index names in snapshot file index_names = [] pattern_toc = re.compile(r'^_(.+)_\d+\.toc$') for f in filenames: match = pattern_toc.search(f) if match and match.group(1) not in index_names: index_names.append(match.group(1)) for index_name in index_names: # extract the index config first zf.extract(self.get_index_config_file(index_name), path=self.__file_storage.folder) index_config = pickle.loads( zf.read(self.get_index_config_file(index_name))) # get index files pattern_toc = re.compile(r'^_{0}_(\d+)\..+$'.format( index_name)) # ex) _myindex_0.toc pattern_seg = re.compile( r'^{0}_([a-z0-9]+)\..+$'.format(index_name) ) # ex) myindex_zseabukc2nbpvh0u.seg pattern_lock = re.compile(r'^{0}_WRITELOCK$'.format( index_name)) # ex) myindex_WRITELOCK index_files = [] for file_name in filenames: if re.match(pattern_toc, file_name): index_files.append(file_name) elif re.match(pattern_seg, file_name): index_files.append(file_name) elif re.match(pattern_lock, file_name): index_files.append(file_name) # extract the index files for index_file in index_files: if index_config.get_storage_type() == 'ram': with self.__ram_storage.create_file( index_file) as r: r.write(zf.read(index_file)) else: zf.extract(index_file, path=self.__file_storage.folder) self.__logger.debug( '{0} has restored from {1}'.format( index_file, filename)) self.__logger.debug( '{0} has restored'.format(index_name)) # extract the raft data raft_data = pickle.loads(zf.read(RAFT_DATA_FILE)) self.__logger.debug( '{0} has restored'.format(RAFT_DATA_FILE)) return raft_data except Exception as ex: self.__logger.error( 'failed to restore indices: {0}'.format(ex)) finally: self.__logger.debug('deserializer has stopped') def is_healthy(self): return self.isHealthy() def is_alive(self): return self.isAlive() def is_ready(self): return self.isReady() def get_addr(self): return self.__self_addr def get_index_files(self, index_name): index_files = [] pattern_toc = re.compile( r'^_{0}_(\d+)\..+$'.format(index_name)) # ex) _myindex_0.toc pattern_seg = re.compile(r'^{0}_([a-z0-9]+)\..+$'.format( index_name)) # ex) myindex_zseabukc2nbpvh0u.seg pattern_lock = re.compile( r'^{0}_WRITELOCK$'.format(index_name)) # ex) myindex_WRITELOCK if self.__index_configs.get(index_name).get_storage_type() == "ram": storage = self.__ram_storage else: storage = self.__file_storage for file_name in list(storage.list()): if re.match(pattern_toc, file_name): index_files.append(file_name) elif re.match(pattern_seg, file_name): index_files.append(file_name) elif re.match(pattern_lock, file_name): index_files.append(file_name) return index_files @staticmethod def get_index_config_file(index_name): return '{0}_CONFIG'.format(index_name) def get_index_names(self): index_names = [] pattern_toc = re.compile(r'^_(.+)_\d+\.toc$') for filename in list(self.__file_storage.list()): match = pattern_toc.search(filename) if match and match.group(1) not in index_names: index_names.append(match.group(1)) for filename in list(self.__ram_storage.list()): match = pattern_toc.search(filename) if match and match.group(1) not in index_names: index_names.append(match.group(1)) return index_names def is_index_exist(self, index_name): return self.__file_storage.index_exists( indexname=index_name) or self.__ram_storage.index_exists( indexname=index_name) def is_index_open(self, index_name): return index_name in self.__indices @replicated def open_index(self, index_name, index_config=None): return self.__open_index(index_name, index_config=index_config) def __open_index(self, index_name, index_config=None): start_time = time.time() index = None try: # open the index index = self.__indices.get(index_name) if index is None: self.__logger.debug('opening {0}'.format(index_name)) if index_config is None: # set saved index config with open( os.path.join( self.__file_storage.folder, self.get_index_config_file(index_name)), 'rb') as f: self.__index_configs[index_name] = pickle.loads( f.read()) else: # set given index config self.__index_configs[index_name] = index_config if self.__index_configs[index_name].get_storage_type( ) == 'ram': index = self.__ram_storage.open_index( indexname=index_name, schema=self.__index_configs[index_name].get_schema()) else: index = self.__file_storage.open_index( indexname=index_name, schema=self.__index_configs[index_name].get_schema()) self.__indices[index_name] = index self.__logger.info('{0} has opened'.format(index_name)) # open the index writer self.__open_writer(index_name) except Exception as ex: self.__logger.error('failed to open {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'open_index') return index @replicated def close_index(self, index_name): return self.__close_index(index_name) def __close_index(self, index_name): start_time = time.time() index = None try: # close the index writer self.__close_writer(index_name) # close the index index = self.__indices.pop(index_name) if index is not None: self.__logger.debug('closing {0}'.format(index_name)) index.close() self.__logger.info('{0} has closed'.format(index_name)) except Exception as ex: self.__logger.error('failed to close {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'close_index') return index @replicated def create_index(self, index_name, index_config): return self.__create_index(index_name, index_config) def __create_index(self, index_name, index_config): if self.is_index_exist(index_name): # open the index return self.__open_index(index_name, index_config=index_config) start_time = time.time() index = None with self.__lock: try: self.__logger.debug('creating {0}'.format(index_name)) # set index config self.__index_configs[index_name] = index_config self.__logger.debug( self.__index_configs[index_name].get_storage_type()) # create the index if self.__index_configs[index_name].get_storage_type( ) == 'ram': index = self.__ram_storage.create_index( self.__index_configs[index_name].get_schema(), indexname=index_name) else: index = self.__file_storage.create_index( self.__index_configs[index_name].get_schema(), indexname=index_name) self.__indices[index_name] = index self.__logger.info('{0} has created'.format(index_name)) # save the index config with open( os.path.join(self.__file_storage.folder, self.get_index_config_file(index_name)), 'wb') as f: f.write(pickle.dumps(index_config)) # open the index writer self.__open_writer(index_name) except Exception as ex: self.__logger.error('failed to create {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'create_index') return index @replicated def delete_index(self, index_name): return self.__delete_index(index_name) def __delete_index(self, index_name): # close index index = self.__close_index(index_name) start_time = time.time() with self.__lock: try: self.__logger.debug('deleting {0}'.format(index_name)) # delete index files for filename in self.get_index_files(index_name): self.__file_storage.delete_file(filename) self.__logger.debug('{0} was deleted'.format(filename)) self.__logger.info('{0} has deleted'.format(index_name)) # delete the index config self.__index_configs.pop(index_name, None) os.remove( os.path.join(self.__file_storage.folder, self.get_index_config_file(index_name))) except Exception as ex: self.__logger.error('failed to delete {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'delete_index') return index def get_index(self, index_name): return self.__get_index(index_name) def __get_index(self, index_name): start_time = time.time() try: index = self.__indices.get(index_name) except Exception as ex: raise ex finally: self.__record_metrics(start_time, 'get_index') return index def __start_auto_commit_timer(self, index_name, period): timer = self.__auto_commit_timers.get(index_name, None) if timer is None: self.__auto_commit_timers[index_name] = threading.Timer( period, self.__auto_commit_index, kwargs={ 'index_name': index_name, 'period': period }) self.__auto_commit_timers[index_name].start() self.__logger.debug( 'auto commit timer for {0} were started'.format(index_name)) def __stop_auto_commit_timer(self, index_name): timer = self.__auto_commit_timers.pop(index_name, None) if timer is not None: timer.cancel() self.__logger.debug( 'auto commit timer for {0} were stopped'.format(index_name)) def __auto_commit_index(self, index_name, period): self.__stop_auto_commit_timer(index_name) self.__commit_index(index_name) self.__start_auto_commit_timer(index_name, period=period) def __open_writer(self, index_name): writer = None try: writer = self.__writers.get(index_name, None) if writer is None or writer.is_closed: self.__logger.debug( 'opening writer for {0}'.format(index_name)) writer = self.__indices.get(index_name).writer() self.__writers[index_name] = writer self.__logger.debug( 'writer for {0} has opened'.format(index_name)) self.__start_auto_commit_timer( index_name, period=self.__index_configs.get( index_name).get_writer_auto_commit_period()) except Exception as ex: self.__logger.error('failed to open writer for {0}: {1}'.format( index_name, ex)) return writer def __close_writer(self, index_name): writer = None try: self.__stop_auto_commit_timer(index_name) # close the index writer = self.__writers.pop(index_name, None) if writer is not None: self.__logger.debug( 'closing writer for {0}'.format(index_name)) writer.commit() self.__logger.debug( 'writer for {0} has closed'.format(index_name)) except Exception as ex: self.__logger.error('failed to close writer for {0}: {1}'.format( index_name, ex)) return writer def __get_writer(self, index_name): return self.__writers.get(index_name, None) def __get_searcher(self, index_name, weighting=None): try: if weighting is None: searcher = self.__indices.get(index_name).searcher() else: searcher = self.__indices.get(index_name).searcher( weighting=weighting) except Exception as ex: raise ex return searcher @replicated def commit_index(self, index_name): return self.__commit_index(index_name) def __commit_index(self, index_name): start_time = time.time() success = False with self.__lock: try: self.__logger.debug('committing {0}'.format(index_name)) self.__get_writer(index_name).commit() self.__open_writer(index_name) # reopen writer self.__logger.info('{0} has committed'.format(index_name)) success = True except Exception as ex: self.__logger.error('failed to commit index {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'commit_index') return success @replicated def rollback_index(self, index_name): return self.__rollback_index(index_name) def __rollback_index(self, index_name): start_time = time.time() success = False with self.__lock: try: self.__logger.debug('rolling back {0}'.format(index_name)) self.__get_writer(index_name).cancel() self.__open_writer(index_name) # reopen writer self.__logger.info('{0} has rolled back'.format(index_name)) success = True except Exception as ex: self.__logger.error('failed to rollback index {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'rollback_index') return success @replicated def optimize_index(self, index_name): return self.__optimize_index(index_name) def __optimize_index(self, index_name): start_time = time.time() success = False with self.__lock: try: self.__logger.debug('optimizing {0}'.format(index_name)) self.__get_writer(index_name).commit(optimize=True, merge=False) self.__open_writer(index_name) # reopen writer self.__logger.info('{0} has optimized'.format(index_name)) success = True except Exception as ex: self.__logger.error('failed to optimize {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'optimize_index') return success def get_doc_count(self, index_name): try: cnt = self.__indices.get(index_name).doc_count() except Exception as ex: raise ex return cnt def get_schema(self, index_name): try: schema = self.__indices.get(index_name).schema except Exception as ex: raise ex return schema @replicated def put_document(self, index_name, doc_id, fields): return self.__put_document(index_name, doc_id, fields) def __put_document(self, index_name, doc_id, fields): doc = copy.deepcopy(fields) doc[self.__index_configs.get(index_name).get_doc_id_field()] = doc_id return self.__put_documents(index_name, [doc]) @replicated def put_documents(self, index_name, docs): return self.__put_documents(index_name, docs) def __put_documents(self, index_name, docs): start_time = time.time() with self.__lock: try: self.__logger.debug( 'putting documents to {0}'.format(index_name)) # count = self.__get_writer(index_name).update_documents(docs) count = 0 for doc in docs: self.__get_writer(index_name).update_document(**doc) count += 1 self.__logger.info('{0} documents has put to {1}'.format( count, index_name)) except Exception as ex: self.__logger.error( 'failed to put documents to {0}: {1}'.format( index_name, ex)) count = -1 finally: self.__record_metrics(start_time, 'put_documents') return count def get_document(self, index_name, doc_id): try: results_page = self.search_documents( index_name, doc_id, self.__index_configs.get(index_name).get_doc_id_field(), 1, page_len=1) if results_page.total > 0: self.__logger.debug('{0} was got from {1}'.format( doc_id, index_name)) else: self.__logger.debug('{0} did not exist in {1}'.format( doc_id, index_name)) except Exception as ex: raise ex return results_page @replicated def delete_document(self, index_name, doc_id): return self.__delete_document(index_name, doc_id) def __delete_document(self, index_name, doc_id): return self.__delete_documents(index_name, [doc_id]) @replicated def delete_documents(self, index_name, doc_ids): return self.__delete_documents(index_name, doc_ids) def __delete_documents(self, index_name, doc_ids): start_time = time.time() with self.__lock: try: self.__logger.debug( 'deleting documents from {0}'.format(index_name)) # count = self.__get_writer(index_name).delete_documents(doc_ids, doc_id_field=self.__index_configs.get( # index_name).get_doc_id_field()) count = 0 for doc_id in doc_ids: count += self.__get_writer(index_name).delete_by_term( self.__index_configs.get( index_name).get_doc_id_field(), doc_id) self.__logger.info('{0} documents has deleted from {1}'.format( count, index_name)) except Exception as ex: self.__logger.error( 'failed to delete documents in bulk to {0}: {1}'.format( index_name, ex)) count = -1 finally: self.__record_metrics(start_time, 'delete_documents') return count def search_documents(self, index_name, query, search_field, page_num, page_len=10, weighting=None, **kwargs): start_time = time.time() try: searcher = self.__get_searcher(index_name, weighting=weighting) query_parser = QueryParser(search_field, self.get_schema(index_name)) query_obj = query_parser.parse(query) results_page = searcher.search_page(query_obj, page_num, pagelen=page_len, **kwargs) self.__logger.info('{0} documents ware searched from {1}'.format( results_page.total, index_name)) except Exception as ex: raise ex finally: self.__record_metrics(start_time, 'search_documents') return results_page @replicated def create_snapshot(self): self.__create_snapshot() def __create_snapshot(self): self.forceLogCompaction() def get_snapshot_file_name(self): return self.__conf.fullDumpFile def is_snapshot_exist(self): return os.path.exists(self.get_snapshot_file_name()) def open_snapshot_file(self): with self.__lock: try: file = open(self.get_snapshot_file_name(), mode='rb') except Exception as ex: raise ex return file
def test_unit_notappended(self): Histogram('h_seconds', 'help', [], registry=self.registry, unit="seconds") self.assertEqual(0, self.registry.get_sample_value('h_seconds_sum'))
def __init__(self, indexer, logger=getLogger(), http_logger=getLogger(), metrics_registry=CollectorRegistry()): self.__indexer = indexer self.__logger = logger self.__http_logger = http_logger self.__metrics_registry = metrics_registry # metrics self.__metrics_requests_total = Counter( '{0}_indexer_http_requests_total'.format(NAME), 'The number of requests.', [ 'method', 'endpoint', 'status_code' ], registry=self.__metrics_registry ) self.__metrics_requests_duration_seconds = Histogram( '{0}_indexer_http_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', [ 'method', 'endpoint' ], registry=self.__metrics_registry ) self.__metrics_requests_bytes_total = Counter( '{0}_indexer_http_requests_bytes_total'.format(NAME), 'A summary of the invocation requests bytes.', [ 'method', 'endpoint' ], registry=self.__metrics_registry ) self.__metrics_responses_bytes_total = Counter( '{0}_indexer_http_responses_bytes_total'.format(NAME), 'A summary of the invocation responses bytes.', [ 'method', 'endpoint' ], registry=self.__metrics_registry ) self.app = Flask('indexer_http') self.app.add_url_rule('/', endpoint='root', view_func=self.__root, methods=['GET']) self.app.add_url_rule('/indices/<index_name>', endpoint='get_index', view_func=self.__get_index, methods=['GET']) self.app.add_url_rule('/indices/<index_name>', endpoint='create_index', view_func=self.__create_index, methods=['PUT']) self.app.add_url_rule('/indices/<index_name>', endpoint='delete_index', view_func=self.__delete_index, methods=['DELETE']) self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='get_document', view_func=self.__get_document, methods=['GET']) self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='put_document', view_func=self.__put_document, methods=['PUT']) self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='delete_document', view_func=self.__delete_document, methods=['DELETE']) self.app.add_url_rule('/indices/<index_name>/documents', endpoint='put_documents', view_func=self.__put_documents, methods=['PUT']) self.app.add_url_rule('/indices/<index_name>/documents', endpoint='delete_documents', view_func=self.__delete_documents, methods=['DELETE']) self.app.add_url_rule('/indices/<index_name>/search', endpoint='search_documents', view_func=self.__search_documents, methods=['GET', 'POST']) self.app.add_url_rule('/indices/<index_name>/optimize', endpoint='optimize_index', view_func=self.__optimize_index, methods=['GET']) self.app.add_url_rule('/indices/<index_name>/commit', endpoint='commit', view_func=self.__commit_index, methods=['GET']) self.app.add_url_rule('/indices/<index_name>/rollback', endpoint='rollback', view_func=self.__rollback_index, methods=['GET']) self.app.add_url_rule('/nodes/<node_name>', endpoint='put_node', view_func=self.__put_node, methods=['PUT']) self.app.add_url_rule('/nodes/<node_name>', endpoint='delete_node', view_func=self.__delete_node, methods=['DELETE']) self.app.add_url_rule('/snapshot', endpoint='get_snapshot', view_func=self.__get_snapshot, methods=['GET']) self.app.add_url_rule('/snapshot', endpoint='put_snapshot', view_func=self.__put_snapshot, methods=['PUT']) self.app.add_url_rule('/metrics', endpoint='metrics', view_func=self.__metrics, methods=['GET']) self.app.add_url_rule('/healthiness', endpoint='healthiness', view_func=self.__healthiness, methods=['GET']) self.app.add_url_rule('/liveness', endpoint='liveness', view_func=self.__liveness, methods=['GET']) self.app.add_url_rule('/readiness', endpoint='readiness', view_func=self.__readiness, methods=['GET']) self.app.add_url_rule('/status', endpoint='status', view_func=self.__get_status, methods=['GET']) # disable Flask default logger self.app.logger.disabled = True getLogger('werkzeug').disabled = True