def test_collect(self): pid = 0 core._ValueClass = core._MultiProcessValue(lambda: pid) labels = dict((i, i) for i in 'abcd') def add_label(key, value): l = labels.copy() l[key] = value return l c = Counter('c', 'help', labelnames=labels.keys(), registry=None) g = Gauge('g', 'help', labelnames=labels.keys(), registry=None) h = Histogram('h', 'help', labelnames=labels.keys(), registry=None) c.labels(**labels).inc(1) g.labels(**labels).set(1) h.labels(**labels).observe(1) pid = 1 c.labels(**labels).inc(1) g.labels(**labels).set(1) h.labels(**labels).observe(5) metrics = dict((m.name, m) for m in self.collector.collect()) self.assertEqual( metrics['c'].samples, [Sample('c_total', labels, 2.0)] ) metrics['g'].samples.sort(key=lambda x: x[1]['pid']) self.assertEqual(metrics['g'].samples, [ Sample('g', add_label('pid', '0'), 1.0), Sample('g', add_label('pid', '1'), 1.0), ]) metrics['h'].samples.sort( key=lambda x: (x[0], float(x[1].get('le', 0))) ) expected_histogram = [ Sample('h_bucket', add_label('le', '0.005'), 0.0), Sample('h_bucket', add_label('le', '0.01'), 0.0), Sample('h_bucket', add_label('le', '0.025'), 0.0), Sample('h_bucket', add_label('le', '0.05'), 0.0), Sample('h_bucket', add_label('le', '0.075'), 0.0), Sample('h_bucket', add_label('le', '0.1'), 0.0), Sample('h_bucket', add_label('le', '0.25'), 0.0), Sample('h_bucket', add_label('le', '0.5'), 0.0), Sample('h_bucket', add_label('le', '0.75'), 0.0), Sample('h_bucket', add_label('le', '1.0'), 1.0), Sample('h_bucket', add_label('le', '2.5'), 1.0), Sample('h_bucket', add_label('le', '5.0'), 2.0), Sample('h_bucket', add_label('le', '7.5'), 2.0), Sample('h_bucket', add_label('le', '10.0'), 2.0), Sample('h_bucket', add_label('le', '+Inf'), 2.0), Sample('h_count', labels, 2.0), Sample('h_sum', labels, 6.0), ] self.assertEqual(metrics['h'].samples, expected_histogram)
def test_merge_no_accumulate(self): pid = 0 core._ValueClass = core._MultiProcessValue(lambda: pid) labels = dict((i, i) for i in 'abcd') def add_label(key, value): l = labels.copy() l[key] = value return l h = Histogram('h', 'help', labelnames=labels.keys(), registry=None) h.labels(**labels).observe(1) pid = 1 h.labels(**labels).observe(5) path = os.path.join(os.environ['prometheus_multiproc_dir'], '*.db') files = glob.glob(path) metrics = dict( (m.name, m) for m in self.collector.merge(files, accumulate=False) ) metrics['h'].samples.sort( key=lambda x: (x[0], float(x[1].get('le', 0))) ) expected_histogram = [ Sample('h_bucket', add_label('le', '0.005'), 0.0), Sample('h_bucket', add_label('le', '0.01'), 0.0), Sample('h_bucket', add_label('le', '0.025'), 0.0), Sample('h_bucket', add_label('le', '0.05'), 0.0), Sample('h_bucket', add_label('le', '0.075'), 0.0), Sample('h_bucket', add_label('le', '0.1'), 0.0), Sample('h_bucket', add_label('le', '0.25'), 0.0), Sample('h_bucket', add_label('le', '0.5'), 0.0), Sample('h_bucket', add_label('le', '0.75'), 0.0), Sample('h_bucket', add_label('le', '1.0'), 1.0), Sample('h_bucket', add_label('le', '2.5'), 0.0), Sample('h_bucket', add_label('le', '5.0'), 1.0), Sample('h_bucket', add_label('le', '7.5'), 0.0), Sample('h_bucket', add_label('le', '10.0'), 0.0), Sample('h_bucket', add_label('le', '+Inf'), 0.0), Sample('h_sum', labels, 6.0), ] self.assertEqual(metrics['h'].samples, expected_histogram)
def test_merge_no_accumulate(self): self.pid = 0 labels = dict((i, i) for i in 'abcd') def add_label(key, value): l = labels.copy() l[key] = value return l h = Histogram('hna', 'help', labelnames=labels.keys(), registry=None) h.labels(**labels).observe(1) self.pid = 1 h.labels(**labels).observe(5) self.collector.accumulate = False metrics = self.collector.collect() self.collector.accumulate = True metric = [x for x in metrics if x.name == 'hna'][0] metric.samples.sort( key=lambda x: (x[0], float(x[1].get('le', 0))) ) expected_histogram = [ Sample('hna_bucket', add_label('le', '0.005'), 0.0), Sample('hna_bucket', add_label('le', '0.01'), 0.0), Sample('hna_bucket', add_label('le', '0.025'), 0.0), Sample('hna_bucket', add_label('le', '0.05'), 0.0), Sample('hna_bucket', add_label('le', '0.075'), 0.0), Sample('hna_bucket', add_label('le', '0.1'), 0.0), Sample('hna_bucket', add_label('le', '0.25'), 0.0), Sample('hna_bucket', add_label('le', '0.5'), 0.0), Sample('hna_bucket', add_label('le', '0.75'), 0.0), Sample('hna_bucket', add_label('le', '1.0'), 1.0), Sample('hna_bucket', add_label('le', '2.5'), 0.0), Sample('hna_bucket', add_label('le', '5.0'), 1.0), Sample('hna_bucket', add_label('le', '7.5'), 0.0), Sample('hna_bucket', add_label('le', '10.0'), 0.0), Sample('hna_bucket', add_label('le', '+Inf'), 0.0), Sample('hna_sum', labels, 6.0), ] self.assertEqual(metric.samples, expected_histogram)
def test_reset_registry_with_labels(self): registry = CollectorRegistry() gauge = Gauge('g', 'help', ['l'], registry=registry) gauge.labels('a').inc() self.assertEqual(1, registry.get_sample_value('g', {'l': 'a'})) counter = Counter('c_total', 'help', ['l'], registry=registry) counter.labels('a').inc() self.assertEqual(1, registry.get_sample_value('c_total', {'l': 'a'})) summary = Summary('s', 'help', ['l'], registry=registry) summary.labels('a').observe(10) self.assertEqual(1, registry.get_sample_value('s_count', {'l': 'a'})) self.assertEqual(10, registry.get_sample_value('s_sum', {'l': 'a'})) histogram = Histogram('h', 'help', ['l'], registry=registry) histogram.labels('a').observe(2) self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '1.0', 'l': 'a'})) self.assertEqual(1, registry.get_sample_value('h_bucket', {'le': '2.5', 'l': 'a'})) self.assertEqual(1, registry.get_sample_value('h_bucket', {'le': '5.0', 'l': 'a'})) self.assertEqual(1, registry.get_sample_value('h_bucket', {'le': '+Inf', 'l': 'a'})) self.assertEqual(1, registry.get_sample_value('h_count', {'l': 'a'})) self.assertEqual(2, registry.get_sample_value('h_sum', {'l': 'a'})) registry.reset() self.assertEqual(0, registry.get_sample_value('g', {'l': 'a'})) self.assertEqual(0, registry.get_sample_value('c_total', {'l': 'a'})) self.assertEqual(0, registry.get_sample_value('s_count', {'l': 'a'})) self.assertEqual(0, registry.get_sample_value('s_sum', {'l': 'a'})) self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '1.0', 'l': 'a'})) self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '2.5', 'l': 'a'})) self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '5.0', 'l': 'a'})) self.assertEqual(0, registry.get_sample_value('h_bucket', {'le': '+Inf', 'l': 'a'})) self.assertEqual(0, registry.get_sample_value('h_count', {'l': 'a'})) self.assertEqual(0, registry.get_sample_value('h_sum', {'l': 'a'}))
class TestHistogram(unittest.TestCase): def setUp(self): self.registry = CollectorRegistry() self.histogram = Histogram('h', 'help', registry=self.registry) self.labels = Histogram('hl', 'help', ['l'], registry=self.registry) def test_histogram(self): self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual(0, self.registry.get_sample_value('h_sum')) self.histogram.observe(2) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual(2, self.registry.get_sample_value('h_sum')) self.histogram.observe(2.5) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(2, self.registry.get_sample_value('h_count')) self.assertEqual(4.5, self.registry.get_sample_value('h_sum')) self.histogram.observe(float("inf")) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 3, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(3, self.registry.get_sample_value('h_count')) self.assertEqual(float("inf"), self.registry.get_sample_value('h_sum')) def test_setting_buckets(self): h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2, float("inf")]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[float("inf")]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[3, 1]) def test_labels(self): self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, labelnames=['le']) self.labels.labels('a').observe(2) self.assertEqual( 0, self.registry.get_sample_value('hl_bucket', { 'le': '1.0', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_bucket', { 'le': '2.5', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_bucket', { 'le': '5.0', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_bucket', { 'le': '+Inf', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_count', {'l': 'a'})) self.assertEqual(2, self.registry.get_sample_value('hl_sum', {'l': 'a'})) def test_function_decorator(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) @self.histogram.time() def f(): pass self.assertEqual(([], None, None, None), inspect.getargspec(f)) f() self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) def test_function_decorator_multithread(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) workers = 3 duration = 0.1 pool = ThreadPoolExecutor(max_workers=workers) @self.histogram.time() def f(): time.sleep(duration) jobs = workers * 3 for i in range(jobs): pool.submit(f) pool.shutdown(wait=True) self.assertEqual(jobs, self.registry.get_sample_value('h_count')) rounding_coefficient = 0.9 total_expected_duration = jobs * duration * rounding_coefficient self.assertLess(total_expected_duration, self.registry.get_sample_value('h_sum')) def test_block_decorator(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) with self.histogram.time(): pass self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
class TestHistogram(unittest.TestCase): def setUp(self): self.registry = CollectorRegistry() self.histogram = Histogram('h', 'help', registry=self.registry) self.labels = Histogram('hl', 'help', ['l'], registry=self.registry) def test_histogram(self): self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual(0, self.registry.get_sample_value('h_sum')) self.histogram.observe(2) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual(2, self.registry.get_sample_value('h_sum')) self.histogram.observe(2.5) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(2, self.registry.get_sample_value('h_count')) self.assertEqual(4.5, self.registry.get_sample_value('h_sum')) self.histogram.observe(float("inf")) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual( 2, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual( 3, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(3, self.registry.get_sample_value('h_count')) self.assertEqual(float("inf"), self.registry.get_sample_value('h_sum')) def test_setting_buckets(self): h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2, float("inf")]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[float("inf")]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[3, 1]) def test_labels(self): self.labels.labels('a').observe(2) self.assertEqual( 0, self.registry.get_sample_value('hl_bucket', { 'le': '1.0', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_bucket', { 'le': '2.5', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_bucket', { 'le': '5.0', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_bucket', { 'le': '+Inf', 'l': 'a' })) self.assertEqual( 1, self.registry.get_sample_value('hl_count', {'l': 'a'})) self.assertEqual(2, self.registry.get_sample_value('hl_sum', {'l': 'a'})) def test_function_decorator(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) @self.histogram.time() def f(): pass self.assertEqual(([], None, None, None), inspect.getargspec(f)) f() self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) def test_block_decorator(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual( 0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) with self.histogram.time(): pass self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual( 1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
class TestHistogram(unittest.TestCase): def setUp(self): self.registry = CollectorRegistry() self.histogram = Histogram('h', 'help', registry=self.registry) self.labels = Histogram('hl', 'help', ['l'], registry=self.registry) def test_repr(self): self.assertEqual(repr(self.histogram), "prometheus_client.metrics.Histogram(h)") self.assertEqual(repr(self.labels), "prometheus_client.metrics.Histogram(hl)") def test_histogram(self): self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual(0, self.registry.get_sample_value('h_sum')) self.histogram.observe(2) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual(2, self.registry.get_sample_value('h_sum')) self.histogram.observe(2.5) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(2, self.registry.get_sample_value('h_count')) self.assertEqual(4.5, self.registry.get_sample_value('h_sum')) self.histogram.observe(float("inf")) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '1.0'})) self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '2.5'})) self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'le': '5.0'})) self.assertEqual(3, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) self.assertEqual(3, self.registry.get_sample_value('h_count')) self.assertEqual(float("inf"), self.registry.get_sample_value('h_sum')) def test_histogram_not_observable(self): """.observe() must fail if the Summary is not observable.""" assert_not_observable(self.labels.observe, 1) def test_setting_buckets(self): h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) h = Histogram('h', 'help', registry=None, buckets=[0, 1, 2, float("inf")]) self.assertEqual([0.0, 1.0, 2.0, float("inf")], h._upper_bounds) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[float("inf")]) self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, buckets=[3, 1]) def test_labels(self): self.assertRaises(ValueError, Histogram, 'h', 'help', registry=None, labelnames=['le']) self.labels.labels('a').observe(2) self.assertEqual(0, self.registry.get_sample_value('hl_bucket', {'le': '1.0', 'l': 'a'})) self.assertEqual(1, self.registry.get_sample_value('hl_bucket', {'le': '2.5', 'l': 'a'})) self.assertEqual(1, self.registry.get_sample_value('hl_bucket', {'le': '5.0', 'l': 'a'})) self.assertEqual(1, self.registry.get_sample_value('hl_bucket', {'le': '+Inf', 'l': 'a'})) self.assertEqual(1, self.registry.get_sample_value('hl_count', {'l': 'a'})) self.assertEqual(2, self.registry.get_sample_value('hl_sum', {'l': 'a'})) def test_function_decorator(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) @self.histogram.time() def f(): pass self.assertEqual(([], None, None, None), getargspec(f)) f() self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) def test_function_decorator_multithread(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) workers = 3 duration = 0.1 pool = ThreadPoolExecutor(max_workers=workers) @self.histogram.time() def f(): time.sleep(duration) jobs = workers * 3 for i in range(jobs): pool.submit(f) pool.shutdown(wait=True) self.assertEqual(jobs, self.registry.get_sample_value('h_count')) rounding_coefficient = 0.9 total_expected_duration = jobs * duration * rounding_coefficient self.assertLess(total_expected_duration, self.registry.get_sample_value('h_sum')) def test_block_decorator(self): self.assertEqual(0, self.registry.get_sample_value('h_count')) self.assertEqual(0, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) with self.histogram.time(): pass self.assertEqual(1, self.registry.get_sample_value('h_count')) self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'})) def test_exemplar_invalid_label_name(self): self.assertRaises(ValueError, self.histogram.observe, 3.0, exemplar={':o)': 'smile'}) self.assertRaises(ValueError, self.histogram.observe, 3.0, exemplar={'1': 'number'}) def test_exemplar_too_long(self): # 129 characters in total should fail. self.assertRaises(ValueError, self.histogram.observe, 1.0, exemplar={ 'abcdefghijklmnopqrstuvwxyz': '26+16 characters', 'x1234567': '8+15 characters', 'zyxwvutsrqponmlkjihgfedcba': '26+16 characters', 'y123456': '7+15 characters', })
class IndexHTTPServicer: def __init__(self, indexer, logger=getLogger(), http_logger=getLogger(), metrics_registry=CollectorRegistry()): self.__indexer = indexer self.__logger = logger self.__http_logger = http_logger self.__metrics_registry = metrics_registry # metrics self.__metrics_requests_total = Counter( '{0}_indexer_http_requests_total'.format(NAME), 'The number of requests.', [ 'method', 'endpoint', 'status_code' ], registry=self.__metrics_registry ) self.__metrics_requests_duration_seconds = Histogram( '{0}_indexer_http_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', [ 'method', 'endpoint' ], registry=self.__metrics_registry ) self.__metrics_requests_bytes_total = Counter( '{0}_indexer_http_requests_bytes_total'.format(NAME), 'A summary of the invocation requests bytes.', [ 'method', 'endpoint' ], registry=self.__metrics_registry ) self.__metrics_responses_bytes_total = Counter( '{0}_indexer_http_responses_bytes_total'.format(NAME), 'A summary of the invocation responses bytes.', [ 'method', 'endpoint' ], registry=self.__metrics_registry ) self.app = Flask('indexer_http') self.app.add_url_rule('/', endpoint='root', view_func=self.__root, methods=['GET']) self.app.add_url_rule('/indices/<index_name>', endpoint='get_index', view_func=self.__get_index, methods=['GET']) self.app.add_url_rule('/indices/<index_name>', endpoint='create_index', view_func=self.__create_index, methods=['PUT']) self.app.add_url_rule('/indices/<index_name>', endpoint='delete_index', view_func=self.__delete_index, methods=['DELETE']) self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='get_document', view_func=self.__get_document, methods=['GET']) self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='put_document', view_func=self.__put_document, methods=['PUT']) self.app.add_url_rule('/indices/<index_name>/documents/<doc_id>', endpoint='delete_document', view_func=self.__delete_document, methods=['DELETE']) self.app.add_url_rule('/indices/<index_name>/documents', endpoint='put_documents', view_func=self.__put_documents, methods=['PUT']) self.app.add_url_rule('/indices/<index_name>/documents', endpoint='delete_documents', view_func=self.__delete_documents, methods=['DELETE']) self.app.add_url_rule('/indices/<index_name>/search', endpoint='search_documents', view_func=self.__search_documents, methods=['GET', 'POST']) self.app.add_url_rule('/indices/<index_name>/optimize', endpoint='optimize_index', view_func=self.__optimize_index, methods=['GET']) self.app.add_url_rule('/indices/<index_name>/commit', endpoint='commit', view_func=self.__commit_index, methods=['GET']) self.app.add_url_rule('/indices/<index_name>/rollback', endpoint='rollback', view_func=self.__rollback_index, methods=['GET']) self.app.add_url_rule('/nodes/<node_name>', endpoint='put_node', view_func=self.__put_node, methods=['PUT']) self.app.add_url_rule('/nodes/<node_name>', endpoint='delete_node', view_func=self.__delete_node, methods=['DELETE']) self.app.add_url_rule('/snapshot', endpoint='get_snapshot', view_func=self.__get_snapshot, methods=['GET']) self.app.add_url_rule('/snapshot', endpoint='put_snapshot', view_func=self.__put_snapshot, methods=['PUT']) self.app.add_url_rule('/metrics', endpoint='metrics', view_func=self.__metrics, methods=['GET']) self.app.add_url_rule('/healthiness', endpoint='healthiness', view_func=self.__healthiness, methods=['GET']) self.app.add_url_rule('/liveness', endpoint='liveness', view_func=self.__liveness, methods=['GET']) self.app.add_url_rule('/readiness', endpoint='readiness', view_func=self.__readiness, methods=['GET']) self.app.add_url_rule('/status', endpoint='status', view_func=self.__get_status, methods=['GET']) # disable Flask default logger self.app.logger.disabled = True getLogger('werkzeug').disabled = True def __record_metrics(self, start_time, req, resp): self.__metrics_requests_total.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else ''), status_code=resp.status_code.value ).inc() self.__metrics_requests_bytes_total.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '') ).inc(req.content_length if req.content_length is not None else 0) self.__metrics_responses_bytes_total.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '') ).inc(resp.content_length if resp.content_length is not None else 0) self.__metrics_requests_duration_seconds.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '') ).observe(time.time() - start_time) return def __root(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response resp = Response() try: resp.status_code = HTTPStatus.OK resp.content_type = 'text/plain; charset="UTF-8"' resp.data = NAME + ' ' + VERSION + ' is running.\n' except Exception as ex: resp.status_code = HTTPStatus.INTERNAL_SERVER_ERROR resp.content_type = 'text/plain; charset="UTF-8"' resp.data = '{0}\n{1}'.format(resp.status_code.phrase, resp.status_code.description) self.__logger.error(ex) return resp def __create_index(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: mime = mimeparse.parse_mime_type(request.headers.get('Content-Type')) charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset') if mime[1] == 'yaml': index_config_dict = yaml.safe_load(request.data.decode(charset)) elif mime[1] == 'json': index_config_dict = json.loads(request.data.decode(charset)) else: raise ValueError('unsupported format') if index_config_dict is None: raise ValueError('index config is None') sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True index_config = IndexConfig(index_config_dict) self.__indexer.create_index(index_name, index_config, sync=sync) if sync: status_code = HTTPStatus.CREATED else: status_code = HTTPStatus.ACCEPTED except (ConstructorError, JSONDecodeError, ValueError) as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.BAD_REQUEST self.__logger.error(ex) except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __get_index(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: index = self.__indexer.get_index(index_name) if index is None: status_code = HTTPStatus.NOT_FOUND else: data['index'] = { 'name': index.indexname, 'doc_count': index.doc_count(), 'doc_count_all': index.doc_count_all(), 'last_modified': index.latest_generation(), 'latest_generation': index.last_modified(), 'version': index.version, 'storage': { 'folder': index.storage.folder, 'supports_mmap': index.storage.supports_mmap, 'readonly': index.storage.readonly, 'files': list(index.storage.list()) } } status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __delete_index(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True self.__indexer.delete_index(index_name, sync=sync) if sync: status_code = HTTPStatus.OK else: status_code = HTTPStatus.ACCEPTED except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __commit_index(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True self.__indexer.commit_index(index_name, sync=sync) if sync: status_code = HTTPStatus.OK else: status_code = HTTPStatus.ACCEPTED except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __rollback_index(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True self.__indexer.rollback_index(index_name, sync=sync) if sync: status_code = HTTPStatus.OK else: status_code = HTTPStatus.ACCEPTED except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __optimize_index(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True self.__indexer.optimize_index(index_name, sync=sync) if sync: status_code = HTTPStatus.OK else: status_code = HTTPStatus.ACCEPTED except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __put_document(self, index_name, doc_id): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: mime = mimeparse.parse_mime_type(request.headers.get('Content-Type')) charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset') if mime[1] == 'yaml': fields_dict = yaml.safe_load(request.data.decode(charset)) elif mime[1] == 'json': fields_dict = json.loads(request.data.decode(charset)) else: raise ValueError('unsupported format') if fields_dict is None: raise ValueError('fields are None') sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True count = self.__indexer.put_document(index_name, doc_id, fields_dict, sync=sync) if sync: if count > 0: data['count'] = count status_code = HTTPStatus.CREATED else: status_code = HTTPStatus.INTERNAL_SERVER_ERROR else: status_code = HTTPStatus.ACCEPTED except (ConstructorError, JSONDecodeError, ValueError) as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.BAD_REQUEST self.__logger.error(ex) except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __get_document(self, index_name, doc_id): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: results_page = self.__indexer.get_document(index_name, doc_id) if results_page.total > 0: fields = {} for i in results_page.results[0].iteritems(): fields[i[0]] = i[1] data['fields'] = fields status_code = HTTPStatus.OK else: status_code = HTTPStatus.NOT_FOUND except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __delete_document(self, index_name, doc_id): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True count = self.__indexer.delete_document(index_name, doc_id, sync=sync) if sync: if count > 0: status_code = HTTPStatus.OK elif count == 0: status_code = HTTPStatus.NOT_FOUND else: status_code = HTTPStatus.INTERNAL_SERVER_ERROR else: status_code = HTTPStatus.ACCEPTED except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __put_documents(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: mime = mimeparse.parse_mime_type(request.headers.get('Content-Type')) charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset') if mime[1] == 'yaml': docs_dict = yaml.safe_load(request.data.decode(charset)) elif mime[1] == 'json': docs_dict = json.loads(request.data.decode(charset)) else: raise ValueError('unsupported format') sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True count = self.__indexer.put_documents(index_name, docs_dict, sync=sync) if sync: if count > 0: data['count'] = count status_code = HTTPStatus.CREATED else: status_code = HTTPStatus.INTERNAL_SERVER_ERROR else: status_code = HTTPStatus.ACCEPTED except (ConstructorError, JSONDecodeError, ValueError) as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.BAD_REQUEST self.__logger.error(ex) except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __delete_documents(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: mime = mimeparse.parse_mime_type(request.headers.get('Content-Type')) charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset') if mime[1] == 'yaml': doc_ids_list = yaml.safe_load(request.data.decode(charset)) elif mime[1] == 'json': doc_ids_list = json.loads(request.data.decode(charset)) else: raise ValueError('unsupported format') sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True count = self.__indexer.delete_documents(index_name, doc_ids_list, sync=sync) if sync: if count > 0: data['count'] = count status_code = HTTPStatus.OK else: status_code = HTTPStatus.INTERNAL_SERVER_ERROR else: status_code = HTTPStatus.ACCEPTED except (ConstructorError, JSONDecodeError, ValueError) as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.BAD_REQUEST self.__logger.error(ex) except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __search_documents(self, index_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, resp) return response data = {} status_code = None try: query = request.args.get('query', default='', type=str) search_field = request.args.get('search_field', default='', type=str) page_num = request.args.get('page_num', default=1, type=int) page_len = request.args.get('page_len', default=10, type=int) weighting = BM25F if len(request.data) > 0: mime = mimeparse.parse_mime_type(request.headers.get('Content-Type')) charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset') if mime[1] == 'yaml': weighting = get_multi_weighting(yaml.safe_load(request.data.decode(charset))) elif mime[1] == 'json': weighting = get_multi_weighting(json.loads(request.data.decode(charset))) else: raise ValueError('unsupported format') results_page = self.__indexer.search_documents(index_name, query, search_field, page_num, page_len=page_len, weighting=weighting) if results_page.pagecount >= page_num or results_page.total <= 0: results = { 'is_last_page': results_page.is_last_page(), 'page_count': results_page.pagecount, 'page_len': results_page.pagelen, 'page_num': results_page.pagenum, 'total': results_page.total, 'offset': results_page.offset } hits = [] for result in results_page.results[results_page.offset:]: fields = {} for item in result.iteritems(): fields[item[0]] = item[1] hit = { 'fields': fields, 'doc_num': result.docnum, 'score': result.score, 'rank': result.rank, 'pos': result.pos } hits.append(hit) results['hits'] = hits data['results'] = results status_code = HTTPStatus.OK else: data['error'] = 'page_num must be <= {0}'.format(results_page.pagecount) status_code = HTTPStatus.BAD_REQUEST except (ConstructorError, JSONDecodeError, ValueError) as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.BAD_REQUEST self.__logger.error(ex) except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __put_node(self, node_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, resp) return response data = {} status_code = None try: self.__indexer.addNodeToCluster(node_name) status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __get_status(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, resp) return response data = {} status_code = None try: data['node_status'] = self.__indexer.getStatus() status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __delete_node(self, node_name): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, resp) return response data = {} status_code = None try: self.__indexer.removeNodeFromCluster(node_name) status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __put_snapshot(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, resp) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True self.__indexer.create_snapshot(sync=sync) status_code = HTTPStatus.ACCEPTED except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __get_snapshot(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response try: if self.__indexer.is_snapshot_exist(): def generate(): with self.__indexer.open_snapshot_file() as f: chunk = f.read(1024) yield chunk resp = Response(generate(), status=HTTPStatus.OK, mimetype='application/zip', headers={ 'Content-Disposition': 'attachment; filename=snapshot.zip' }) else: resp = Response(status=HTTPStatus.NOT_FOUND) except Exception as ex: resp = Response(status=HTTPStatus.INTERNAL_SERVER_ERROR) self.__logger.error(ex) return resp def __healthiness(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: healthy = self.__indexer.is_healthy() data['healthy'] = healthy if healthy: status_code = HTTPStatus.OK else: status_code = HTTPStatus.SERVICE_UNAVAILABLE data['error'] = 'node is not healthy' except Exception as ex: data['healthy'] = False data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __liveness(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: alive = self.__indexer.is_alive() data['liveness'] = alive if alive: status_code = HTTPStatus.OK else: status_code = HTTPStatus.SERVICE_UNAVAILABLE data['error'] = 'node is not alive' except Exception as ex: data['liveness'] = False data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __readiness(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: ready = self.__indexer.is_ready() data['readiness'] = ready if ready: status_code = HTTPStatus.OK else: status_code = HTTPStatus.SERVICE_UNAVAILABLE data['error'] = 'node is not ready' except Exception as ex: data['readiness'] = False data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = {'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description} output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __metrics(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response resp = Response() try: resp.status_code = HTTPStatus.OK resp.content_type = CONTENT_TYPE_LATEST resp.data = generate_latest(self.__metrics_registry) except Exception as ex: resp.status_code = HTTPStatus.INTERNAL_SERVER_ERROR resp.content_type = 'text/plain; charset="UTF-8"' resp.data = '{0}\n{1}'.format(resp.status_code.phrase, resp.status_code.description) self.__logger.error(ex) return resp
class Manager(RaftNode): def __init__(self, host='localhost', port=7070, seed_addr=None, conf=SyncObjConf(), data_dir='/tmp/cockatrice/management', grpc_port=5050, grpc_max_workers=10, http_port=8080, logger=getLogger(), http_logger=getLogger(), metrics_registry=CollectorRegistry()): self.__host = host self.__port = port self.__seed_addr = seed_addr self.__conf = conf self.__data_dir = data_dir self.__grpc_port = grpc_port self.__grpc_max_workers = grpc_max_workers self.__http_port = http_port self.__logger = logger self.__http_logger = http_logger self.__metrics_registry = metrics_registry # metrics self.__metrics_requests_total = Counter( '{0}_manager_requests_total'.format(NAME), 'The number of requests.', ['func'], registry=self.__metrics_registry) self.__metrics_requests_duration_seconds = Histogram( '{0}_manager_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', ['func'], registry=self.__metrics_registry) self.__self_addr = '{0}:{1}'.format(self.__host, self.__port) self.__peer_addrs = [] if self.__seed_addr is None else get_peers( self.__seed_addr) self.__other_addrs = [ peer_addr for peer_addr in self.__peer_addrs if peer_addr != self.__self_addr ] self.__conf.serializer = self.__serialize self.__conf.deserializer = self.__deserialize self.__conf.validate() self.__data = LockedDict() self.__lock = RLock() # add this node to the cluster if self.__self_addr not in self.__peer_addrs and self.__seed_addr is not None: Thread(target=add_node, kwargs={ 'node_name': self.__self_addr, 'bind_addr': self.__seed_addr, 'timeout': 0.5 }).start() # create data dir os.makedirs(self.__data_dir, exist_ok=True) # copy snapshot from the leader node if self.__seed_addr is not None: try: leader = get_status(bind_addr=self.__seed_addr, timeout=0.5)['leader'] self.__logger.info('copying snapshot from {0}'.format(leader)) snapshot = get_snapshot(bind_addr=leader, timeout=0.5) if snapshot is not None: with open(self.__conf.fullDumpFile, 'wb') as f: f.write(snapshot) self.__logger.info( 'snapshot copied from {0}'.format(leader)) except Exception as ex: self.__logger.error( 'failed to copy snapshot from {0}: {1}'.format(leader, ex)) # start node super(Manager, self).__init__(self.__self_addr, self.__other_addrs, conf=self.__conf) self.__logger.info('state machine has started') while not self.isReady(): # recovering data self.__logger.debug('waiting for cluster ready') time.sleep(1) self.__logger.info('cluster ready') # start gRPC self.__grpc_server = grpc.server( futures.ThreadPoolExecutor(max_workers=self.__grpc_max_workers)) add_ManagementServicer_to_server( ManagementGRPCServicer(self, logger=self.__logger, metrics_registry=self.__metrics_registry), self.__grpc_server) self.__grpc_server.add_insecure_port('{0}:{1}'.format( self.__host, self.__grpc_port)) self.__grpc_server.start() self.__logger.info('gRPC server has started') # start HTTP server self.__http_servicer = ManagementHTTPServicer(self, self.__logger, self.__http_logger, self.__metrics_registry) self.__http_server = HTTPServer(self.__host, self.__http_port, self.__http_servicer) self.__http_server.start() self.__logger.info('HTTP server has started') self.__logger.info('manager has started') def stop(self): # stop HTTP server self.__http_server.stop() self.__logger.info('HTTP server has stopped') # stop gRPC server self.__grpc_server.stop(grace=0.0) self.__logger.info('gRPC server has stopped') # stop node self.destroy() self.__logger.info('state machine has stopped') self.__logger.info('manager has stopped') def __record_metrics(self, start_time, func_name): self.__metrics_requests_total.labels(func=func_name).inc() self.__metrics_requests_duration_seconds.labels( func=func_name).observe(time.time() - start_time) # serializer def __serialize(self, filename, raft_data): with self.__lock: try: self.__logger.info('serializer has started') with zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED) as f: # store the federation data f.writestr('federation.bin', pickle.dumps(self.__data)) self.__logger.debug( 'federation data has stored in {0}'.format(filename)) # store the raft data f.writestr(RAFT_DATA_FILE, pickle.dumps(raft_data)) self.__logger.info( '{0} has restored'.format(RAFT_DATA_FILE)) self.__logger.info('snapshot has created') except Exception as ex: self.__logger.error( 'failed to create snapshot: {0}'.format(ex)) finally: self.__logger.info('serializer has stopped') # deserializer def __deserialize(self, filename): raft_data = None with self.__lock: try: self.__logger.info('deserializer has started') with zipfile.ZipFile(filename, 'r') as zf: # extract the federation data zf.extract('federation.bin', path=self.__data_dir) self.__data = pickle.loads(zf.read('federation.bin')) self.__logger.info('federation.bin has restored') # restore the raft data raft_data = pickle.loads(zf.read(RAFT_DATA_FILE)) self.__logger.info( 'raft.{0} has restored'.format(RAFT_DATA_FILE)) self.__logger.info('snapshot has restored') except Exception as ex: self.__logger.error( 'failed to restore indices: {0}'.format(ex)) finally: self.__logger.info('deserializer has stopped') return raft_data def is_healthy(self): return self.isHealthy() def is_alive(self): return self.isAlive() def is_ready(self): return self.isReady() def __key_value_to_dict(self, key, value): keys = [k for k in key.split('/') if k != ''] if len(keys) > 1: value = self.__key_value_to_dict('/'.join(keys[1:]), value) return {keys[0]: value} def __put(self, key, value): start_time = time.time() try: if key == '/': self.__data.update(value) else: self.__data.update(self.__key_value_to_dict(key, value)) finally: self.__record_metrics(start_time, 'put') @replicated def put(self, key, value): self.__put(key, value) def get(self, key): start_time = time.time() try: value = self.__data keys = [k for k in key.split('/') if k != ''] for k in keys: value = value.get(k, None) if value is None: return None finally: self.__record_metrics(start_time, 'get') return value def __delete(self, key): start_time = time.time() try: if key == '/': value = dict(self.__data) self.__clear() else: keys = [k for k in key.split('/') if k != ''] value = self.__data i = 0 while i < len(keys): if len(keys[i:]) == 1: return value.pop(keys[i], None) value = value.get(keys[i], None) if value is None: return None i += 1 finally: self.__record_metrics(start_time, 'delete') return value @replicated def delete(self, key): return self.__delete(key) def __clear(self): start_time = time.time() try: self.__data.clear() finally: self.__record_metrics(start_time, 'clear') @replicated def clear(self): self.__clear()
class Indexer(RaftNode): def __init__(self, host='localhost', port=7070, seed_addr=None, conf=SyncObjConf(), data_dir='/tmp/cockatrice/index', grpc_port=5050, grpc_max_workers=10, http_port=8080, logger=getLogger(), http_logger=getLogger(), metrics_registry=CollectorRegistry()): self.__host = host self.__port = port self.__seed_addr = seed_addr self.__conf = conf self.__data_dir = data_dir self.__grpc_port = grpc_port self.__grpc_max_workers = grpc_max_workers self.__http_port = http_port self.__logger = logger self.__http_logger = http_logger self.__metrics_registry = metrics_registry # metrics self.__metrics_core_documents = Gauge( '{0}_indexer_index_documents'.format(NAME), 'The number of documents.', [ 'index_name', ], registry=self.__metrics_registry) self.__metrics_requests_total = Counter( '{0}_indexer_requests_total'.format(NAME), 'The number of requests.', ['func'], registry=self.__metrics_registry) self.__metrics_requests_duration_seconds = Histogram( '{0}_indexer_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', ['func'], registry=self.__metrics_registry) self.__self_addr = '{0}:{1}'.format(self.__host, self.__port) self.__peer_addrs = [] if self.__seed_addr is None else get_peers( bind_addr=self.__seed_addr, timeout=10) self.__other_addrs = [ peer_addr for peer_addr in self.__peer_addrs if peer_addr != self.__self_addr ] self.__conf.serializer = self.__serialize self.__conf.deserializer = self.__deserialize self.__conf.validate() self.__indices = {} self.__index_configs = {} self.__writers = {} self.__auto_commit_timers = {} self.__lock = RLock() # create data dir os.makedirs(self.__data_dir, exist_ok=True) self.__file_storage = FileStorage(self.__data_dir, supports_mmap=True, readonly=False, debug=False) self.__ram_storage = RamStorage() # if seed addr specified and self node does not exist in the cluster, add self node to the cluster if self.__seed_addr is not None and self.__self_addr not in self.__peer_addrs: Thread(target=add_node, kwargs={ 'node_name': self.__self_addr, 'bind_addr': self.__seed_addr, 'timeout': 10 }).start() # copy snapshot from the leader node if self.__seed_addr is not None: try: metadata = get_metadata(bind_addr=get_leader( bind_addr=self.__seed_addr, timeout=10), timeout=10) response = requests.get('http://{0}/snapshot'.format( metadata['http_addr'])) if response.status_code == HTTPStatus.OK: with open(self.__conf.fullDumpFile, 'wb') as f: f.write(response.content) except Exception as ex: self.__logger.error('failed to copy snapshot: {0}'.format(ex)) # start node metadata = { 'grpc_addr': '{0}:{1}'.format(self.__host, self.__grpc_port), 'http_addr': '{0}:{1}'.format(self.__host, self.__http_port) } self.__logger.info('starting raft state machine') super(Indexer, self).__init__(self.__self_addr, self.__peer_addrs, conf=self.__conf, metadata=metadata) self.__logger.info('raft state machine has started') if os.path.exists(self.__conf.fullDumpFile): self.__logger.debug('snapshot exists: {0}'.format( self.__conf.fullDumpFile)) else: pass while not self.isReady(): # recovering data self.__logger.debug('waiting for cluster ready') self.__logger.debug(self.getStatus()) time.sleep(1) self.__logger.info('cluster ready') self.__logger.debug(self.getStatus()) # open existing indices on startup for index_name in self.get_index_names(): self.__open_index(index_name, index_config=None) # record index metrics timer self.metrics_timer = Timer(10, self.__record_index_metrics) self.metrics_timer.start() # start gRPC self.__grpc_server = grpc.server( futures.ThreadPoolExecutor(max_workers=self.__grpc_max_workers)) add_IndexServicer_to_server( IndexGRPCServicer(self, logger=self.__logger, metrics_registry=self.__metrics_registry), self.__grpc_server) self.__grpc_server.add_insecure_port('{0}:{1}'.format( self.__host, self.__grpc_port)) self.__grpc_server.start() self.__logger.info('gRPC server has started') # start HTTP server self.__http_servicer = IndexHTTPServicer(self, self.__logger, self.__http_logger, self.__metrics_registry) self.__http_server = HTTPServer(self.__host, self.__http_port, self.__http_servicer) self.__http_server.start() self.__logger.info('HTTP server has started') self.__logger.info('indexer has started') def stop(self): # stop HTTP server self.__http_server.stop() self.__logger.info('HTTP server has stopped') # stop gRPC server self.__grpc_server.stop(grace=0.0) self.__logger.info('gRPC server has stopped') self.metrics_timer.cancel() # close indices for index_name in list(self.__indices.keys()): self.__close_index(index_name) self.destroy() self.__logger.info('index core has stopped') def __record_index_metrics(self): for index_name in list(self.__indices.keys()): try: self.__metrics_core_documents.labels( index_name=index_name).set(self.get_doc_count(index_name)) except Exception as ex: self.__logger.error(ex) def __record_metrics(self, start_time, func_name): self.__metrics_requests_total.labels(func=func_name).inc() self.__metrics_requests_duration_seconds.labels( func=func_name).observe(time.time() - start_time) # def __serialize_indices(self, filename): # with self.__lock: # try: # self.__logger.info('starting serialize indices') # # except Exception as ex: # self.__logger.error('failed to create snapshot: {0}'.format(ex)) # finally: # self.__logger.info('serialize indices has finished') # def __serialize_raft_data(self, filename, raft_data): # with self.__lock: # pass # index serializer def __serialize(self, filename, raft_data): with self.__lock: try: self.__logger.debug('serializer has started') # store the index files and raft logs to the snapshot file with zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED) as f: for index_name in self.get_index_names(): self.__commit_index(index_name) # with self.__get_writer(index_name).writelock: # with self.__indices[index_name].lock('WRITELOCK'): # index files for index_filename in self.get_index_files(index_name): if self.__index_configs.get( index_name).get_storage_type() == "ram": with self.__ram_storage.open_file( index_filename) as r: f.writestr(index_filename, r.read()) else: f.write( os.path.join(self.__file_storage.folder, index_filename), index_filename) self.__logger.debug('{0} has stored in {1}'.format( index_filename, filename)) # index config file f.write( os.path.join( self.__file_storage.folder, self.get_index_config_file(index_name)), self.get_index_config_file(index_name)) self.__logger.debug('{0} has stored in {1}'.format( self.get_index_config_file(index_name), filename)) # store the raft data f.writestr(RAFT_DATA_FILE, pickle.dumps(raft_data)) self.__logger.debug( '{0} has restored'.format(RAFT_DATA_FILE)) self.__logger.debug('snapshot has created') except Exception as ex: self.__logger.error( 'failed to create snapshot: {0}'.format(ex)) finally: self.__logger.debug('serializer has stopped') # index deserializer def __deserialize(self, filename): with self.__lock: try: self.__logger.debug('deserializer has started') with zipfile.ZipFile(filename, 'r') as zf: # get file names in snapshot file filenames = list(zf.namelist()) # get index names in snapshot file index_names = [] pattern_toc = re.compile(r'^_(.+)_\d+\.toc$') for f in filenames: match = pattern_toc.search(f) if match and match.group(1) not in index_names: index_names.append(match.group(1)) for index_name in index_names: # extract the index config first zf.extract(self.get_index_config_file(index_name), path=self.__file_storage.folder) index_config = pickle.loads( zf.read(self.get_index_config_file(index_name))) # get index files pattern_toc = re.compile(r'^_{0}_(\d+)\..+$'.format( index_name)) # ex) _myindex_0.toc pattern_seg = re.compile( r'^{0}_([a-z0-9]+)\..+$'.format(index_name) ) # ex) myindex_zseabukc2nbpvh0u.seg pattern_lock = re.compile(r'^{0}_WRITELOCK$'.format( index_name)) # ex) myindex_WRITELOCK index_files = [] for file_name in filenames: if re.match(pattern_toc, file_name): index_files.append(file_name) elif re.match(pattern_seg, file_name): index_files.append(file_name) elif re.match(pattern_lock, file_name): index_files.append(file_name) # extract the index files for index_file in index_files: if index_config.get_storage_type() == 'ram': with self.__ram_storage.create_file( index_file) as r: r.write(zf.read(index_file)) else: zf.extract(index_file, path=self.__file_storage.folder) self.__logger.debug( '{0} has restored from {1}'.format( index_file, filename)) self.__logger.debug( '{0} has restored'.format(index_name)) # extract the raft data raft_data = pickle.loads(zf.read(RAFT_DATA_FILE)) self.__logger.debug( '{0} has restored'.format(RAFT_DATA_FILE)) return raft_data except Exception as ex: self.__logger.error( 'failed to restore indices: {0}'.format(ex)) finally: self.__logger.debug('deserializer has stopped') def is_healthy(self): return self.isHealthy() def is_alive(self): return self.isAlive() def is_ready(self): return self.isReady() def get_addr(self): return self.__self_addr def get_index_files(self, index_name): index_files = [] pattern_toc = re.compile( r'^_{0}_(\d+)\..+$'.format(index_name)) # ex) _myindex_0.toc pattern_seg = re.compile(r'^{0}_([a-z0-9]+)\..+$'.format( index_name)) # ex) myindex_zseabukc2nbpvh0u.seg pattern_lock = re.compile( r'^{0}_WRITELOCK$'.format(index_name)) # ex) myindex_WRITELOCK if self.__index_configs.get(index_name).get_storage_type() == "ram": storage = self.__ram_storage else: storage = self.__file_storage for file_name in list(storage.list()): if re.match(pattern_toc, file_name): index_files.append(file_name) elif re.match(pattern_seg, file_name): index_files.append(file_name) elif re.match(pattern_lock, file_name): index_files.append(file_name) return index_files @staticmethod def get_index_config_file(index_name): return '{0}_CONFIG'.format(index_name) def get_index_names(self): index_names = [] pattern_toc = re.compile(r'^_(.+)_\d+\.toc$') for filename in list(self.__file_storage.list()): match = pattern_toc.search(filename) if match and match.group(1) not in index_names: index_names.append(match.group(1)) for filename in list(self.__ram_storage.list()): match = pattern_toc.search(filename) if match and match.group(1) not in index_names: index_names.append(match.group(1)) return index_names def is_index_exist(self, index_name): return self.__file_storage.index_exists( indexname=index_name) or self.__ram_storage.index_exists( indexname=index_name) def is_index_open(self, index_name): return index_name in self.__indices @replicated def open_index(self, index_name, index_config=None): return self.__open_index(index_name, index_config=index_config) def __open_index(self, index_name, index_config=None): start_time = time.time() index = None try: # open the index index = self.__indices.get(index_name) if index is None: self.__logger.debug('opening {0}'.format(index_name)) if index_config is None: # set saved index config with open( os.path.join( self.__file_storage.folder, self.get_index_config_file(index_name)), 'rb') as f: self.__index_configs[index_name] = pickle.loads( f.read()) else: # set given index config self.__index_configs[index_name] = index_config if self.__index_configs[index_name].get_storage_type( ) == 'ram': index = self.__ram_storage.open_index( indexname=index_name, schema=self.__index_configs[index_name].get_schema()) else: index = self.__file_storage.open_index( indexname=index_name, schema=self.__index_configs[index_name].get_schema()) self.__indices[index_name] = index self.__logger.info('{0} has opened'.format(index_name)) # open the index writer self.__open_writer(index_name) except Exception as ex: self.__logger.error('failed to open {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'open_index') return index @replicated def close_index(self, index_name): return self.__close_index(index_name) def __close_index(self, index_name): start_time = time.time() index = None try: # close the index writer self.__close_writer(index_name) # close the index index = self.__indices.pop(index_name) if index is not None: self.__logger.debug('closing {0}'.format(index_name)) index.close() self.__logger.info('{0} has closed'.format(index_name)) except Exception as ex: self.__logger.error('failed to close {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'close_index') return index @replicated def create_index(self, index_name, index_config): return self.__create_index(index_name, index_config) def __create_index(self, index_name, index_config): if self.is_index_exist(index_name): # open the index return self.__open_index(index_name, index_config=index_config) start_time = time.time() index = None with self.__lock: try: self.__logger.debug('creating {0}'.format(index_name)) # set index config self.__index_configs[index_name] = index_config self.__logger.debug( self.__index_configs[index_name].get_storage_type()) # create the index if self.__index_configs[index_name].get_storage_type( ) == 'ram': index = self.__ram_storage.create_index( self.__index_configs[index_name].get_schema(), indexname=index_name) else: index = self.__file_storage.create_index( self.__index_configs[index_name].get_schema(), indexname=index_name) self.__indices[index_name] = index self.__logger.info('{0} has created'.format(index_name)) # save the index config with open( os.path.join(self.__file_storage.folder, self.get_index_config_file(index_name)), 'wb') as f: f.write(pickle.dumps(index_config)) # open the index writer self.__open_writer(index_name) except Exception as ex: self.__logger.error('failed to create {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'create_index') return index @replicated def delete_index(self, index_name): return self.__delete_index(index_name) def __delete_index(self, index_name): # close index index = self.__close_index(index_name) start_time = time.time() with self.__lock: try: self.__logger.debug('deleting {0}'.format(index_name)) # delete index files for filename in self.get_index_files(index_name): self.__file_storage.delete_file(filename) self.__logger.debug('{0} was deleted'.format(filename)) self.__logger.info('{0} has deleted'.format(index_name)) # delete the index config self.__index_configs.pop(index_name, None) os.remove( os.path.join(self.__file_storage.folder, self.get_index_config_file(index_name))) except Exception as ex: self.__logger.error('failed to delete {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'delete_index') return index def get_index(self, index_name): return self.__get_index(index_name) def __get_index(self, index_name): start_time = time.time() try: index = self.__indices.get(index_name) except Exception as ex: raise ex finally: self.__record_metrics(start_time, 'get_index') return index def __start_auto_commit_timer(self, index_name, period): timer = self.__auto_commit_timers.get(index_name, None) if timer is None: self.__auto_commit_timers[index_name] = threading.Timer( period, self.__auto_commit_index, kwargs={ 'index_name': index_name, 'period': period }) self.__auto_commit_timers[index_name].start() self.__logger.debug( 'auto commit timer for {0} were started'.format(index_name)) def __stop_auto_commit_timer(self, index_name): timer = self.__auto_commit_timers.pop(index_name, None) if timer is not None: timer.cancel() self.__logger.debug( 'auto commit timer for {0} were stopped'.format(index_name)) def __auto_commit_index(self, index_name, period): self.__stop_auto_commit_timer(index_name) self.__commit_index(index_name) self.__start_auto_commit_timer(index_name, period=period) def __open_writer(self, index_name): writer = None try: writer = self.__writers.get(index_name, None) if writer is None or writer.is_closed: self.__logger.debug( 'opening writer for {0}'.format(index_name)) writer = self.__indices.get(index_name).writer() self.__writers[index_name] = writer self.__logger.debug( 'writer for {0} has opened'.format(index_name)) self.__start_auto_commit_timer( index_name, period=self.__index_configs.get( index_name).get_writer_auto_commit_period()) except Exception as ex: self.__logger.error('failed to open writer for {0}: {1}'.format( index_name, ex)) return writer def __close_writer(self, index_name): writer = None try: self.__stop_auto_commit_timer(index_name) # close the index writer = self.__writers.pop(index_name, None) if writer is not None: self.__logger.debug( 'closing writer for {0}'.format(index_name)) writer.commit() self.__logger.debug( 'writer for {0} has closed'.format(index_name)) except Exception as ex: self.__logger.error('failed to close writer for {0}: {1}'.format( index_name, ex)) return writer def __get_writer(self, index_name): return self.__writers.get(index_name, None) def __get_searcher(self, index_name, weighting=None): try: if weighting is None: searcher = self.__indices.get(index_name).searcher() else: searcher = self.__indices.get(index_name).searcher( weighting=weighting) except Exception as ex: raise ex return searcher @replicated def commit_index(self, index_name): return self.__commit_index(index_name) def __commit_index(self, index_name): start_time = time.time() success = False with self.__lock: try: self.__logger.debug('committing {0}'.format(index_name)) self.__get_writer(index_name).commit() self.__open_writer(index_name) # reopen writer self.__logger.info('{0} has committed'.format(index_name)) success = True except Exception as ex: self.__logger.error('failed to commit index {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'commit_index') return success @replicated def rollback_index(self, index_name): return self.__rollback_index(index_name) def __rollback_index(self, index_name): start_time = time.time() success = False with self.__lock: try: self.__logger.debug('rolling back {0}'.format(index_name)) self.__get_writer(index_name).cancel() self.__open_writer(index_name) # reopen writer self.__logger.info('{0} has rolled back'.format(index_name)) success = True except Exception as ex: self.__logger.error('failed to rollback index {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'rollback_index') return success @replicated def optimize_index(self, index_name): return self.__optimize_index(index_name) def __optimize_index(self, index_name): start_time = time.time() success = False with self.__lock: try: self.__logger.debug('optimizing {0}'.format(index_name)) self.__get_writer(index_name).commit(optimize=True, merge=False) self.__open_writer(index_name) # reopen writer self.__logger.info('{0} has optimized'.format(index_name)) success = True except Exception as ex: self.__logger.error('failed to optimize {0}: {1}'.format( index_name, ex)) finally: self.__record_metrics(start_time, 'optimize_index') return success def get_doc_count(self, index_name): try: cnt = self.__indices.get(index_name).doc_count() except Exception as ex: raise ex return cnt def get_schema(self, index_name): try: schema = self.__indices.get(index_name).schema except Exception as ex: raise ex return schema @replicated def put_document(self, index_name, doc_id, fields): return self.__put_document(index_name, doc_id, fields) def __put_document(self, index_name, doc_id, fields): doc = copy.deepcopy(fields) doc[self.__index_configs.get(index_name).get_doc_id_field()] = doc_id return self.__put_documents(index_name, [doc]) @replicated def put_documents(self, index_name, docs): return self.__put_documents(index_name, docs) def __put_documents(self, index_name, docs): start_time = time.time() with self.__lock: try: self.__logger.debug( 'putting documents to {0}'.format(index_name)) # count = self.__get_writer(index_name).update_documents(docs) count = 0 for doc in docs: self.__get_writer(index_name).update_document(**doc) count += 1 self.__logger.info('{0} documents has put to {1}'.format( count, index_name)) except Exception as ex: self.__logger.error( 'failed to put documents to {0}: {1}'.format( index_name, ex)) count = -1 finally: self.__record_metrics(start_time, 'put_documents') return count def get_document(self, index_name, doc_id): try: results_page = self.search_documents( index_name, doc_id, self.__index_configs.get(index_name).get_doc_id_field(), 1, page_len=1) if results_page.total > 0: self.__logger.debug('{0} was got from {1}'.format( doc_id, index_name)) else: self.__logger.debug('{0} did not exist in {1}'.format( doc_id, index_name)) except Exception as ex: raise ex return results_page @replicated def delete_document(self, index_name, doc_id): return self.__delete_document(index_name, doc_id) def __delete_document(self, index_name, doc_id): return self.__delete_documents(index_name, [doc_id]) @replicated def delete_documents(self, index_name, doc_ids): return self.__delete_documents(index_name, doc_ids) def __delete_documents(self, index_name, doc_ids): start_time = time.time() with self.__lock: try: self.__logger.debug( 'deleting documents from {0}'.format(index_name)) # count = self.__get_writer(index_name).delete_documents(doc_ids, doc_id_field=self.__index_configs.get( # index_name).get_doc_id_field()) count = 0 for doc_id in doc_ids: count += self.__get_writer(index_name).delete_by_term( self.__index_configs.get( index_name).get_doc_id_field(), doc_id) self.__logger.info('{0} documents has deleted from {1}'.format( count, index_name)) except Exception as ex: self.__logger.error( 'failed to delete documents in bulk to {0}: {1}'.format( index_name, ex)) count = -1 finally: self.__record_metrics(start_time, 'delete_documents') return count def search_documents(self, index_name, query, search_field, page_num, page_len=10, weighting=None, **kwargs): start_time = time.time() try: searcher = self.__get_searcher(index_name, weighting=weighting) query_parser = QueryParser(search_field, self.get_schema(index_name)) query_obj = query_parser.parse(query) results_page = searcher.search_page(query_obj, page_num, pagelen=page_len, **kwargs) self.__logger.info('{0} documents ware searched from {1}'.format( results_page.total, index_name)) except Exception as ex: raise ex finally: self.__record_metrics(start_time, 'search_documents') return results_page @replicated def create_snapshot(self): self.__create_snapshot() def __create_snapshot(self): self.forceLogCompaction() def get_snapshot_file_name(self): return self.__conf.fullDumpFile def is_snapshot_exist(self): return os.path.exists(self.get_snapshot_file_name()) def open_snapshot_file(self): with self.__lock: try: file = open(self.get_snapshot_file_name(), mode='rb') except Exception as ex: raise ex return file
class ManagementHTTPServicer: def __init__(self, manager, logger=getLogger(), http_logger=getLogger(), metrics_registry=CollectorRegistry()): self.__manager = manager self.__logger = logger self.__http_logger = http_logger self.__metrics_registry = metrics_registry # metrics self.__metrics_requests_total = Counter( '{0}_manager_http_requests_total'.format(NAME), 'The number of requests.', ['method', 'endpoint', 'status_code'], registry=self.__metrics_registry) self.__metrics_requests_duration_seconds = Histogram( '{0}_manager_http_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', ['method', 'endpoint'], registry=self.__metrics_registry) self.__metrics_requests_bytes_total = Counter( '{0}_manager_http_requests_bytes_total'.format(NAME), 'A summary of the invocation requests bytes.', ['method', 'endpoint'], registry=self.__metrics_registry) self.__metrics_responses_bytes_total = Counter( '{0}_manager_http_responses_bytes_total'.format(NAME), 'A summary of the invocation responses bytes.', ['method', 'endpoint'], registry=self.__metrics_registry) self.app = Flask('supervise_http_server') self.app.add_url_rule('/data', endpoint='put', view_func=self.__put, methods=['PUT'], strict_slashes=False) self.app.add_url_rule('/data', endpoint='get', view_func=self.__get, methods=['GET'], strict_slashes=False) self.app.add_url_rule('/data', endpoint='delete', view_func=self.__delete, methods=['DELETE'], strict_slashes=False) self.app.add_url_rule('/data/<path:key>', endpoint='put', view_func=self.__put, methods=['PUT']) self.app.add_url_rule('/data/<path:key>', endpoint='get', view_func=self.__get, methods=['GET']) self.app.add_url_rule('/data/<path:key>', endpoint='delete', view_func=self.__delete, methods=['DELETE']) self.app.add_url_rule('/metrics', endpoint='metrics', view_func=self.__metrics, methods=['GET']) self.app.add_url_rule('/healthiness', endpoint='healthiness', view_func=self.__healthiness, methods=['GET']) self.app.add_url_rule('/liveness', endpoint='liveness', view_func=self.__liveness, methods=['GET']) self.app.add_url_rule('/readiness', endpoint='readiness', view_func=self.__readiness, methods=['GET']) self.app.add_url_rule('/status', endpoint='status', view_func=self.__get_status, methods=['GET']) # disable Flask default logger self.app.logger.disabled = True getLogger('werkzeug').disabled = True def __record_metrics(self, start_time, req, resp): self.__metrics_requests_total.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else ''), status_code=resp.status_code.value).inc() self.__metrics_requests_bytes_total.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '')).inc( req.content_length if req.content_length is not None else 0) self.__metrics_responses_bytes_total.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '')).inc( resp.content_length if resp.content_length is not None else 0) self.__metrics_requests_duration_seconds.labels( method=req.method, endpoint=req.path + ('?{0}'.format(req.query_string.decode('utf-8')) if len(req.query_string) > 0 else '')).observe(time.time() - start_time) return def __put(self, key=''): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: mime = mimeparse.parse_mime_type( request.headers.get('Content-Type')) charset = 'utf-8' if mime[2].get( 'charset') is None else mime[2].get('charset') if mime[1] == 'yaml': value = yaml.safe_load(request.data.decode(charset)) elif mime[1] == 'json': value = json.loads(request.data.decode(charset)) else: # handle as a string value = request.data.decode(charset) sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True self.__manager.put(key if key.startswith('/') else '/' + key, value, sync=sync) if sync: status_code = HTTPStatus.CREATED else: status_code = HTTPStatus.ACCEPTED except (ConstructorError, JSONDecodeError, ValueError) as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.BAD_REQUEST self.__logger.error(ex) except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __get(self, key=''): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: value = self.__manager.get(key if key.startswith('/') else '/' + key) if value is None: status_code = HTTPStatus.NOT_FOUND else: data['value'] = value status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __delete(self, key=''): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: sync = False if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS: sync = True value = self.__manager.delete(key if key.startswith('/') else '/' + key, sync=sync) if value is None: status_code = HTTPStatus.NOT_FOUND else: status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __get_status(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, resp) return response data = {} status_code = None try: data['node_status'] = self.__manager.getStatus() status_code = HTTPStatus.OK except Exception as ex: data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __healthiness(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: healthy = self.__manager.is_healthy() data['healthy'] = healthy if healthy: status_code = HTTPStatus.OK else: status_code = HTTPStatus.SERVICE_UNAVAILABLE data['error'] = 'node is not healthy' except Exception as ex: data['healthy'] = False data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __liveness(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: alive = self.__manager.is_alive() data['liveness'] = alive if alive: status_code = HTTPStatus.OK else: status_code = HTTPStatus.SERVICE_UNAVAILABLE data['error'] = 'node is not alive' except Exception as ex: data['liveness'] = False data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __readiness(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response data = {} status_code = None try: ready = self.__manager.is_ready() data['readiness'] = ready if ready: status_code = HTTPStatus.OK else: status_code = HTTPStatus.SERVICE_UNAVAILABLE data['error'] = 'node is not ready' except Exception as ex: data['readiness'] = False data['error'] = '{0}'.format(ex.args[0]) status_code = HTTPStatus.INTERNAL_SERVER_ERROR self.__logger.error(ex) finally: data['time'] = time.time() - start_time data['status'] = { 'code': status_code.value, 'phrase': status_code.phrase, 'description': status_code.description } output = request.args.get('output', default='json', type=str).lower() # make response resp = make_response(data, output) resp.status_code = status_code return resp def __metrics(self): start_time = time.time() @after_this_request def to_do_after_this_request(response): record_log(request, response, logger=self.__http_logger) self.__record_metrics(start_time, request, response) return response resp = Response() try: resp.status_code = HTTPStatus.OK resp.content_type = CONTENT_TYPE_LATEST resp.data = generate_latest(self.__metrics_registry) except Exception as ex: resp.status_code = HTTPStatus.INTERNAL_SERVER_ERROR resp.content_type = 'text/plain; charset="UTF-8"' resp.data = '{0}\n{1}'.format(resp.status_code.phrase, resp.status_code.description) self.__logger.error(ex) return resp
class IndexGRPCServicer(IndexServicer): def __init__(self, indexer, logger=getLogger(), metrics_registry=CollectorRegistry()): self.__indexer = indexer self.__logger = logger self.__metrics_registry = metrics_registry # metrics self.__metrics_requests_total = Counter( '{0}_indexer_grpc_requests_total'.format(NAME), 'The number of requests.', [ 'func' ], registry=self.__metrics_registry ) self.__metrics_requests_duration_seconds = Histogram( '{0}_indexer_grpc_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', [ 'func' ], registry=self.__metrics_registry ) def __record_metrics(self, start_time, func_name): self.__metrics_requests_total.labels( func=func_name ).inc() self.__metrics_requests_duration_seconds.labels( func=func_name ).observe(time.time() - start_time) return def CreateIndex(self, request, context): start_time = time.time() response = CreateIndexResponse() try: index_config = IndexConfig(pickle.loads(request.index_config)) index = self.__indexer.create_index(request.index_name, index_config, sync=request.sync) if request.sync: if index is None: response.status.success = False response.status.message = 'failed to create {0}'.format(request.index_name) else: response.index_stats.name = index.indexname response.index_stats.doc_count = index.doc_count() response.index_stats.doc_count_all = index.doc_count_all() response.index_stats.latest_generation = index.latest_generation() response.index_stats.version = index.version response.index_stats.storage.folder = index.storage.folder response.index_stats.storage.supports_mmap = index.storage.supports_mmap response.index_stats.storage.readonly = index.storage.readonly response.index_stats.storage.files.extend(index.storage.list()) response.status.success = True response.status.message = '{0} was successfully created or opened'.format(index.indexname) else: response.status.success = True response.status.message = 'request was successfully accepted to create {0}'.format(request.index_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'create_index') return response def GetIndex(self, request, context): start_time = time.time() response = GetIndexResponse() try: index = self.__indexer.get_index(request.index_name) if index is None: response.status.success = False response.status.message = '{0} does not exist'.format(request.index_name) else: response.index_stats.name = index.indexname response.index_stats.doc_count = index.doc_count() response.index_stats.doc_count_all = index.doc_count_all() response.index_stats.latest_generation = index.latest_generation() response.index_stats.last_modified = index.last_modified() response.index_stats.version = index.version response.index_stats.storage.folder = index.storage.folder response.index_stats.storage.supports_mmap = index.storage.supports_mmap response.index_stats.storage.readonly = index.storage.readonly response.index_stats.storage.files.extend(index.storage.list()) response.status.success = True response.status.message = '{0} was successfully retrieved'.format(index.indexname) except Exception as ex: response.status.success = False response.status.message = ex.args[0] finally: self.__record_metrics(start_time, 'get_index') return response def DeleteIndex(self, request, context): start_time = time.time() response = DeleteIndexResponse() try: index = self.__indexer.delete_index(request.index_name, sync=request.sync) if request.sync: if index is None: response.status.success = False response.status.message = 'failed to delete {0}'.format(request.index_name) else: response.index_stats.name = index.indexname # response.index_stats.doc_count = index.doc_count() # response.index_stats.doc_count_all = index.doc_count_all() response.index_stats.latest_generation = index.latest_generation() # response.index_stats.version = index.version response.index_stats.storage.folder = index.storage.folder response.index_stats.storage.supports_mmap = index.storage.supports_mmap response.index_stats.storage.readonly = index.storage.readonly response.index_stats.storage.files.extend(index.storage.list()) response.status.success = True response.status.message = '{0} was successfully deleted'.format(index.indexname) else: response.status.success = True response.status.message = 'request was successfully accepted to close {0}'.format(request.index_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'delete_index') return response def OpenIndex(self, request, context): start_time = time.time() response = OpenIndexResponse() try: index_config = None if request.index_config == b'' else IndexConfig(pickle.loads(request.index_config)) index = self.__indexer.open_index(request.index_name, index_config=index_config, sync=request.sync) if request.sync: if index is None: response.status.success = False response.status.message = 'failed to open {0}'.format(request.index_name) else: response.index_stats.name = index.indexname response.index_stats.doc_count = index.doc_count() response.index_stats.doc_count_all = index.doc_count_all() response.index_stats.latest_generation = index.latest_generation() response.index_stats.version = index.version response.index_stats.storage.folder = index.storage.folder response.index_stats.storage.supports_mmap = index.storage.supports_mmap response.index_stats.storage.readonly = index.storage.readonly response.index_stats.storage.files.extend(index.storage.list()) response.status.success = True response.status.message = '{0} was successfully opened'.format(index.indexname) else: response.status.success = True response.status.message = 'request was successfully accepted to open {0}'.format(request.index_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'open_index') return response def CloseIndex(self, request, context): start_time = time.time() response = CloseIndexResponse() try: index = self.__indexer.close_index(request.index_name, sync=request.sync) if request.sync: if index is None: response.status.success = False response.status.message = 'failed to close {0}'.format(request.index_name) else: response.index_stats.name = index.indexname response.index_stats.doc_count = index.doc_count() response.index_stats.doc_count_all = index.doc_count_all() response.index_stats.latest_generation = index.latest_generation() response.index_stats.version = index.version response.index_stats.storage.folder = index.storage.folder response.index_stats.storage.supports_mmap = index.storage.supports_mmap response.index_stats.storage.readonly = index.storage.readonly response.index_stats.storage.files.extend(index.storage.list()) response.status.success = True response.status.message = '{0} was successfully closed'.format(index.indexname) else: response.status.success = True response.status.message = 'request was successfully accepted to close {0}'.format(request.index_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'close_index') return response def CommitIndex(self, request, context): start_time = time.time() response = CommitIndexResponse() try: self.__indexer.commit_index(request.index_name, sync=request.sync) response.status.success = True response.status.message = '{0} was successfully committed'.format(request.index_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'commit_index') return response def RollbackIndex(self, request, context): start_time = time.time() response = RollbackIndexResponse() try: self.__indexer.rollback_index(request.index_name, sync=request.sync) response.status.success = True response.status.message = '{0} was successfully rolled back'.format(request.index_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'rollback_index') return response def OptimizeIndex(self, request, context): start_time = time.time() response = OptimizeIndexResponse() try: index = self.__indexer.optimize_index(request.index_name, sync=request.sync) if request.sync: if index is None: response.status.success = False response.status.message = 'failed to optimize {0}'.format(request.index_name) else: response.index_stats.name = index.indexname response.index_stats.doc_count = index.doc_count() response.index_stats.doc_count_all = index.doc_count_all() response.index_stats.latest_generation = index.latest_generation() response.index_stats.version = index.version response.index_stats.storage.folder = index.storage.folder response.index_stats.storage.supports_mmap = index.storage.supports_mmap response.index_stats.storage.readonly = index.storage.readonly response.index_stats.storage.files.extend(index.storage.list()) response.status.success = True response.status.message = '{0} was successfully optimized'.format(index.indexname) else: response.status.success = True response.status.message = 'request was successfully accepted to optimize {0}'.format(request.index_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'optimize_index') return response def PutDocument(self, request, context): start_time = time.time() response = PutDocumentResponse() try: count = self.__indexer.put_document(request.index_name, request.doc_id, pickle.loads(request.fields), sync=request.sync) if request.sync: response.count = count if response.count > 0: response.status.success = True response.status.message = '{0} was successfully put to {1}'.format(request.doc_id, request.index_name) else: response.status.success = False response.status.message = 'failed to put {0} to {1}'.format(request.document.id, request.index_name) else: response.status.success = True response.status.message = 'request was successfully accepted to put {0} to {1}'.format(request.doc_id, request.index_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'put_document') return response def GetDocument(self, request, context): start_time = time.time() response = GetDocumentResponse() try: results_page = self.__indexer.get_document(request.index_name, request.doc_id) if results_page.total > 0: fields = {} for i in results_page.results[0].iteritems(): fields[i[0]] = i[1] response.fields = pickle.dumps(fields) response.status.success = True response.status.message = '{0} was successfully got from {1}'.format(request.doc_id, request.index_name) else: response.status.success = False response.status.message = '{0} does not exist in {1}'.format(request.doc_id, request.index_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'get_document') return response def DeleteDocument(self, request, context): start_time = time.time() response = DeleteDocumentResponse() try: count = self.__indexer.delete_document(request.index_name, request.doc_id, sync=request.sync) if request.sync: response.count = count if response.count > 0: response.status.success = True response.status.message = '{0} was successfully deleted from {1}'.format(request.doc_id, request.index_name) elif response.count == 0: response.status.success = False response.status.message = '{0} does not exist in {1}'.format(request.doc_id, request.index_name) else: response.status.success = False response.status.message = 'failed to delete {0} to {1}'.format(request.document.id, request.index_name) else: response.status.success = True response.status.message = 'request was successfully accepted to delete {0} to {1}'.format( request.doc_id, request.index_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'delete_document') return response def PutDocuments(self, request, context): start_time = time.time() response = PutDocumentsResponse() try: count = self.__indexer.put_documents(request.index_name, pickle.loads(request.docs), sync=request.sync) if request.sync: response.count = count if response.count > 0: response.status.success = True response.status.message = '{0} documents were successfully put to {1}'.format(response.count, request.index_name) else: response.status.success = False response.status.message = 'failed to put documents to {0}'.format(request.index_name) else: response.status.success = True response.status.message = 'request was successfully accepted to put documents to {0}'.format( request.index_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'put_documents') return response def DeleteDocuments(self, request, context): start_time = time.time() response = DeleteDocumentsResponse() try: count = self.__indexer.delete_documents(request.index_name, pickle.loads(request.doc_ids), sync=request.sync) if request.sync: response.count = count if response.count > 0: response.status.success = True response.status.message = '{0} documents were successfully deleted from {1}'.format(response.count, request.index_name) else: response.status.success = False response.status.message = 'failed to delete documents from {0}'.format(request.index_name) else: response.status.success = True response.status.message = 'request was successfully accepted to delete documents to {0}'.format( request.index_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'delete_documents') return response def SearchDocuments(self, request, context): start_time = time.time() response = SearchDocumentsResponse() try: search_field = request.search_field if request.search_field != '' else self.__indexer.get_schema( request.index_name).get_default_search_field() weighting = BM25F if request.weighting == b'' else get_multi_weighting(pickle.loads(request.weighting)) results_page = self.__indexer.search_documents(request.index_name, request.query, search_field, request.page_num, page_len=request.page_len, weighting=weighting) if results_page.pagecount >= request.page_num or results_page.total <= 0: results = { 'is_last_page': results_page.is_last_page(), 'page_count': results_page.pagecount, 'page_len': results_page.pagelen, 'page_num': results_page.pagenum, 'total': results_page.total, 'offset': results_page.offset } hits = [] for result in results_page.results[results_page.offset:]: fields = {} for item in result.iteritems(): fields[item[0]] = item[1] hit = { 'fields': fields, 'doc_num': result.docnum, 'score': result.score, 'rank': result.rank, 'pos': result.pos } hits.append(hit) results['hits'] = hits response.results = pickle.dumps(results) response.status.success = True response.status.message = '{0} documents were successfully searched from {1}'.format(results_page.total, request.index_name) else: response.status.success = False response.status.message = 'page_num must be <= {0}'.format(results_page.pagecount) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'search_documents') return response def PutNode(self, request, context): start_time = time.time() response = PutNodeResponse() try: self.__indexer.addNodeToCluster(request.node_name) response.status.success = True response.status.message = '{0} was successfully added to the cluster'.format(request.node_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'put_node') return response def DeleteNode(self, request, context): start_time = time.time() response = DeleteNodeResponse() try: self.__indexer.removeNodeFromCluster(request.node_name) response.status.success = True response.status.message = '{0} was successfully deleted from the cluster'.format(request.node_name) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'delete_node') return response def IsSnapshotExist(self, request, context): response = IsSnapshotExistResponse() try: response.exist = self.__indexer.is_snapshot_exist() response.status.success = True response.status.message = 'snapshot exists' if response.exist else 'snapshot does not exist' except Exception as ex: response.status.success = False response.status.message = str(ex) return response def CreateSnapshot(self, request, context): response = CreateSnapshotResponse() try: self.__indexer.create_snapshot(sync=request.sync) response.status.success = True response.status.message = 'request was successfully accepted' except Exception as ex: response.status.success = False response.status.message = str(ex) return response def GetSnapshot(self, request, context): start_time = time.time() def get_snapshot_chunks(chunk_size=1024): with self.__indexer.open_snapshot_file() as f: while True: chunk = f.read(chunk_size) if len(chunk) == 0: return status = Status() status.success = True status.message = 'successfully got snapshot chunk' yield GetSnapshotResponse(name=self.__indexer.get_snapshot_file_name(), chunk=chunk, status=status) try: response = get_snapshot_chunks(chunk_size=request.chunk_size) except Exception as ex: response = GetSnapshotResponse() response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'get_snapshot') return response def IsHealthy(self, request, context): response = IsHealthyResponse() try: response.healthy = self.__indexer.is_healthy() response.status.success = True response.status.message = 'node is alive' if response.healthy else 'node is dead' except Exception as ex: response.status.success = False response.status.message = str(ex) return response def IsAlive(self, request, context): response = IsAliveResponse() try: response.alive = self.__indexer.is_alive() response.status.success = True response.status.message = 'node is alive' if response.alive else 'node is dead' except Exception as ex: response.status.success = False response.status.message = str(ex) return response def IsReady(self, request, context): response = IsReadyResponse() try: response.ready = self.__indexer.is_ready() response.status.success = True response.status.message = 'cluster is ready' if response.ready else 'cluster not ready' except Exception as ex: response.status.success = False response.status.message = str(ex) return response def GetStatus(self, request, context): response = GetStatusResponse() try: response.node_status = pickle.dumps(self.__indexer.getStatus()) response.status.success = True response.status.message = 'successfully got cluster status' except Exception as ex: response.status.success = False response.status.message = str(ex) return response
class ManagementGRPCServicer(ManagementServicer): def __init__(self, manager, logger=getLogger(), metrics_registry=CollectorRegistry()): self.__manager = manager self.__logger = logger self.__metrics_registry = metrics_registry # metrics self.__metrics_requests_total = Counter( '{0}_manager_grpc_requests_total'.format(NAME), 'The number of requests.', [ 'func' ], registry=self.__metrics_registry ) self.__metrics_requests_duration_seconds = Histogram( '{0}_manager_grpc_requests_duration_seconds'.format(NAME), 'The invocation duration in seconds.', [ 'func' ], registry=self.__metrics_registry ) def __record_metrics(self, start_time, func_name): self.__metrics_requests_total.labels( func=func_name ).inc() self.__metrics_requests_duration_seconds.labels( func=func_name ).observe(time.time() - start_time) return def Put(self, request, context): start_time = time.time() response = PutResponse() try: self.__manager.put(request.key, pickle.loads(request.value), sync=request.sync) if request.sync: response.status.success = True response.status.message = '{0} was successfully created or opened'.format(request.key) else: response.status.success = True response.status.message = 'request was successfully accepted to put {0}'.format(request.key) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'put') return response def Get(self, request, context): start_time = time.time() response = GetResponse() try: value = self.__manager.get(request.key) if value is None: response.status.success = False response.status.message = '{0} does not exist'.format(request.key) else: response.value = pickle.dumps(value) response.status.success = True response.status.message = '{0} was successfully retrieved'.format(request.key) except Exception as ex: response.status.success = False response.status.message = ex.args[0] finally: self.__record_metrics(start_time, 'get') return response def Delete(self, request, context): start_time = time.time() response = DeleteResponse() try: value = self.__manager.delete(request.key, sync=request.sync) if request.sync: if value is None: if request.key == '/': response.status.success = True response.status.message = '{0} was successfully deleted'.format(request.key) else: response.status.success = False response.status.message = '{0} does not exist'.format(request.key) else: response.value = pickle.dumps(value) response.status.success = True response.status.message = '{0} was successfully deleted'.format(request.key) else: response.status.success = True response.status.message = 'request was successfully accepted to delete {0}'.format(request.key) except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'get') return response def Clear(self, request, context): start_time = time.time() response = ClearResponse() try: self.__manager.clear(sync=request.sync) if request.sync: response.status.success = True response.status.message = 'successfully cleared' else: response.status.success = True response.status.message = 'request was successfully accepted to clear' except Exception as ex: response.status.success = False response.status.message = str(ex) finally: self.__record_metrics(start_time, 'clear') return response