def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(second=0, microsecond=0) delta = datetime.timedelta(milliseconds=1) nmsgs = 100000 print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C', 'D'], } for it in att.generate_messages(dt, delta, nmsgs, 'col1', **tags): chan.send(it) for it in att.generate_messages(dt, delta, nmsgs, 'col2', **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed columns = ['col1', 'col2'] test_join_query_forward(columns, dt, delta, nmsgs) test_join_query_backward(columns, dt, delta, nmsgs) test_join_query_forward_by_series(columns, dt, delta, nmsgs) test_join_query_backward_by_series(columns, dt, delta, nmsgs) except: traceback.print_exc() sys.exit(1) finally: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(second=0, microsecond=0) delta = datetime.timedelta(milliseconds=1) nmsgs = 100000 print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C', 'D'], } for it in att.generate_messages(dt, delta, nmsgs, 'col1', **tags): chan.send(it) for it in att.generate_messages(dt, delta, nmsgs, 'col2', **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed columns = [ 'col1', 'col2' ] test_join_query_forward(columns, dt, delta, nmsgs) test_join_query_backward(columns, dt, delta, nmsgs) test_join_query_forward_by_series(columns, dt, delta, nmsgs) test_join_query_backward_by_series(columns, dt, delta, nmsgs) except: traceback.print_exc() sys.exit(1) finally: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path, protocol): if not os.path.exists(path): print("Path {0} doesn't exists".format(path)) sys.exit(1) akumulid = att.Akumulid(path) # delete database akumulid.delete_database() # create empty database akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) try: # fill data in host = '127.0.0.1' udpport = 8383 tcpport = 8282 if protocol == 'TCP': chan = TCPChan(host, tcpport) elif protocol == 'UDP': chan = UDPChan(host, udpport) else: print('Unknown protocol "{0}"'.format(protocol)) dt = datetime.datetime.utcnow() delta = datetime.timedelta(milliseconds=1) nmsgs = 1000000 print("Sending {0} messages through {1}...".format(nmsgs, protocol)) for it in att.generate_messages(dt, delta, nmsgs, 'temp', tag='test'): chan.send(it) # check stats httpport = 8181 statsurl = "http://{0}:{1}/stats".format(host, httpport) rawstats = urllib.urlopen(statsurl).read() stats = json.loads(rawstats) # some space should be used volume0space = stats["volume_0"]["free_space"] volume1space = stats["volume_1"]["free_space"] if volume0space == volume1space: print("Test #1 failed. Nothing was written to disk, /stats:") print(rawstats) sys.exit(10) else: print("Test #1 passed") except: traceback.print_exc() sys.exit(1) finally: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path, debug=False): if not os.path.exists(path): print("Path {0} doesn't exists".format(path)) sys.exit(1) akumulid = att.Akumulid(path) if not debug: # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) else: print("Akumulid should be started first") try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(second=0, microsecond=0) delta = datetime.timedelta(milliseconds=1) nmsgs = 100000 print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } for it in att.generate_messages(dt, delta, nmsgs, 'test', **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed test_read_all_in_backward_direction(dt, delta, nmsgs) test_group_by_tag_in_backward_direction(dt, delta, nmsgs) test_where_clause_in_backward_direction(dt, delta, nmsgs) test_where_clause_with_groupby_in_backward_direction(dt, delta, nmsgs) test_metadata_query(tags) test_read_in_forward_direction(dt, delta, nmsgs) test_late_write(dt, delta, nmsgs, chan) test_paa_in_backward_direction("Test #8 - PAA", dt, delta, nmsgs, lambda buf: float(sum(buf))/len(buf), "paa") test_paa_in_backward_direction("Test #9 - median PAA", dt, delta, nmsgs, med, "median-paa") test_paa_in_backward_direction("Test #10 - max PAA", dt, delta, nmsgs, max, "max-paa") test_paa_in_backward_direction("Test #11 - min PAA", dt, delta, nmsgs, min, "min-paa") test_paa_in_backward_direction("Test #12 - first wins PAA", dt, delta, nmsgs, lambda buf: buf[0], "first-paa") test_paa_in_backward_direction("Test #13 - last wins PAA", dt, delta, nmsgs, lambda buf: buf[-1], "last-paa") except: traceback.print_exc() sys.exit(1) finally: if not debug: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(nmsgs): # fill data in host = '127.0.0.1' tcpport = 8282 chan = TCPChan(host, tcpport) dt = datetime.datetime.utcnow() delta = datetime.timedelta(milliseconds=1) print("Sending {0} messages through TCP...".format(nmsgs)) for it in att.generate_messages(dt, delta, nmsgs, 'temp', tag='test'): chan.send(it) print("Starting date-time: {0}".format(dt.strftime("%Y%m%dT%H%M%S.%f")))
def writer(dt, delta, N): try: chan = att.TCPChan(HOST, TCPPORT) # fill data in print("Sending {0} messages through TCP...".format(N)) tags = { "tag": ['Foo'], } print("Generating {0} messages...".format(N)) messages = att.generate_messages(dt, delta, N, 'test', **tags) for it in messages: chan.send(it) print("{0} messages sent".format(N)) time.sleep(5) except: print("Exception in writer") traceback.print_exc() raise
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(second=0, microsecond=0) delta = datetime.timedelta(milliseconds=1) nmsgs = N_MSG for metric in METRICS: print("Sending {0} messages through TCP...".format(nmsgs)) for it in att.generate_messages(dt, delta, nmsgs, metric, **SERIES_TAGS): chan.send(it) time.sleep(5) # wait untill all messagess will be processed # Test cases test_suggest_metric() test_suggest_metric_prefix() test_suggest_tag() test_suggest_tag_prefix() test_suggest_value() test_suggest_value_prefix() test_search_all_names() test_search_names_with_metric() test_search_names_with_tag() finally: print("Stopping server...") akumulid.stop() time.sleep(5) on_exit()
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(hour=0, minute=0, second=10, microsecond=0) delta = datetime.timedelta(seconds=1) nmsgs = 3600*24 print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } for it in att.generate_messages(dt, delta, nmsgs, 'test', **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed # Test normal operation test_group_aggregate_all_forward (dt, delta, nmsgs, '1m') test_group_aggregate_all_forward (dt, delta, nmsgs, '10m') test_group_aggregate_all_forward (dt, delta, nmsgs, '1h') test_group_aggregate_all_backward(dt, delta, nmsgs, '1m') test_group_aggregate_all_backward(dt, delta, nmsgs, '10m') test_group_aggregate_all_backward(dt, delta, nmsgs, '1h') finally: print("Stopping server...") akumulid.stop() time.sleep(5) on_exit()
def writer(dt, delta, N): try: chan = att.TCPChan(HOST, TCPPORT) # fill data in print("Sending {0} messages through TCP...".format(N)) tags = { "tag": ['Foo'], } print("Generating first {0} messages...".format(N / 2)) messages = att.generate_messages(dt, delta, N, 'test', **tags) for it in itertools.islice(messages, N / 2): chan.send(it) time.sleep(10) print("Generating last {0} messages...".format(N / 2)) for it in messages: chan.send(it) print("{0} messages sent".format(N)) time.sleep(10) except: print("Exception in writer") traceback.print_exc() sys.exit(1)
def writer(dt, delta, N): try: chan = att.TCPChan(HOST, TCPPORT) # fill data in print("Sending {0} messages through TCP...".format(N)) tags = { "tag": ['Foo'], } print("Generating first {0} messages...".format(N/2)) messages = att.generate_messages(dt, delta, N, 'test', **tags) for it in itertools.islice(messages, N/2): chan.send(it) time.sleep(10) print("Generating last {0} messages...".format(N/2)) for it in messages: chan.send(it) print("{0} messages sent".format(N)) time.sleep(10) except: print("Exception in writer") traceback.print_exc() sys.exit(1)
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) nmsgs = 100000 dt = datetime.datetime.utcnow() - (datetime.timedelta(milliseconds=1) * nmsgs) delta = datetime.timedelta(milliseconds=1) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } for it in att.generate_messages(dt, delta, nmsgs, 'test', **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed print("Trying to close channel") chan.close() test_read_all(dt, delta, nmsgs) except: traceback.print_exc() sys.exit(1) finally: print("Stopping server...") akumulid.stop() time.sleep(5) print("Server stopped") print("Starting server...") akumulid.serve() time.sleep(5) print("Server started") try: test_read_all(dt, delta, nmsgs) # Try to write new data dt = datetime.datetime.utcnow() chan = att.TCPChan(HOST, TCPPORT) # fill data in print("Sending {0} messages through TCP second time...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } for it in att.generate_messages(dt, delta, nmsgs, 'test', **tags): chan.send(it) time.sleep(5) test_read_all(dt, delta, nmsgs) finally: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path): akumulid = att.create_akumulid(path) # delete database akumulid.delete_database() # create empty database akumulid.create_test_database() # start ./akumulid server print("Starting server...") dt = datetime.datetime.utcnow() delta = datetime.timedelta(milliseconds=1) try: akumulid.serve() time.sleep(5) chan = TCPChan(host, tcpport) # This data will be evicted from the database for it in att.generate_messages(dt, delta, 2000, 'evicted', tag=["foo", "bar"]): chan.send(it) except: traceback.print_exc() sys.exit(1) finally: print("Stopping server...") akumulid.stop() time.sleep(5) try: akumulid.serve() time.sleep(5) # fill data in statsurl = "http://{0}:{1}/api/stats".format(host, httpport) chan = TCPChan(host, tcpport) def get_free_space(): rawstats = urllib.urlopen(statsurl).read() stats = json.loads(rawstats) volspace = 0 volspace += int(stats["volume_0"]["free_space"]) volspace += int(stats["volume_1"]["free_space"]) volspace += int(stats["volume_2"]["free_space"]) volspace += int(stats["volume_3"]["free_space"]) return volspace print("Sending messages...") prevspace = get_free_space() batch_size = 1000 for ix, it in enumerate(att.infinite_msg_stream(batch_size, 'temp', tag='test')): chan.send(it) if ix % 1000 == 0: volspace = get_free_space() print("{0} msgs written, free space in the database: {1}".format(ix*batch_size, volspace)) if prevspace < volspace: # free space increased because volume was recycled print("Volume recycle occured") break prevspace = volspace # Read data back if backward direction (cached values should be included) read_in_backward_direction(batch_size) except: traceback.print_exc() sys.exit(1) finally: print("Stopping server...") akumulid.stop() time.sleep(5) try: akumulid.serve() time.sleep(5) require_empty_response('evicted') chan = TCPChan(host, tcpport) for it in att.generate_messages(dt + datetime.timedelta(milliseconds=2000), delta, 2000, 'evicted', tag=["foo", "bar"]): # Send next 2000 messages that belongs to evicted series chan.send(it) except: traceback.print_exc() sys.exit(1) finally: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path, debug=False): if not os.path.exists(path): print("Path {0} doesn't exists".format(path)) sys.exit(1) akumulid = att.Akumulid(path) if not debug: # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) else: print("Akumulid should be started first") try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(second=0, microsecond=0) delta = datetime.timedelta(milliseconds=1) nmsgs = 100000 print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } for it in att.generate_messages(dt, delta, nmsgs, 'test', **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed test_read_all_in_backward_direction(dt, delta, nmsgs) test_group_by_tag_in_backward_direction(dt, delta, nmsgs) test_where_clause_in_backward_direction(dt, delta, nmsgs) test_where_clause_with_groupby_in_backward_direction(dt, delta, nmsgs) test_metadata_query(tags) test_read_in_forward_direction(dt, delta, nmsgs) test_late_write(dt, delta, nmsgs, chan) test_paa_in_backward_direction("Test #8 - PAA", dt, delta, nmsgs, lambda buf: float(sum(buf)) / len(buf), "paa") test_paa_in_backward_direction("Test #9 - median PAA", dt, delta, nmsgs, med, "median-paa") test_paa_in_backward_direction("Test #10 - max PAA", dt, delta, nmsgs, max, "max-paa") test_paa_in_backward_direction("Test #11 - min PAA", dt, delta, nmsgs, min, "min-paa") test_paa_in_backward_direction("Test #12 - first wins PAA", dt, delta, nmsgs, lambda buf: buf[0], "first-paa") test_paa_in_backward_direction("Test #13 - last wins PAA", dt, delta, nmsgs, lambda buf: buf[-1], "last-paa") except: traceback.print_exc() sys.exit(1) finally: if not debug: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) nmsgs = 100000 dt = datetime.datetime.utcnow() - (datetime.timedelta(milliseconds=1)*nmsgs) delta = datetime.timedelta(milliseconds=1) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } for it in att.generate_messages(dt, delta, nmsgs, 'test', **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed print("Trying to close channel") chan.close() test_read_all(dt, delta, nmsgs) except: traceback.print_exc() sys.exit(1) finally: print("Stopping server...") akumulid.stop() time.sleep(5) print("Server stopped") print("Starting server...") akumulid.serve() time.sleep(5) print("Server started") try: test_read_all(dt, delta, nmsgs) # Try to write new data dt = datetime.datetime.utcnow() chan = att.TCPChan(HOST, TCPPORT) # fill data in print("Sending {0} messages through TCP second time...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } for it in att.generate_messages(dt, delta, nmsgs, 'test', **tags): chan.send(it) time.sleep(5) test_read_all(dt, delta, nmsgs) finally: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(second=0, microsecond=0) delta = datetime.timedelta(milliseconds=1) nmsgs = 100000 print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } for it in att.generate_messages(dt, delta, nmsgs, 'test', **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed # Test normal operation test_read_all_in_backward_direction(dt, delta, nmsgs) test_group_by_tag_in_backward_direction(dt, delta, nmsgs) test_where_clause_in_backward_direction(dt, delta, nmsgs) test_where_clause_with_groupby_in_backward_direction(dt, delta, nmsgs) test_metadata_query(tags) test_read_in_forward_direction(dt, delta, nmsgs) test_late_write(dt, delta, nmsgs, chan) test_aggregate_all(dt, delta, nmsgs) test_aggregate_last(dt, delta, nmsgs) test_aggregate_last_timestamp(dt, delta, nmsgs) test_aggregate_all_group_by(dt, delta, nmsgs) test_aggregate_where(dt, delta, nmsgs) test_group_aggregate_all_forward (dt, delta, nmsgs, 10) test_group_aggregate_all_forward (dt, delta, nmsgs, 100) test_group_aggregate_all_forward (dt, delta, nmsgs, 1000) test_group_aggregate_all_forward (dt, delta, nmsgs, 100000) test_group_aggregate_all_backward(dt, delta, nmsgs, 10) test_group_aggregate_all_backward(dt, delta, nmsgs, 100) test_group_aggregate_all_backward(dt, delta, nmsgs, 1000) test_group_aggregate_all_backward(dt, delta, nmsgs, 100000) # Test error handling select_from_nonexistent_metric(dt, delta, nmsgs) aggregate_nonexistent_metric(dt, delta, nmsgs) group_aggregate_nonexistent_metric(dt, delta, nmsgs) join_nonexistent_metrics(dt, delta, nmsgs) select_nonexistent_time_range(dt, delta, nmsgs) aggregate_nonexistent_time_range(dt, delta, nmsgs) group_aggregate_nonexistent_time_range(dt, delta, nmsgs) join_nonexistent_time_range(dt, delta, nmsgs) check_bad_query_handling() finally: print("Stopping server...") akumulid.stop() time.sleep(5) on_exit()