def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) try: chan = att.TCPChan(HOST, TCPPORT) # Case 1 invalid_sample = "+cpuload host=machine1\\r\\n:1418224205000000000\\r\\r+25.0\\r\\n\n" # reportd in issue#173 chan.send(invalid_sample) #time.sleep(1) # wait untill all messagess will be processed query = {"select":"cpuload","range": {"from":1418224205000000000, "to":1418224505000000000}} queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) # response should be empty for line in response: print("Unexpected response: {0}".format(line)) raise ValueError("Unexpected response") err = chan.recv() print(err) if not err.startswith("-PARSER"): raise ValueError("Error message expected") chan.close() # Case 2 chan = att.TCPChan(HOST, TCPPORT) invalid_sample = "+cpuload host=machine2\r\n:1418224205000000000\r\n+25.0" chan.send(invalid_sample) time.sleep(1) response = urlopen(queryurl, json.dumps(query)) # response should be empty for line in response: print("Unexpected response: {0}".format(line)) raise ValueError("Unexpected response") # No error message expected because the write is incomplete chan.close() except: traceback.print_exc() sys.exit(1) finally: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(hour=0, minute=0, second=10, microsecond=0) delta = datetime.timedelta(seconds=1) nmsgs = 1000 snames = [ '!foo A=1 B=1', '!foo A=1 B=2', '!foo A=2 B=1', '!foo A=2 B=2', ] print("Sending {0} messages through TCP...".format(nmsgs * len(snames))) cnt = 0 timestamp = dt for it in range(0, nmsgs): for sname in snames: timestr = timestamp.strftime('+%Y%m%dT%H%M%S.%f') event = "{0} event {1} for {2} generated".format( cnt, sname, timestr) msg = "+{0}\r\n+{1}\r\n+{2}\r\n".format( sname, timestr, event[:it + 1]) allevents.append((timestamp, event[:it + 1])) chan.send(msg) cnt += 1 timestamp = timestamp + delta time.sleep(5) # wait untill all messagess will be processed test_select_events_forward(dt, delta, nmsgs * len(snames)) test_select_events_backward(dt, delta, nmsgs * len(snames)) finally: print("Stopping server...") akumulid.stop() time.sleep(5) on_exit()
def main(path, debug=False): if not os.path.exists(path): print("Path {0} doesn't exists".format(path)) sys.exit(1) akumulid = att.Akumulid(path) if not debug: # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) else: print("Akumulid should be started first") try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(second=0, microsecond=0) delta = datetime.timedelta(milliseconds=1) nmsgs = 100000 print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } def getval(ix): return float(ix % (nmsgs * 10)) for it in att.generate_messages2(dt, delta, nmsgs, 'test', getval, **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed test_sax_in_backward_direction(dt, delta, nmsgs) except: traceback.print_exc() sys.exit(1) finally: if not debug: print("Stopping server...") akumulid.stop() time.sleep(5)
def writer(dt, delta, N): try: chan = att.TCPChan(HOST, TCPPORT) # fill data in print("Sending {0} messages through TCP...".format(N)) tags = { "tag": ['Foo'], } print("Generating {0} messages...".format(N)) messages = att.generate_messages(dt, delta, N, 'test', **tags) for it in messages: chan.send(it) print("{0} messages sent".format(N)) time.sleep(5) except: print("Exception in writer") traceback.print_exc() raise
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(second=0, microsecond=0) delta = datetime.timedelta(milliseconds=1) nmsgs = 100000 print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C', 'D'], } for it in att.generate_messages(dt, delta, nmsgs, 'col1', **tags): chan.send(it) for it in att.generate_messages(dt, delta, nmsgs, 'col2', **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed columns = ['col1', 'col2'] test_join_query_forward(columns, dt, delta, nmsgs) test_join_query_backward(columns, dt, delta, nmsgs) test_join_query_forward_by_series(columns, dt, delta, nmsgs) test_join_query_backward_by_series(columns, dt, delta, nmsgs) except: traceback.print_exc() sys.exit(1) finally: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(second=0, microsecond=0) delta = datetime.timedelta(milliseconds=1) nmsgs = N_MSG for metric in METRICS: print("Sending {0} messages through TCP...".format(nmsgs)) for it in att.generate_messages(dt, delta, nmsgs, metric, **SERIES_TAGS): chan.send(it) time.sleep(5) # wait untill all messagess will be processed # Test cases test_suggest_metric() test_suggest_metric_prefix() test_suggest_tag() test_suggest_tag_prefix() test_suggest_value() test_suggest_value_prefix() test_search_all_names() test_search_names_with_metric() test_search_names_with_tag() finally: print("Stopping server...") akumulid.stop() time.sleep(5) on_exit()
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(hour=0, minute=0, second=10, microsecond=0) delta = datetime.timedelta(seconds=1) nmsgs = 3600*24 print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } for it in att.generate_messages(dt, delta, nmsgs, 'test', **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed # Test normal operation test_group_aggregate_all_forward (dt, delta, nmsgs, '1m') test_group_aggregate_all_forward (dt, delta, nmsgs, '10m') test_group_aggregate_all_forward (dt, delta, nmsgs, '1h') test_group_aggregate_all_backward(dt, delta, nmsgs, '1m') test_group_aggregate_all_backward(dt, delta, nmsgs, '10m') test_group_aggregate_all_backward(dt, delta, nmsgs, '1h') finally: print("Stopping server...") akumulid.stop() time.sleep(5) on_exit()
def writer(dt, delta, N): try: chan = att.TCPChan(HOST, TCPPORT) # fill data in print("Sending {0} messages through TCP...".format(N)) tags = { "tag": ['Foo'], } print("Generating first {0} messages...".format(N / 2)) messages = att.generate_messages(dt, delta, N, 'test', **tags) for it in itertools.islice(messages, N / 2): chan.send(it) time.sleep(10) print("Generating last {0} messages...".format(N / 2)) for it in messages: chan.send(it) print("{0} messages sent".format(N)) time.sleep(10) except: print("Exception in writer") traceback.print_exc() sys.exit(1)
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) nmsgs = 100000 dt = datetime.datetime.utcnow() - (datetime.timedelta(milliseconds=1) * nmsgs) delta = datetime.timedelta(milliseconds=1) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } for it in att.generate_messages(dt, delta, nmsgs, 'test', **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed print("Trying to close channel") chan.close() test_read_all(dt, delta, nmsgs) except: traceback.print_exc() sys.exit(1) finally: print("Stopping server...") akumulid.stop() time.sleep(5) print("Server stopped") print("Starting server...") akumulid.serve() time.sleep(5) print("Server started") try: test_read_all(dt, delta, nmsgs) # Try to write new data dt = datetime.datetime.utcnow() chan = att.TCPChan(HOST, TCPPORT) # fill data in print("Sending {0} messages through TCP second time...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } for it in att.generate_messages(dt, delta, nmsgs, 'test', **tags): chan.send(it) time.sleep(5) test_read_all(dt, delta, nmsgs) finally: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(second=0, microsecond=0) delta = datetime.timedelta(milliseconds=1) nmsgs = 100000 print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C', 'D'], } values = [float(x) for x in range(-100, 100)] for it in att.generate_messages4(dt, delta, nmsgs, 'col1', values, **tags): chan.send(it) for it in att.generate_messages4(dt, delta, nmsgs, 'col2', values, **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed test_filter_query_forward('col1', dt, delta, [-20, 20], nmsgs) test_filter_query_backward('col1', dt, delta, [-20, 20], nmsgs) test_filter_query_forward_by_time('col1', dt, delta, [-20, 20], nmsgs) test_filter_query_backward_by_time('col1', dt, delta, [-20, 20], nmsgs) test_filter_query_empty('col1', dt, delta, nmsgs) test_join_query_forward(['col1', 'col2'], [[-20, 20], [40, 60]], dt, delta, nmsgs) test_join_query_backward(['col1', 'col2'], [[-20, 20], [40, 60]], dt, delta, nmsgs) test_join_query_forward_by_time(['col1', 'col2'], [[-20, 20], [40, 60]], dt, delta, nmsgs) test_join_query_backward_by_time(['col1', 'col2'], [[-20, 20], [40, 60]], dt, delta, nmsgs) test_group_aggregate_all_forward('col1', dt, delta, nmsgs, 20000, "all") test_group_aggregate_all_forward('col1', dt, delta, nmsgs, 10000, "all") test_group_aggregate_all_forward('col1', dt, delta, nmsgs, 5000, "all") test_group_aggregate_all_forward('col1', dt, delta, nmsgs, 20000, "any") test_group_aggregate_all_forward('col1', dt, delta, nmsgs, 10000, "any") test_group_aggregate_all_forward('col1', dt, delta, nmsgs, 5000, "any") except: traceback.print_exc() sys.exit(1) finally: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path, debug=False): if not os.path.exists(path): print("Path {0} doesn't exists".format(path)) sys.exit(1) akumulid = att.Akumulid(path) if not debug: # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) else: print("Akumulid should be started first") try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(second=0, microsecond=0) delta = datetime.timedelta(milliseconds=1) nmsgs = 100000 print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } for it in att.generate_messages(dt, delta, nmsgs, 'test', **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed test_read_all_in_backward_direction(dt, delta, nmsgs) test_group_by_tag_in_backward_direction(dt, delta, nmsgs) test_where_clause_in_backward_direction(dt, delta, nmsgs) test_where_clause_with_groupby_in_backward_direction(dt, delta, nmsgs) test_metadata_query(tags) test_read_in_forward_direction(dt, delta, nmsgs) test_late_write(dt, delta, nmsgs, chan) test_paa_in_backward_direction("Test #8 - PAA", dt, delta, nmsgs, lambda buf: float(sum(buf)) / len(buf), "paa") test_paa_in_backward_direction("Test #9 - median PAA", dt, delta, nmsgs, med, "median-paa") test_paa_in_backward_direction("Test #10 - max PAA", dt, delta, nmsgs, max, "max-paa") test_paa_in_backward_direction("Test #11 - min PAA", dt, delta, nmsgs, min, "min-paa") test_paa_in_backward_direction("Test #12 - first wins PAA", dt, delta, nmsgs, lambda buf: buf[0], "first-paa") test_paa_in_backward_direction("Test #13 - last wins PAA", dt, delta, nmsgs, lambda buf: buf[-1], "last-paa") except: traceback.print_exc() sys.exit(1) finally: if not debug: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path): if not os.path.exists(path): print("Path {0} doesn't exists".format(path)) sys.exit(1) akumulid = att.Akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) nmsgs = 100000 tags = [ { "tag3": "D", "tag2": "B", "tag1": "A" }, { "tag3": "E", "tag2": "B", "tag1": "A" }, { "tag3": "F", "tag2": "B", "tag1": "A" }, { "tag3": "G", "tag2": "B", "tag1": "A" }, { "tag3": "H", "tag2": "B", "tag1": "A" }, { "tag3": "D", "tag2": "C", "tag1": "A" }, { "tag3": "E", "tag2": "C", "tag1": "A" }, { "tag3": "F", "tag2": "C", "tag1": "A" }, { "tag3": "G", "tag2": "C", "tag1": "A" }, { "tag3": "H", "tag2": "C", "tag1": "A" }, ] dt = datetime.datetime.utcnow() - (datetime.timedelta(milliseconds=1) * nmsgs) delta = datetime.timedelta(milliseconds=1) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in print("Sending {0} messages through TCP...".format(nmsgs)) for it in att.generate_messages3(dt, delta, nmsgs, 'test', tags): chan.send(it) # kill process akumulid.terminate() except: traceback.print_exc() sys.exit(1) finally: print("Server terminated") print("Starting server...") akumulid.serve() time.sleep(5) print("Server started") try: test_read_all(tags, dt, delta, nmsgs) # Try to write some data chan = att.TCPChan(HOST, TCPPORT) dt = datetime.datetime.utcnow() print("Sending {0} messages through TCP...".format(nmsgs)) for it in att.generate_messages3(dt, delta, nmsgs, 'test', tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed print("Trying to close channel") chan.close() test_read_all(tags, dt, delta, nmsgs) finally: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path): if not os.path.exists(path): print("Path {0} doesn't exists".format(path)) sys.exit(1) akumulid = att.Akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) nmsgs = 100000 tags = [ { "tag3": "D", "tag2": "B", "tag1": "A" }, { "tag3": "E", "tag2": "B", "tag1": "A" }, { "tag3": "F", "tag2": "B", "tag1": "A" }, { "tag3": "G", "tag2": "B", "tag1": "A" }, { "tag3": "H", "tag2": "B", "tag1": "A" }, { "tag3": "D", "tag2": "C", "tag1": "A" }, { "tag3": "E", "tag2": "C", "tag1": "A" }, { "tag3": "F", "tag2": "C", "tag1": "A" }, { "tag3": "G", "tag2": "C", "tag1": "A" }, { "tag3": "H", "tag2": "C", "tag1": "A" }, ] dt = datetime.datetime.utcnow() - (datetime.timedelta(milliseconds=1) * nmsgs) delta = datetime.timedelta(milliseconds=1) try: chan = att.TCPChan(HOST, TCPPORT) print("Sending {0} messages through TCP...".format(nmsgs)) # Send 1 message for each series in rare set for it in att.generate_messages3(dt, delta, 10, 'rare', tags): chan.send(it) # Send a lot of other messages to ensure that the raer set is on disk for it in att.generate_messages3(dt, delta, 1000, 'bulk', tags): chan.send(it) chan.close() time.sleep(1) # kill process akumulid.terminate() except: traceback.print_exc() sys.exit(1) finally: print("Server terminated") print("Starting server...") akumulid.serve() time.sleep(5) print("Server started") try: test_select('rare', tags, dt, delta, 10) test_select('bulk', tags, dt, delta, 1000) finally: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) try: chan = att.TCPChan(HOST, TCPPORT) # fill data in dt = datetime.datetime.utcnow().replace(second=0, microsecond=0) delta = datetime.timedelta(milliseconds=1) nmsgs = 100000 print("Sending {0} messages through TCP...".format(nmsgs)) tags = { "tag1": ['A'], "tag2": ['B', 'C'], "tag3": ['D', 'E', 'F', 'G', 'H'], } for it in att.generate_messages(dt, delta, nmsgs, 'test', **tags): chan.send(it) time.sleep(5) # wait untill all messagess will be processed # Test normal operation test_read_all_in_backward_direction(dt, delta, nmsgs) test_group_by_tag_in_backward_direction(dt, delta, nmsgs) test_where_clause_in_backward_direction(dt, delta, nmsgs) test_where_clause_with_groupby_in_backward_direction(dt, delta, nmsgs) test_metadata_query(tags) test_read_in_forward_direction(dt, delta, nmsgs) test_late_write(dt, delta, nmsgs, chan) test_aggregate_all(dt, delta, nmsgs) test_aggregate_last(dt, delta, nmsgs) test_aggregate_last_timestamp(dt, delta, nmsgs) test_aggregate_all_group_by(dt, delta, nmsgs) test_aggregate_where(dt, delta, nmsgs) test_group_aggregate_all_forward (dt, delta, nmsgs, 10) test_group_aggregate_all_forward (dt, delta, nmsgs, 100) test_group_aggregate_all_forward (dt, delta, nmsgs, 1000) test_group_aggregate_all_forward (dt, delta, nmsgs, 100000) test_group_aggregate_all_backward(dt, delta, nmsgs, 10) test_group_aggregate_all_backward(dt, delta, nmsgs, 100) test_group_aggregate_all_backward(dt, delta, nmsgs, 1000) test_group_aggregate_all_backward(dt, delta, nmsgs, 100000) # Test error handling select_from_nonexistent_metric(dt, delta, nmsgs) aggregate_nonexistent_metric(dt, delta, nmsgs) group_aggregate_nonexistent_metric(dt, delta, nmsgs) join_nonexistent_metrics(dt, delta, nmsgs) select_nonexistent_time_range(dt, delta, nmsgs) aggregate_nonexistent_time_range(dt, delta, nmsgs) group_aggregate_nonexistent_time_range(dt, delta, nmsgs) join_nonexistent_time_range(dt, delta, nmsgs) check_bad_query_handling() finally: print("Stopping server...") akumulid.stop() time.sleep(5) on_exit()
def main(path): if not os.path.exists(path): print("Path {0} doesn't exists".format(path)) sys.exit(1) akumulid = att.Akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) nmsgs = 100000 tags = [ { "tag3": "D", "tag2": "B", "tag1": "A" }, { "tag3": "E", "tag2": "B", "tag1": "A" }, { "tag3": "F", "tag2": "B", "tag1": "A" }, { "tag3": "G", "tag2": "B", "tag1": "A" }, { "tag3": "H", "tag2": "B", "tag1": "A" }, { "tag3": "D", "tag2": "C", "tag1": "A" }, { "tag3": "E", "tag2": "C", "tag1": "A" }, { "tag3": "F", "tag2": "C", "tag1": "A" }, { "tag3": "G", "tag2": "C", "tag1": "A" }, { "tag3": "H", "tag2": "C", "tag1": "A" }, ] dt = datetime.datetime.utcnow() - (datetime.timedelta(milliseconds=1) * nmsgs) delta = datetime.timedelta(milliseconds=1) try: chan = att.TCPChan(HOST, TCPPORT) print("Sending {0} messages through TCP...".format(nmsgs)) for it in att.generate_messages3(dt, delta, 100, 'test', tags): chan.send(it) chan.close() test_select('test', tags, dt, delta, 100) time.sleep(5) # kill process akumulid.terminate() except: traceback.print_exc() sys.exit(1) finally: print("Server terminated") try: statinfo = os.stat(os.path.expanduser("~/.akumuli/inputlog0_0.ils")) print("inputlog0_0.ils size is {0}".format(statinfo.st_size)) statinfo = os.stat(os.path.expanduser("~/.akumuli/inputlog0_1.ils")) print("inputlog0_1.ils size is {0}".format(statinfo.st_size)) except: pass print("Starting server...") akumulid.serve() time.sleep(5) print("Server started") try: require_empty('test', dt, delta, 100) finally: print("Stopping server...") akumulid.stop() time.sleep(5)
def main(path): akumulid = att.create_akumulid(path) # Reset database akumulid.delete_database() akumulid.create_database() # start ./akumulid server print("Starting server...") akumulid.serve() time.sleep(5) def get_tags(): for ix in xrange(0, NSERIES): yield {"tag1": "A", "tag2": str(ix)} tags = list(get_tags()) dt = datetime.datetime.utcnow() - (datetime.timedelta(milliseconds=1) * 10) delta = datetime.timedelta(milliseconds=1) try: chan = att.TCPChan(HOST, TCPPORT) print("Sending {0} messages through TCP...".format(10 * NSERIES)) # Send 10 messages for each series in the set for ix, it in enumerate( att.generate_messages5(dt, delta, 10, 'test', tags)): chan.send(it) if ix % 100000 == 0: print("{0} series created".format(ix)) chan.close() time.sleep(15) # kill process akumulid.terminate() except: traceback.print_exc() akumulid.terminate() sys.exit(1) finally: print("Server terminated") print("Starting recovery...") akumulid.serve() while True: try: # Wait until server will respond to stas query # which mean that the recovery is completed. statsurl = "http://{0}:{1}/api/stats".format(HOST, HTTPPORT) _ = urllib.urlopen(statsurl).read() except: time.sleep(1) continue break print("Recovery completed") try: test_metadata("test", tags) finally: print("Stopping server...") akumulid.stop() time.sleep(5)