def line2tup(seq): for line in seq: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) yield tagline, timestamp, value
def line2tup(seq): for line in seq: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) yield tagline, timestamp, value
def test_group_aggregate_all_forward(dtstart, delta, N, step): """Aggregate all data and check result""" nseries = 10 begin = dtstart end = dtstart + delta*(N + 1) agg_funcs = ["min", "max", "count", "sum"] query = att.make_group_aggregate_query("test", begin, end, agg_funcs, step, output=dict(format='csv'), where={"tag3": "D", "tag2": "C"}) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) iterations = 0 for line in response: try: columns = line.split(',') timestamp = att.parse_timestamp(columns[1].strip()) tserrormsg = "Unexpected timestamp value: {0}".format(columns[1].strip()) if timestamp.second != dtstart.second: raise ValueError(tserrormsg) if timestamp.microsecond != dtstart.microsecond: raise ValueError(tserrormsg) iterations += 1 except: print("Error at line: {0}".format(line)) raise if iterations == 0: raise ValueError("Results incomplete")
def test_aggregate_all(dtstart, delta, N): """Aggregate all data and check result""" begin = dtstart + delta * (N - 1) end = dtstart - delta query = att.make_aggregate_query("test", begin, end, "sum", output=dict(format='csv')) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) expected_tags = [ "tag3=D", "tag3=E", "tag3=F", "tag3=G", "tag3=H", ] M = N / 10 expected_values = [ 5 * M**2 - 5 * M, 5 * M**2 - 4 * M, 5 * M**2 - 3 * M, 5 * M**2 - 2 * M, 5 * M**2 - M, 5 * M**2, 5 * M**2 + M, 5 * M**2 + 2 * M, 5 * M**2 + 3 * M, 5 * M**2 + 4 * M, 5 * M**2 + 5 * M, ] iterations = 0 for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tag = expected_tags[iterations % len(expected_tags)] exp_val = expected_values[iterations % len(expected_values)] if abs(value - exp_val) > 10E-5: msg = "Invalid value, expected: {0}, actual: {1}".format( exp_val, value) print(msg) raise ValueError(msg) if tagline.endswith(exp_tag) == False: msg = "Unexpected tag value: {0}, expected: {1}".format( tagline, exp_tag) raise ValueError(msg) iterations += 1 except: print("Error at line: {0}".format(line)) raise if iterations != len(expected_tags) * 2: raise ValueError("Results incomplete")
def test_group_aggregate_join_backward(dtstart, delta, N, step, agg_func): """Aggregate all data and check result""" begin = dtstart + delta * N end = dtstart - delta metrics = ["cpu.user", "cpu.syst"] query = att.make_group_aggregate_join_query(metrics, agg_func, begin, end, step, output={"format": "csv"}, where={ "tag3": "D", "tag2": "C" }, apply=[{ "name": "eval", "expr": "cpu.user - cpu.syst" }]) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) iterations = 0 exptimestamp = begin for line in response: try: columns = line.split(',') if len(columns) != 3: raise ValueError("Unexpected number of columns in the output") sname = columns[0] if not sname.startswith("|".join(metrics)): raise ValueError("Unexpected series name {0}".format( columns[0])) timestamp = att.parse_timestamp(columns[1].strip()) if timestamp != exptimestamp: tserrormsg = "Actual timestamp value: {0}\nExpected timestamp value {1}".format( columns[1].strip(), exptimestamp) raise ValueError(tserrormsg) exptimestamp -= delta # Check that all three values are the same zero = int(columns[2]) if zero != 0: raise ValueError("Unexpected value {0}".format(zero)) iterations += 1 except: print("Error at line: {0}".format(line)) raise if iterations != N: raise ValueError( "Invalid number of result {0} expected {1} received".format( N, iterations))
def reader(dtstart, delta, N): # Start writer process wproc = multiprocessing.Process(name='Writer', target=writer, args=[dtstart, delta, N]) wproc.start() try: window = att.get_window_width() end = dtstart + delta * (N - 1) - 2 * window begin = dtstart timedelta = end - begin points_required = int( math.ceil( (timedelta.seconds * 1000000.0 + timedelta.microseconds) / (delta.seconds * 1000000.0 + delta.microseconds))) + 1 query_params = {"output": {"format": "csv"}} query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = 0 iterations = 0 print("Test #1 - continuous queries") for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = 'test tag=Foo' att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value * 1.0, value, iterations) exp_ts += delta exp_value += 1 iterations += 1 except: print("Error at line: {0}".format(line)) raise print("Query completed") # Check that we received all values if iterations != points_required: raise ValueError( "Expect {0} data points, get {1} data points".format( points_required, iterations)) print("Test #1 passed") finally: wproc.join()
def test_group_aggregate_join_forward(dtstart, delta, N, step, agg_func): """Aggregate all data and check result""" begin = dtstart end = dtstart + delta * (N + 1) metrics = ["cpu.user", "cpu.system", "cpu.idle"] query = att.make_group_aggregate_join_query(metrics, agg_func, begin, end, step, output={"format": "csv"}, where={ "tag3": "D", "tag2": "C" }) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) iterations = 0 exptimestamp = begin for line in response: try: columns = line.split(',') if len(columns) != 5: raise ValueError("Unexpected number of columns in the output") sname = columns[0] if not sname.startswith("|".join(metrics)): raise ValueError("Unexpected series name {0}".format( columns[0])) timestamp = att.parse_timestamp(columns[1].strip()) if timestamp != exptimestamp: tserrormsg = "Actual timestamp value: {0}\nExpected timestamp value {1}".format( columns[1].strip(), exptimestamp) raise ValueError(tserrormsg) exptimestamp += delta # Check that all three values are the same user, syst, idle = tuple([float(it) for it in columns[2:]]) if user != syst or syst != idle: raise ValueError("Unexpected value {0} {1} {2}".format( user, syst, idle)) iterations += 1 except: print("Error at line: {0}".format(line)) raise if iterations != N: raise ValueError( "Invalid number of result {0} expected {1} received".format( N, iterations))
def test_read_in_forward_direction(dtstart, delta, N): """Read data in forward direction""" window = att.get_window_width() end = dtstart + delta * (N - 1) - window begin = dtstart timedelta = end - begin points_required = int( math.ceil((timedelta.seconds * 1000000.0 + timedelta.microseconds) / (delta.seconds * 1000000.0 + delta.microseconds))) + 1 # We need to add 1 because query will include both begin and end timestamps. query_params = { "output": { "format": "csv" }, } query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = 0 iterations = 0 print("Test #6 - filter by tag") expected_tags = [ "tag3=D", "tag3=E", "tag3=F", "tag3=G", "tag3=H", ] for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = expected_tags[(iterations) % len(expected_tags)] att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value * 1.0, value, iterations) exp_ts += delta exp_value += 1 iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != points_required: raise ValueError("Expect {0} data points, get {1} data points".format( points_required, iterations)) print("Test #6 passed")
def line2tup(seq): for ix, line in enumerate(seq): try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) yield tagline, timestamp, value except: print("Error in line: {0}".format(ix)) print(line) raise
def line2tup(seq): for ix, line in enumerate(seq): try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) yield tagline, timestamp, value except: print("Error in line: {0}".format(ix)) print(line) raise
def test_where_clause_with_groupby_in_backward_direction(dtstart, delta, N): """Filter data by tag and group by another tag""" begin = dtstart + delta * (N - 1) end = dtstart query_params = { "output": { "format": "csv" }, "group-by": { "tag": "tag3" }, "where": { "tag2": ["C"], # read only odd } } query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = N - 1 iterations = 0 print("Test #4 - where + group-by") expected_tags = [ "test tag3=D", "test tag3=E", "test tag3=F", "test tag3=G", "test tag3=H", ] for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = expected_tags[(N - iterations - 1) % len(expected_tags)] att.check_values(exp_tags, tagline, 'EQ', exp_ts, timestamp, exp_value * 1.0, value, iterations) exp_ts -= 2 * delta exp_value -= 2 iterations += 2 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != N: raise ValueError("Expect {0} data points, get {1} data points".format( N, iterations)) print("Test #4 passed")
def test_group_by_tag_in_backward_direction(dtstart, delta, N): """Read all data in backward direction. All data should be received as expected.""" begin = dtstart + delta * (N - 1) end = dtstart query_params = { "output": { "format": "csv" }, "group-by": { "tag": "tag3" }, } query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = N - 1 iterations = 0 print("Test #2 - group by tag in backward direction") expected_tags = [ "test tag3=D", "test tag3=E", "test tag3=F", "test tag3=G", "test tag3=H", ] for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = expected_tags[(N - iterations - 1) % len(expected_tags)] att.check_values(exp_tags, tagline, 'EQ', exp_ts, timestamp, exp_value * 1.0, value, iterations) exp_ts -= delta exp_value -= 1 iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != N: raise ValueError("Expect {0} data points, get {1} data points".format( N, iterations)) print("Test #2 passed")
def test_aggregate_all(dtstart, delta, N): """Aggregate all data and check result""" begin = dtstart + delta*(N-1) end = dtstart - delta query = att.make_aggregate_query("test", begin, end, "sum", output=dict(format='csv')) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) expected_tags = [ "tag3=D", "tag3=E", "tag3=F", "tag3=G", "tag3=H", ] M = N/10 expected_values = [ 5*M**2 - 5*M, 5*M**2 - 4*M, 5*M**2 - 3*M, 5*M**2 - 2*M, 5*M**2 - M, 5*M**2, 5*M**2 + M, 5*M**2 + 2*M, 5*M**2 + 3*M, 5*M**2 + 4*M, 5*M**2 + 5*M, ] iterations = 0 for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tag = expected_tags[iterations % len(expected_tags)] exp_val = expected_values[iterations % len(expected_values)] if abs(value - exp_val) > 10E-5: msg = "Invalid value, expected: {0}, actual: {1}".format(exp_val, value) print(msg) raise ValueError(msg) if tagline.endswith(exp_tag) == False: msg = "Unexpected tag value: {0}, expected: {1}".format(tagline, exp_tag) raise ValueError(msg) iterations += 1 except: print("Error at line: {0}".format(line)) raise if iterations != len(expected_tags)*2: raise ValueError("Results incomplete")
def test_aggregate_last_timestamp(dtstart, delta, N): """Aggregate all data and check result""" begin = dtstart end = dtstart + delta*N query = att.make_aggregate_query("test", begin, end, "last_timestamp", output=dict(format='csv')) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) expected_tags = [ "tag3=D", "tag3=E", "tag3=F", "tag3=G", "tag3=H", ] expected_timestamps = [ dtstart + (N-10)*delta, dtstart + (N-9)*delta, dtstart + (N-8)*delta, dtstart + (N-7)*delta, dtstart + (N-6)*delta, dtstart + (N-5)*delta, dtstart + (N-4)*delta, dtstart + (N-3)*delta, dtstart + (N-2)*delta, dtstart + (N-1)*delta, ] iterations = 0 for line in response: try: columns = line.split(',') if len(columns) != 2: msg = "Invalid reply format, 2 columns expected, actual: {0}".format(len(columns)) raise ValueError(msg) tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) exp_ts = expected_timestamps[iterations % len(expected_timestamps)] exp_tag = expected_tags[iterations % len(expected_tags)] if tagline.endswith(exp_tag) == False: msg = "Unexpected tag value: {0}, expected: {1}".format(tagline, exp_tag) raise ValueError(msg) if timestamp != exp_ts: msg = "Unexpected timestamp: {0}, expected: {1}".format(timestamp, exp_ts) raise ValueError(msg) iterations += 1 except: print("Error at line: {0}".format(line)) raise if iterations != len(expected_tags)*2: raise ValueError("Results incomplete")
def reader(dtstart, delta, N): # Start writer process wproc = multiprocessing.Process(name='Writer', target=writer, args=[dtstart, delta, N]) wproc.start() try: window = att.get_window_width() end = dtstart + delta*(N-1) - 2*window begin = dtstart timedelta = end - begin points_required = int(math.ceil((timedelta.seconds*1000000.0 + timedelta.microseconds) / (delta.seconds*1000000.0 + delta.microseconds))) + 1 query_params = {"output": { "format": "csv" }} query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = 0 iterations = 0 print("Test #1 - continuous queries") for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = 'test tag=Foo' att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value*1.0, value, iterations) exp_ts += delta exp_value += 1 iterations += 1 except: print("Error at line: {0}".format(line)) raise print("Query completed") # Check that we received all values if iterations != points_required: raise ValueError("Expect {0} data points, get {1} data points".format(points_required, iterations)) print("Test #1 passed") finally: wproc.join()
def test_read_in_forward_direction(dtstart, delta, N): """Read data in forward direction""" window = att.get_window_width() end = dtstart + delta*(N-1) - window begin = dtstart timedelta = end - begin points_required = int(math.ceil((timedelta.seconds*1000000.0 + timedelta.microseconds) / (delta.seconds*1000000.0 + delta.microseconds))) + 1 # We need to add 1 because query will include both begin and end timestamps. query_params = { "output": { "format": "csv" }, } query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = 0 iterations = 0 print("Test #6 - filter by tag") expected_tags = [ "tag3=D", "tag3=E", "tag3=F", "tag3=G", "tag3=H", ] for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = expected_tags[(iterations) % len(expected_tags)] att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value*1.0, value, iterations) exp_ts += delta exp_value += 1 iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != points_required: raise ValueError("Expect {0} data points, get {1} data points".format(points_required, iterations)) print("Test #6 passed")
def test_join_query_backward(columns, dtstart, delta, N): """Read data in forward direction""" begin = dtstart + delta * (N - 1) end = dtstart - delta timedelta = begin - end query_params = { "output": { "format": "csv" }, } query = att.make_join_query(columns, begin, end, **query_params) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = N - 1 iterations = 0 expected_tags = [ "tag2=B", "tag2=C", "tag2=D", ] print("Test #2 - read forward, order by time") for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) values = [float(it.strip()) for it in columns[2:]] exp_tags = expected_tags[(N - iterations - 1) % len(expected_tags)] for value in values: att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value * 1.0, value, iterations) exp_ts -= delta exp_value -= 1 iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != N: raise ValueError("Expect {0} data points, get {1} data points".format( points_required, iterations)) print("Test #2 - passed")
def test_read_in_forward_direction(dtstart, delta, N): """Read data in forward direction""" begin = dtstart end = dtstart + delta * (N + 1) timedelta = end - begin query_params = { "output": { "format": "csv" }, } query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = 0 iterations = 0 expected_tags = [ "tag3=D", "tag3=E", "tag3=F", "tag3=G", "tag3=H", ] for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = expected_tags[(iterations) % len(expected_tags)] att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value * 1.0, value, iterations) exp_ts += delta exp_value += 1 iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != N: raise ValueError("Expect {0} data points, get {1} data points".format( points_required, iterations))
def test_where_clause_with_groupby_in_backward_direction(dtstart, delta, N): """Filter data by tag and group by another tag""" begin = dtstart + delta*(N-1) end = dtstart query_params = { "output": { "format": "csv" }, "group-by": { "tag": "tag3" }, "where": { "tag2": ["C"], # read only odd } } query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = N-1 iterations = 0 print("Test #4 - where + group-by") expected_tags = [ "test tag3=D", "test tag3=E", "test tag3=F", "test tag3=G", "test tag3=H", ] for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = expected_tags[(N - iterations - 1) % len(expected_tags)] att.check_values(exp_tags, tagline, 'EQ', exp_ts, timestamp, exp_value*1.0, value, iterations) exp_ts -= 2*delta exp_value -= 2 iterations += 2 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != N: raise ValueError("Expect {0} data points, get {1} data points".format(N, iterations)) print("Test #4 passed")
def test_group_by_tag_in_backward_direction(dtstart, delta, N): """Read all data in backward direction. All data should be received as expected.""" begin = dtstart + delta*(N-1) end = dtstart query_params = { "output": { "format": "csv" }, "group-by": { "tag": "tag3" }, } query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = N-1 iterations = 0 print("Test #2 - group by tag in backward direction") expected_tags = [ "test tag3=D", "test tag3=E", "test tag3=F", "test tag3=G", "test tag3=H", ] for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = expected_tags[(N-iterations-1) % len(expected_tags)] att.check_values(exp_tags, tagline, 'EQ', exp_ts, timestamp, exp_value*1.0, value, iterations) exp_ts -= delta exp_value -= 1 iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != N: raise ValueError("Expect {0} data points, get {1} data points".format(N, iterations)) print("Test #2 passed")
def test_select_events_backward(dtstart, delta, N): """Read events in backward direction""" nseries = 10 end = dtstart - delta begin = dtstart + delta * (N + 1) query = { "select-events": "!foo", "range": { "from": begin.strftime('%Y%m%dT%H%M%S.%f'), "to": end.strftime('%Y%m%dT%H%M%S.%f'), }, "order-by": "time", "output": { "format": "csv" } } queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) iterations = 0 for line in response: try: expts, expname = allevents[-(iterations + 1)] columns = line.split(',') timestamp = att.parse_timestamp(columns[1].strip()) event = columns[2].lstrip().rstrip('\n') if timestamp != expts: print("Unexpected timestamp in line {0}".format(line)) raise ValueError("Wrong timestamp {0}, expected {1}".format( str(timestamp), str(expts))) if expname != event: print("Unexpected value in line {0}".format(line)) raise ValueError("Wrong value {0}, expected {1}".format( event, expname)) iterations += 1 except: print("Error at line: {0}".format(line)) raise if iterations != len(allevents): raise ValueError( "Results incomplete, {0} received, {1} expected".format( iterations, len(allevents)))
def test_join_query_backward(columns, dtstart, delta, N): """Read data in forward direction""" begin = dtstart + delta*(N - 1) end = dtstart - delta timedelta = begin - end query_params = { "output": { "format": "csv" }, } query = att.make_join_query(columns, begin, end, **query_params) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = N - 1 iterations = 0 expected_tags = [ "tag2=B", "tag2=C", "tag2=D", ] print("Test #2 - read forward, order by time") for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) values = [float(it.strip()) for it in columns[2:]] exp_tags = expected_tags[(N - iterations - 1) % len(expected_tags)] for value in values: att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value*1.0, value, iterations) exp_ts -= delta exp_value -= 1 iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != N: raise ValueError("Expect {0} data points, get {1} data points".format(points_required, iterations)) print("Test #2 - passed")
def test_read_all(dtstart, delta, N): """Read all data in backward direction. All data should be received as expected.""" begin = dtstart + delta * N end = dtstart - delta query = att.makequery("test", begin, end, output=dict(format='csv')) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) expected_tags = [ "tag3=D", "tag3=E", "tag3=F", "tag3=G", "tag3=H", ] exp_ts = dtstart + delta * (N - 1) exp_value = N - 1 iterations = 0 print("Test - read all data in backward direction") for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = expected_tags[(N - iterations - 1) % len(expected_tags)] att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value * 1.0, value, iterations) exp_ts -= delta exp_value -= 1 iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != N: raise ValueError("Expect {0} data points, get {1} data points".format( N, iterations)) print("Test passed")
def test_read_in_forward_direction(dtstart, delta, N): """Read data in forward direction""" begin = dtstart end = dtstart + delta*(N + 1) timedelta = end - begin query_params = { "output": { "format": "csv" }, } query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = 0 iterations = 0 expected_tags = [ "tag3=D", "tag3=E", "tag3=F", "tag3=G", "tag3=H", ] for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = expected_tags[(iterations) % len(expected_tags)] att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value*1.0, value, iterations) exp_ts += delta exp_value += 1 iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != N: raise ValueError("Expect {0} data points, get {1} data points".format(points_required, iterations))
def group_aggregate_query(metric, begin, end, agg_funcs, step, **extra_args): '''Query database, return series of tuples''' query = att.make_group_aggregate_query(metric, begin, end, agg_funcs, "{0}ms".format(step), output=dict(format='csv'), **extra_args) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) tuple_len = len(agg_funcs) for line in response: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) output = {"seris": tagline, "timestamp": timestamp} for i in range(0, tuple_len): value = float(columns[2 + i].strip()) output[agg_funcs[i]] = value yield output
def test_read_all(dtstart, delta, N): """Read all data in backward direction. All data should be received as expected.""" begin = dtstart + delta*N end = dtstart - delta query = att.makequery("test", begin, end, output=dict(format='csv')) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) expected_tags = [ "tag3=D", "tag3=E", "tag3=F", "tag3=G", "tag3=H", ] exp_ts = dtstart + delta*(N-1) exp_value = N-1 iterations = 0 print("Test - read all data in backward direction") for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = expected_tags[(N-iterations-1) % len(expected_tags)] att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value*1.0, value, iterations) exp_ts -= delta exp_value -= 1 iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != N: raise ValueError("Expect {0} data points, get {1} data points".format(N, iterations)) print("Test passed")
def test_read_all(dtstart, delta, N): """Read all data in backward direction. All data should be received as expected.""" begin = dtstart + delta * (N - 1) end = dtstart query = att.makequery("test", begin, end, output=dict(format='csv')) queryurl = "http://{0}:{1}".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) expected_tags = [ "tag3=D", "tag3=E", "tag3=F", "tag3=G", "tag3=H", ] exp_ts = None exp_value = N - 1 iterations = 0 print("Test - read all data in backward direction") for line in response: try: columns = line.split(',') timestamp = att.parse_timestamp(columns[1].strip()) if exp_ts is None: exp_ts = timestamp if exp_ts and exp_ts != timestamp: raise ValueError("Invalid timestamp at {0}".format(iterations)) exp_ts -= delta iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations == 0: raise ValueError("Unable to read any data") print("Test passed")
def test_read_all(dtstart, delta, N): """Read all data in backward direction. All data should be received as expected.""" begin = dtstart + delta*(N-1) end = dtstart query = att.makequery("test", begin, end, output=dict(format='csv')) queryurl = "http://{0}:{1}".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) expected_tags = [ "tag3=D", "tag3=E", "tag3=F", "tag3=G", "tag3=H", ] exp_ts = None exp_value = N-1 iterations = 0 print("Test - read all data in backward direction") for line in response: try: columns = line.split(',') timestamp = att.parse_timestamp(columns[1].strip()) if exp_ts is None: exp_ts = timestamp if exp_ts and exp_ts != timestamp: raise ValueError("Invalid timestamp at {0}".format(iterations)) exp_ts -= delta iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations == 0: raise ValueError("Unable to read any data") print("Test passed")
def test_read_all(exp_tags, dtstart, delta, N): """Read all series one by one in backward direction. All data should be received as expected.""" for tags in exp_tags: begin = dtstart + delta*(N-1) end = dtstart query_params = { "output": { "format": "csv" }, "where": tags } query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = None print("Test - read all data in backward direction") prev_line = '' iterations = 0 for line in response: try: columns = line.split(',') timestamp = att.parse_timestamp(columns[1].strip()) if exp_ts is None: exp_ts = timestamp if exp_ts and exp_ts != timestamp: raise ValueError("Invalid timestamp at {0}, expected {1}, actual {2}".format(iterations, exp_ts, timestamp)) exp_ts -= delta iterations += 1 prev_line = line except ValueError as err: print(err) raise # Check that we received all values if iterations == 0: raise ValueError("Unable to read any data") print("Test passed")
def test_aggregate_all_group_by(dtstart, delta, N): """Aggregate all data and check result""" begin = dtstart + delta * (N - 1) end = dtstart - delta query_params = { "output": { "format": "csv" }, "group-by": ["tag1"], } query = att.make_aggregate_query("test", begin, end, "sum", **query_params) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) # All values will be collapsed into one! expected_tags = [ "tag1=A", ] expected_values = [0.5 * (N**2 - N)] iterations = 0 for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) if abs(value - expected_values[0]) > 10E-5: msg = "Invalid value, expected: {0}, actual: {1}".format( expected_values[0], value) print(msg) raise ValueError(msg) if tagline.endswith(expected_tags[0]) == False: msg = "Unexpected tag value: {0}, expected: {1}".format( tagline, expected_tags[0]) raise ValueError(msg) iterations += 1 except: print("Error at line: {0}".format(line)) raise if iterations != 1: raise ValueError("Bad number of results")
def test_read_all(exp_tags, dtstart, delta, N): """Read all series one by one in backward direction. All data should be received as expected.""" for tags in exp_tags: begin = dtstart + delta * (N - 1) end = dtstart query_params = {"output": {"format": "csv"}, "where": tags} query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = None print("Test - read all data in backward direction") prev_line = '' iterations = 0 for line in response: try: columns = line.split(',') timestamp = att.parse_timestamp(columns[1].strip()) if exp_ts is None: exp_ts = timestamp if exp_ts and exp_ts != timestamp: raise ValueError( "Invalid timestamp at {0}, expected {1}, actual {2}". format(iterations, exp_ts, timestamp)) exp_ts -= delta iterations += 1 prev_line = line except ValueError as err: print(err) raise # Check that we received all values if iterations == 0: raise ValueError("Unable to read any data") print("Test passed")
def group_aggregate_query(metric, begin, end, agg_funcs, step, **extra_args): '''Query database, return series of tuples''' query = att.make_group_aggregate_query(metric, begin, end, agg_funcs, "{0}ms".format(step), output=dict(format='csv'), **extra_args) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) tuple_len = len(agg_funcs) for line in response: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) output = {"seris": tagline, "timestamp": timestamp} for i in range(0, tuple_len): value = float(columns[2 + i].strip()) output[agg_funcs[i]] = value yield output
def test_aggregate_all_group_by(dtstart, delta, N): """Aggregate all data and check result""" begin = dtstart + delta*(N-1) end = dtstart - delta query_params = { "output": { "format": "csv" }, "group-by": [ "tag1" ], } query = att.make_aggregate_query("test", begin, end, "sum", **query_params) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) # All values will be collapsed into one! expected_tags = [ "tag1=A", ] expected_values = [ 0.5*(N**2 - N) ] iterations = 0 for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) if abs(value - expected_values[0]) > 10E-5: msg = "Invalid value, expected: {0}, actual: {1}".format(expected_values[0], value) print(msg) raise ValueError(msg) if tagline.endswith(expected_tags[0]) == False: msg = "Unexpected tag value: {0}, expected: {1}".format(tagline, expected_tags[0]) raise ValueError(msg) iterations += 1 except: print("Error at line: {0}".format(line)) raise if iterations != 1: raise ValueError("Bad number of results")
def test_group_aggregate_all_backward(dtstart, delta, N, nsteps): """Aggregate all data and check result""" nseries = 10 begin = dtstart + delta*(N-1) end = dtstart - delta step = int((delta * N * 1000).total_seconds() / nsteps) agg_funcs = ["min", "max", "count", "sum"] query = att.make_group_aggregate_query("test", begin, end, agg_funcs, "{0}ms".format(step), output=dict(format='csv')) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) expected_tags = [ "tag3=D", "tag3=E", "tag3=F", "tag3=G", "tag3=H", ] registerd_values = {} iterations = 0 for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) min_value = float(columns[2].strip()) max_value = float(columns[3].strip()) cnt_value = float(columns[4].strip()) sum_value = float(columns[4].strip()) max_index = len(expected_tags) - 1 exp_tag = expected_tags[max_index - (iterations % len(expected_tags))] if tagline.endswith(exp_tag) == False: msg = "Unexpected tag value: {0}, expected: {1}".format(tagline, exp_tag) raise ValueError(msg) cnt_expected = N/nsteps/nseries if cnt_expected == 0: # expected count is less then 1 but not 0 # there is more than 1 step per value in raw series cnt_expected = 1 if cnt_value != cnt_expected: msg = "Invalid cnt value, expected: {0}, actual: {1}".format(cnt_expected, cnt_value) raise ValueError(msg) prev_val = registerd_values.get(tagline) if prev_val is not None: if abs(prev_val['min'] - max_value) - nseries > 10E-5: msg = "Invalid value, expected: {0}, actual: {1}".format(prev_val['min'], max_value) raise ValueError(msg) new_val = dict(max=max_value, min=min_value, cnt=cnt_value, sum=sum_value) registerd_values[tagline] = new_val iterations += 1 except: print("Error at line: {0}".format(line)) raise if iterations == 0: raise ValueError("Results incomplete")
def test_paa_in_backward_direction(testname, dtstart, delta, N, fn, query): expected_values = [ reversed(range(9, 100000, 10)), reversed(range(8, 100000, 10)), reversed(range(7, 100000, 10)), reversed(range(6, 100000, 10)), reversed(range(5, 100000, 10)), reversed(range(4, 100000, 10)), reversed(range(3, 100000, 10)), reversed(range(2, 100000, 10)), reversed(range(1, 100000, 10)), reversed(range(0, 100000, 10)), ] def sliding_window(values, winlen, func): top = [0] * winlen for ix, it in enumerate(values): k = ix % winlen top[k] = it if (ix + 1) % winlen == 0: yield func(top) def round_robin(sequences, maxlen): l = len(sequences) for i in xrange(0, maxlen): seq = sequences[i % l] it = seq.next() yield it begin = dtstart + delta * N end = dtstart query_params = { "sample": [{ "name": query }], "output": { "format": "csv" }, "group-by": { "time": "1s" }, } query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin iterations = 0 print(testname) expected_tags = [ "tag3=H", "tag3=G", "tag3=F", "tag3=E", "tag3=D", ] sequences = [sliding_window(it, 100, fn) for it in expected_values] exp_values = round_robin(sequences, N) for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = expected_tags[iterations % len(expected_tags)] exp_value = exp_values.next() if timestamp != exp_ts: raise ValueError("Expected {0}, actual {1}".format( exp_ts, timestamp)) if value != exp_value: raise ValueError("Expected {0}, actual {1}".format( exp_value, value)) if not tagline.endswith(exp_tags): raise ValueError("Expected {0}, actual {1}".format( exp_tags, tagline)) if (iterations + 1) % 10 == 0: exp_ts -= datetime.timedelta(seconds=1) iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != 990: raise ValueError("Expect {0} data points, get {1} data points".format( 990, iterations)) print("{0} passed".format(testname[:testname.index(" - ")]))
def test_paa_in_backward_direction(dtstart, delta, N, fn, query): expected_values = [ reversed(range(9, 100000, 10)), reversed(range(8, 100000, 10)), reversed(range(7, 100000, 10)), reversed(range(6, 100000, 10)), reversed(range(5, 100000, 10)), reversed(range(4, 100000, 10)), reversed(range(3, 100000, 10)), reversed(range(2, 100000, 10)), reversed(range(1, 100000, 10)), reversed(range(0, 100000, 10)), ] def sliding_window(values, winlen, func): top = [0]*winlen for ix, it in enumerate(values): k = ix % winlen top[k] = it if (ix + 1) % winlen == 0: yield func(top) def round_robin(sequences, maxlen): l = len(sequences) for i in xrange(0, maxlen): seq = sequences[i % l] it = seq.next() yield it begin = dtstart + delta*N end = dtstart query_params = { "sample": [{ "name": query }], "output": { "format": "csv" }, "group-by":{ "time": "1s" }, } query = att.makequery("test", begin, end, **query_params) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin iterations = 0 expected_tags = [ "tag3=H", "tag3=G", "tag3=F", "tag3=E", "tag3=D", ] sequences = [sliding_window(it, 100, fn) for it in expected_values] exp_values = round_robin(sequences, N) for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) value = float(columns[2].strip()) exp_tags = expected_tags[iterations % len(expected_tags)] exp_value = exp_values.next() if timestamp != exp_ts: raise ValueError("Expected {0}, actual {1}".format(exp_ts, timestamp)) if value != exp_value: raise ValueError("Expected {0}, actual {1}".format(exp_value, value)) if not tagline.endswith(exp_tags): raise ValueError("Expected {0}, actual {1}".format(exp_tags, tagline)) if (iterations + 1) % 10 == 0: exp_ts -= datetime.timedelta(seconds=1) iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != 990: raise ValueError("Expect {0} data points, get {1} data points".format(990, iterations))
def test_join_query_backward_by_series(columns, dtstart, delta, N): """Read data in forward direction""" begin = dtstart + delta * (N - 1) end = dtstart - delta timedelta = begin - end query_params = {"output": {"format": "csv"}, "order-by": "series"} query = att.make_join_query(columns, begin, end, **query_params) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = N - 1 iterations = 0 expected_tags = [ "tag2=B", "tag2=C", "tag2=D", ] bsize = count_elements("col1", "tag2", "B", begin, end) csize = count_elements("col1", "tag2", "C", begin, end) dsize = count_elements("col1", "tag2", "D", begin, end) sizes = [ bsize, csize, dsize, ] steps = [ bsize, bsize + csize, bsize + csize + dsize, ] nseries = len(expected_tags) print("Test #4 - read forward, order by series") prev_tag = None reset_ix = 0 for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) values = [float(it.strip()) for it in columns[2:]] tagix = 0 while iterations >= steps[tagix]: tagix += 1 exp_tags = expected_tags[tagix] if prev_tag != tagline: exp_ts = dtstart + reset_ix * delta + delta * (sizes[tagix] - 1) * nseries exp_value = reset_ix + (sizes[tagix] - 1) * nseries prev_tag = tagline reset_ix += 1 for value in values: att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value * 1.0, value, iterations) exp_ts -= nseries * delta exp_value -= nseries iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != N: raise ValueError("Expect {0} data points, get {1} data points".format( points_required, iterations)) print("Test #4 - passed")
def test_join_query_backward_by_series(columns, dtstart, delta, N): """Read data in forward direction""" begin = dtstart + delta*(N - 1) end = dtstart - delta timedelta = begin - end query_params = { "output": { "format": "csv" }, "order-by": "series" } query = att.make_join_query(columns, begin, end, **query_params) queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT) response = urlopen(queryurl, json.dumps(query)) exp_ts = begin exp_value = N-1 iterations = 0 expected_tags = [ "tag2=B", "tag2=C", "tag2=D", ] bsize = count_elements("col1", "tag2", "B", begin, end) csize = count_elements("col1", "tag2", "C", begin, end) dsize = count_elements("col1", "tag2", "D", begin, end) sizes = [ bsize, csize, dsize, ] steps = [ bsize, bsize + csize, bsize + csize + dsize, ] nseries = len(expected_tags) print("Test #4 - read forward, order by series") prev_tag = None reset_ix = 0 for line in response: try: columns = line.split(',') tagline = columns[0].strip() timestamp = att.parse_timestamp(columns[1].strip()) values = [float(it.strip()) for it in columns[2:]] tagix = 0 while iterations >= steps[tagix]: tagix += 1 exp_tags = expected_tags[tagix] if prev_tag != tagline: exp_ts = dtstart + reset_ix*delta + delta*(sizes[tagix]-1)*nseries exp_value = reset_ix + (sizes[tagix]-1)*nseries prev_tag = tagline reset_ix += 1 for value in values: att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value*1.0, value, iterations) exp_ts -= nseries*delta exp_value -= nseries iterations += 1 except: print("Error at line: {0}".format(line)) raise # Check that we received all values if iterations != N: raise ValueError("Expect {0} data points, get {1} data points".format(points_required, iterations)) print("Test #4 - passed")