def require_empty(metric_name, dtstart, delta, N):
"""Try to read all series one by one in backward direction.
The result should be empty."""
begin = dtstart - delta * N
end = dtstart + delta * N
query_params = {
"output": {
"format": "csv"
},
}
query = att.makequery(metric_name, begin, end, **query_params)
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
print("Test - check that response is empty")
errormsg = False
for line in response:
# There are two variants:
# - the data is fully recovered (which means that the wal is enabled)
# - the query returns '-no data' response (which means that the metadata wasn't saved)
# - the query returns epty response (which means that the metadata was saved)
if not errormsg and line.startswith('-'):
errormsg = True
else:
raise ValueError("Unexpected value")
print("Test passed")
def require_empty(metric_name, dtstart, delta, N):
"""Try to read all series one by one in backward direction.
The result should be empty."""
begin = dtstart - delta*N
end = dtstart + delta*N
query_params = {
"output": { "format": "csv" },
}
query = att.makequery(metric_name, begin, end, **query_params)
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
print("Test - check that response is empty")
errormsg = False
for line in response:
# There are two variants:
# - the data is fully recovered (which means that the wal is enabled)
# - the query returns '-no data' response (which means that the metadata wasn't saved)
# - the query returns epty response (which means that the metadata was saved)
if not errormsg and line.startswith('-'):
errormsg = True
else:
raise ValueError("Unexpected value")
print("Test passed")
def reader(dtstart, delta, N):
# Start writer process
wproc = multiprocessing.Process(name='Writer',
target=writer,
args=[dtstart, delta, N])
wproc.start()
def cmp_tuples(lhs, rhs):
# ignore tags
timedelta = lhs[1] - rhs[1]
if timedelta != delta:
raise ValueError(
"Invalid timestamps, current {0}, previous {1}".format(
lhs[1], rhs[1]))
valdelta = lhs[2] - rhs[2]
if valdelta - 1.0 > 0.000001:
raise ValueError("Invalid value, current {0}, previous {1}".format(
lhs[2], rhs[2]))
try:
print("Test #1")
end = dtstart + delta * N
begin = dtstart
timedelta = end - begin
query_params = {"output": {"format": "csv"}}
http_err_cnt = 0
while True:
try:
query = att.makequery("test", begin, end, **query_params)
print("Query: {0}".format(query))
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
def count_lines(seq):
global processed
for msg in seq:
yield msg
processed += 1
tuples = line2tup(count_lines(response))
first, last = require_continuous(tuples, cmp_tuples)
print("First: {0}".format(
first and first[1].strftime("%Y%m%dT%H%M%S.%f") or "None"))
print("Last : {0}".format(
last and last[1].strftime("%Y%m%dT%H%M%S.%f") or "None"))
if last is not None:
begin = last[1]
if first[1] == (end - delta):
break
except HTTPError as err:
print("HTTP error: {0}".format(err))
http_err_cnt += 1
if http_err_cnt == 10:
raise
print("Test passed")
finally:
print("{0} messages processed".format(processed))
wproc.join()
def test_sax_in_backward_direction(dtstart, delta, N):
begin = dtstart + delta * N
end = dtstart
query_params = {
"sample": [{
"name": "sax",
"alphabet_size": "5",
"window_width": "10"
}],
"output": {
"format": "csv"
},
"group-by": {
"time": "1ms"
},
}
query = att.makequery("test", begin, end, **query_params)
print(query)
queryurl = "http://{0}:{1}".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
iterations = 0
print("Test")
expected_tags = [
"tag3=H",
"tag3=G",
"tag3=F",
"tag3=E",
"tag3=D",
]
exp_value = "aabbccddee"
for line in response:
print(line)
try:
columns = line.split(',')
tagline = columns[0].strip()
value = columns[2].strip()
exp_tags = expected_tags[iterations % len(expected_tags)]
if value != exp_value:
raise ValueError("Expected {0}, actual {1}".format(
exp_value, value))
if not tagline.endswith(exp_tags):
raise ValueError("Expected {0}, actual {1}".format(
exp_tags, tagline))
if (iterations + 1) % 50 == 0:
exp_ts -= datetime.timedelta(seconds=5)
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != N / 10:
raise ValueError("Expect {0} data points, get {1} data points".format(
N / 10, iterations))
print("Test passed")
def reader(dtstart, delta, N):
# Start writer process
wproc = multiprocessing.Process(name='Writer',
target=writer,
args=[dtstart, delta, N])
wproc.start()
try:
window = att.get_window_width()
end = dtstart + delta * (N - 1) - 2 * window
begin = dtstart
timedelta = end - begin
points_required = int(
math.ceil(
(timedelta.seconds * 1000000.0 + timedelta.microseconds) /
(delta.seconds * 1000000.0 + delta.microseconds))) + 1
query_params = {"output": {"format": "csv"}}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = begin
exp_value = 0
iterations = 0
print("Test #1 - continuous queries")
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = 'test tag=Foo'
att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp,
exp_value * 1.0, value, iterations)
exp_ts += delta
exp_value += 1
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
print("Query completed")
# Check that we received all values
if iterations != points_required:
raise ValueError(
"Expect {0} data points, get {1} data points".format(
points_required, iterations))
print("Test #1 passed")
finally:
wproc.join()
def test_read_in_forward_direction(dtstart, delta, N):
"""Read data in forward direction"""
window = att.get_window_width()
end = dtstart + delta * (N - 1) - window
begin = dtstart
timedelta = end - begin
points_required = int(
math.ceil((timedelta.seconds * 1000000.0 + timedelta.microseconds) /
(delta.seconds * 1000000.0 + delta.microseconds))) + 1
# We need to add 1 because query will include both begin and end timestamps.
query_params = {
"output": {
"format": "csv"
},
}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = begin
exp_value = 0
iterations = 0
print("Test #6 - filter by tag")
expected_tags = [
"tag3=D",
"tag3=E",
"tag3=F",
"tag3=G",
"tag3=H",
]
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = expected_tags[(iterations) % len(expected_tags)]
att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp,
exp_value * 1.0, value, iterations)
exp_ts += delta
exp_value += 1
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != points_required:
raise ValueError("Expect {0} data points, get {1} data points".format(
points_required, iterations))
print("Test #6 passed")
def test_where_clause_with_groupby_in_backward_direction(dtstart, delta, N):
"""Filter data by tag and group by another tag"""
begin = dtstart + delta * (N - 1)
end = dtstart
query_params = {
"output": {
"format": "csv"
},
"group-by": {
"tag": "tag3"
},
"where": {
"tag2": ["C"], # read only odd
}
}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = begin
exp_value = N - 1
iterations = 0
print("Test #4 - where + group-by")
expected_tags = [
"test tag3=D",
"test tag3=E",
"test tag3=F",
"test tag3=G",
"test tag3=H",
]
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = expected_tags[(N - iterations - 1) % len(expected_tags)]
att.check_values(exp_tags, tagline, 'EQ', exp_ts, timestamp,
exp_value * 1.0, value, iterations)
exp_ts -= 2 * delta
exp_value -= 2
iterations += 2
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != N:
raise ValueError("Expect {0} data points, get {1} data points".format(
N, iterations))
print("Test #4 passed")
def test_group_by_tag_in_backward_direction(dtstart, delta, N):
"""Read all data in backward direction.
All data should be received as expected."""
begin = dtstart + delta * (N - 1)
end = dtstart
query_params = {
"output": {
"format": "csv"
},
"group-by": {
"tag": "tag3"
},
}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = begin
exp_value = N - 1
iterations = 0
print("Test #2 - group by tag in backward direction")
expected_tags = [
"test tag3=D",
"test tag3=E",
"test tag3=F",
"test tag3=G",
"test tag3=H",
]
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = expected_tags[(N - iterations - 1) % len(expected_tags)]
att.check_values(exp_tags, tagline, 'EQ', exp_ts, timestamp,
exp_value * 1.0, value, iterations)
exp_ts -= delta
exp_value -= 1
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != N:
raise ValueError("Expect {0} data points, get {1} data points".format(
N, iterations))
print("Test #2 passed")
def reader(dtstart, delta, N):
# Start writer process
wproc = multiprocessing.Process(name='Writer', target=writer, args=[dtstart, delta, N])
wproc.start()
def cmp_tuples(lhs, rhs):
# ignore tags
timedelta = lhs[1] - rhs[1]
if timedelta != delta:
raise ValueError("Invalid timestamps, current {0}, previous {1}".format(lhs[1], rhs[1]))
valdelta = lhs[2] - rhs[2]
if valdelta - 1.0 > 0.000001:
raise ValueError("Invalid value, current {0}, previous {1}".format(lhs[2], rhs[2]))
try:
print("Test #1")
end = dtstart + delta*N
begin = dtstart
timedelta = end - begin
query_params = {"output": { "format": "csv" }}
http_err_cnt = 0
while True:
try:
query = att.makequery("test", begin, end, **query_params)
print("Query: {0}".format(query))
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
def count_lines(seq):
global processed
for msg in seq:
yield msg
processed += 1
tuples = line2tup(count_lines(response))
first, last = require_continuous(tuples, cmp_tuples)
print("First: {0}".format(first and first[1].strftime("%Y%m%dT%H%M%S.%f") or "None"))
print("Last : {0}".format( last and last[1].strftime("%Y%m%dT%H%M%S.%f") or "None"))
if last is not None:
begin = last[1]
if first[1] == (end - delta):
break
except HTTPError as err:
print("HTTP error: {0}".format(err))
http_err_cnt += 1
if http_err_cnt == 10:
raise
print("Test passed")
finally:
print("{0} messages processed".format(processed))
wproc.join()
def test_read_in_forward_direction(dtstart, delta, N):
"""Read data in forward direction"""
window = att.get_window_width()
end = dtstart + delta*(N-1) - window
begin = dtstart
timedelta = end - begin
points_required = int(math.ceil((timedelta.seconds*1000000.0 + timedelta.microseconds) / (delta.seconds*1000000.0 + delta.microseconds))) + 1
# We need to add 1 because query will include both begin and end timestamps.
query_params = {
"output": { "format": "csv" },
}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = begin
exp_value = 0
iterations = 0
print("Test #6 - filter by tag")
expected_tags = [
"tag3=D",
"tag3=E",
"tag3=F",
"tag3=G",
"tag3=H",
]
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = expected_tags[(iterations) % len(expected_tags)]
att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value*1.0, value, iterations)
exp_ts += delta
exp_value += 1
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != points_required:
raise ValueError("Expect {0} data points, get {1} data points".format(points_required, iterations))
print("Test #6 passed")
def reader(dtstart, delta, N):
# Start writer process
wproc = multiprocessing.Process(name='Writer', target=writer, args=[dtstart, delta, N])
wproc.start()
try:
window = att.get_window_width()
end = dtstart + delta*(N-1) - 2*window
begin = dtstart
timedelta = end - begin
points_required = int(math.ceil((timedelta.seconds*1000000.0 + timedelta.microseconds) /
(delta.seconds*1000000.0 + delta.microseconds))) + 1
query_params = {"output": { "format": "csv" }}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = begin
exp_value = 0
iterations = 0
print("Test #1 - continuous queries")
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = 'test tag=Foo'
att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value*1.0, value, iterations)
exp_ts += delta
exp_value += 1
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
print("Query completed")
# Check that we received all values
if iterations != points_required:
raise ValueError("Expect {0} data points, get {1} data points".format(points_required, iterations))
print("Test #1 passed")
finally:
wproc.join()
def test_sax_in_backward_direction(dtstart, delta, N):
begin = dtstart + delta*N
end = dtstart
query_params = {
"sample": [{ "name": "sax",
"alphabet_size": "5",
"window_width": "10" }],
"output": { "format": "csv" },
"group-by":{ "time": "1ms" },
}
query = att.makequery("test", begin, end, **query_params)
print(query)
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
iterations = 0
print("Test")
expected_tags = [
"tag3=H",
"tag3=G",
"tag3=F",
"tag3=E",
"tag3=D",
]
exp_value = "aabbccddee"
for line in response:
print(line)
try:
columns = line.split(',')
tagline = columns[0].strip()
value = columns[2].strip()
exp_tags = expected_tags[iterations % len(expected_tags)]
if value != exp_value:
raise ValueError("Expected {0}, actual {1}".format(exp_value, value))
if not tagline.endswith(exp_tags):
raise ValueError("Expected {0}, actual {1}".format(exp_tags, tagline))
if (iterations + 1) % 50 == 0:
exp_ts -= datetime.timedelta(seconds=5)
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != N/10:
raise ValueError("Expect {0} data points, get {1} data points".format(N/10, iterations))
print("Test passed")
def require_empty_response(metric_name):
"""Read all data in backward direction.
No data should be returned, otherwise the error will be generated.
"""
begin = datetime.datetime(year=2100, month=1, day=1)
end = datetime.datetime(year=1970, month=1, day=1)
query = att.makequery(metric_name, begin, end, output=dict(format='csv'))
queryurl = "http://{0}:{1}/api/query".format(host, httpport)
response = urllib.urlopen(queryurl, json.dumps(query))
iterations = 0
print("Test #2 - read evicted")
for line in response:
raise ValueError("Unexpected value returned: " + line)
print("Test passed")
def test_where_clause_with_groupby_in_backward_direction(dtstart, delta, N):
"""Filter data by tag and group by another tag"""
begin = dtstart + delta*(N-1)
end = dtstart
query_params = {
"output": { "format": "csv" },
"group-by": { "tag": "tag3" },
"where": {
"tag2": ["C"], # read only odd
}
}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = begin
exp_value = N-1
iterations = 0
print("Test #4 - where + group-by")
expected_tags = [
"test tag3=D",
"test tag3=E",
"test tag3=F",
"test tag3=G",
"test tag3=H",
]
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = expected_tags[(N - iterations - 1) % len(expected_tags)]
att.check_values(exp_tags, tagline, 'EQ', exp_ts, timestamp, exp_value*1.0, value, iterations)
exp_ts -= 2*delta
exp_value -= 2
iterations += 2
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != N:
raise ValueError("Expect {0} data points, get {1} data points".format(N, iterations))
print("Test #4 passed")
def test_read_in_forward_direction(dtstart, delta, N):
"""Read data in forward direction"""
begin = dtstart
end = dtstart + delta * (N + 1)
timedelta = end - begin
query_params = {
"output": {
"format": "csv"
},
}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = begin
exp_value = 0
iterations = 0
expected_tags = [
"tag3=D",
"tag3=E",
"tag3=F",
"tag3=G",
"tag3=H",
]
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = expected_tags[(iterations) % len(expected_tags)]
att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp,
exp_value * 1.0, value, iterations)
exp_ts += delta
exp_value += 1
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != N:
raise ValueError("Expect {0} data points, get {1} data points".format(
points_required, iterations))
def test_group_by_tag_in_backward_direction(dtstart, delta, N):
"""Read all data in backward direction.
All data should be received as expected."""
begin = dtstart + delta*(N-1)
end = dtstart
query_params = {
"output": { "format": "csv" },
"group-by": { "tag": "tag3" },
}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = begin
exp_value = N-1
iterations = 0
print("Test #2 - group by tag in backward direction")
expected_tags = [
"test tag3=D",
"test tag3=E",
"test tag3=F",
"test tag3=G",
"test tag3=H",
]
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = expected_tags[(N-iterations-1) % len(expected_tags)]
att.check_values(exp_tags, tagline, 'EQ', exp_ts, timestamp, exp_value*1.0, value, iterations)
exp_ts -= delta
exp_value -= 1
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != N:
raise ValueError("Expect {0} data points, get {1} data points".format(N, iterations))
print("Test #2 passed")
def test_read_all(dtstart, delta, N):
"""Read all data in backward direction.
All data should be received as expected."""
begin = dtstart + delta * N
end = dtstart - delta
query = att.makequery("test", begin, end, output=dict(format='csv'))
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
expected_tags = [
"tag3=D",
"tag3=E",
"tag3=F",
"tag3=G",
"tag3=H",
]
exp_ts = dtstart + delta * (N - 1)
exp_value = N - 1
iterations = 0
print("Test - read all data in backward direction")
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = expected_tags[(N - iterations - 1) % len(expected_tags)]
att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp,
exp_value * 1.0, value, iterations)
exp_ts -= delta
exp_value -= 1
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != N:
raise ValueError("Expect {0} data points, get {1} data points".format(
N, iterations))
print("Test passed")
def test_read_in_forward_direction(dtstart, delta, N):
"""Read data in forward direction"""
begin = dtstart
end = dtstart + delta*(N + 1)
timedelta = end - begin
query_params = {
"output": { "format": "csv" },
}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = begin
exp_value = 0
iterations = 0
expected_tags = [
"tag3=D",
"tag3=E",
"tag3=F",
"tag3=G",
"tag3=H",
]
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = expected_tags[(iterations) % len(expected_tags)]
att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value*1.0, value, iterations)
exp_ts += delta
exp_value += 1
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != N:
raise ValueError("Expect {0} data points, get {1} data points".format(points_required, iterations))
def test_read_all(dtstart, delta, N):
"""Read all data in backward direction.
All data should be received as expected."""
begin = dtstart + delta * (N - 1)
end = dtstart
query = att.makequery("test", begin, end, output=dict(format='csv'))
queryurl = "http://{0}:{1}".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
expected_tags = [
"tag3=D",
"tag3=E",
"tag3=F",
"tag3=G",
"tag3=H",
]
exp_ts = None
exp_value = N - 1
iterations = 0
print("Test - read all data in backward direction")
for line in response:
try:
columns = line.split(',')
timestamp = att.parse_timestamp(columns[1].strip())
if exp_ts is None:
exp_ts = timestamp
if exp_ts and exp_ts != timestamp:
raise ValueError("Invalid timestamp at {0}".format(iterations))
exp_ts -= delta
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations == 0:
raise ValueError("Unable to read any data")
print("Test passed")
def test_read_all(dtstart, delta, N):
"""Read all data in backward direction.
All data should be received as expected."""
begin = dtstart + delta*N
end = dtstart - delta
query = att.makequery("test", begin, end, output=dict(format='csv'))
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
expected_tags = [
"tag3=D",
"tag3=E",
"tag3=F",
"tag3=G",
"tag3=H",
]
exp_ts = dtstart + delta*(N-1)
exp_value = N-1
iterations = 0
print("Test - read all data in backward direction")
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = expected_tags[(N-iterations-1) % len(expected_tags)]
att.check_values(exp_tags, tagline, 'ENDS', exp_ts, timestamp, exp_value*1.0, value, iterations)
exp_ts -= delta
exp_value -= 1
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != N:
raise ValueError("Expect {0} data points, get {1} data points".format(N, iterations))
print("Test passed")
def test_read_all(dtstart, delta, N):
"""Read all data in backward direction.
All data should be received as expected."""
begin = dtstart + delta*(N-1)
end = dtstart
query = att.makequery("test", begin, end, output=dict(format='csv'))
queryurl = "http://{0}:{1}".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
expected_tags = [
"tag3=D",
"tag3=E",
"tag3=F",
"tag3=G",
"tag3=H",
]
exp_ts = None
exp_value = N-1
iterations = 0
print("Test - read all data in backward direction")
for line in response:
try:
columns = line.split(',')
timestamp = att.parse_timestamp(columns[1].strip())
if exp_ts is None:
exp_ts = timestamp
if exp_ts and exp_ts != timestamp:
raise ValueError("Invalid timestamp at {0}".format(iterations))
exp_ts -= delta
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations == 0:
raise ValueError("Unable to read any data")
print("Test passed")
def test_read_all(exp_tags, dtstart, delta, N):
"""Read all series one by one in backward direction.
All data should be received as expected."""
for tags in exp_tags:
begin = dtstart + delta*(N-1)
end = dtstart
query_params = {
"output": { "format": "csv" },
"where": tags
}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = None
print("Test - read all data in backward direction")
prev_line = ''
iterations = 0
for line in response:
try:
columns = line.split(',')
timestamp = att.parse_timestamp(columns[1].strip())
if exp_ts is None:
exp_ts = timestamp
if exp_ts and exp_ts != timestamp:
raise ValueError("Invalid timestamp at {0}, expected {1}, actual {2}".format(iterations, exp_ts, timestamp))
exp_ts -= delta
iterations += 1
prev_line = line
except ValueError as err:
print(err)
raise
# Check that we received all values
if iterations == 0:
raise ValueError("Unable to read any data")
print("Test passed")
def test_read_all(exp_tags, dtstart, delta, N):
"""Read all series one by one in backward direction.
All data should be received as expected."""
for tags in exp_tags:
begin = dtstart + delta * (N - 1)
end = dtstart
query_params = {"output": {"format": "csv"}, "where": tags}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = None
print("Test - read all data in backward direction")
prev_line = ''
iterations = 0
for line in response:
try:
columns = line.split(',')
timestamp = att.parse_timestamp(columns[1].strip())
if exp_ts is None:
exp_ts = timestamp
if exp_ts and exp_ts != timestamp:
raise ValueError(
"Invalid timestamp at {0}, expected {1}, actual {2}".
format(iterations, exp_ts, timestamp))
exp_ts -= delta
iterations += 1
prev_line = line
except ValueError as err:
print(err)
raise
# Check that we received all values
if iterations == 0:
raise ValueError("Unable to read any data")
print("Test passed")
def read_in_backward_direction(batch_size):
"""Read all data in backward direction.
All data should be received as expected."""
begin = datetime.datetime(year=2100, month=1, day=1)
end = datetime.datetime(year=1970, month=1, day=1)
query = att.makequery("temp", begin, end, output=dict(format='csv'))
queryurl = "http://{0}:{1}".format(host, httpport)
response = urllib.urlopen(queryurl, json.dumps(query))
iterations = 0
print("Test #1 - read all data in backward direction")
pivot = None
exp_value = None
val_count = 0
num_off = 0
failcnt = 0
for line in response:
try:
columns = line.split(',')
#tagline = columns[0].strip()
#timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
if exp_value is None and pivot is None:
pivot = int(float(value))
if pivot is not None and exp_value is None:
if pivot != int(float(value)):
print("Off-elements count: %d" % num_off)
exp_value = int(float(value))
else:
num_off += 1
if exp_value:
val_count += 1
if float(exp_value) != value:
failcnt += 1
print("Unexpected value at {0}, actual {1}, expected {2}".format(iterations, value, exp_value))
exp_value = None
pivot = None
val_count = 0
num_off = 0
if exp_value:
if val_count % batch_size == 0:
exp_value -= 1
if iterations % batch_size == 0:
if iterations % (batch_size*1000) == 0:
print("Read {0}".format(iterations))
iterations += 1
except:
print("Error at line {0}: `{1}`".format(iterations, line))
raise
# Check that we received all values
if iterations == 0:
raise ValueError("Expect {0} data points, get {1} data points".format('--', iterations))
if failcnt != 0:
raise ValueError("Some data was lost")
print("Test passed")
def test_paa_in_backward_direction(dtstart, delta, N, fn, query):
expected_values = [
reversed(range(9, 100000, 10)),
reversed(range(8, 100000, 10)),
reversed(range(7, 100000, 10)),
reversed(range(6, 100000, 10)),
reversed(range(5, 100000, 10)),
reversed(range(4, 100000, 10)),
reversed(range(3, 100000, 10)),
reversed(range(2, 100000, 10)),
reversed(range(1, 100000, 10)),
reversed(range(0, 100000, 10)),
]
def sliding_window(values, winlen, func):
top = [0]*winlen
for ix, it in enumerate(values):
k = ix % winlen
top[k] = it
if (ix + 1) % winlen == 0:
yield func(top)
def round_robin(sequences, maxlen):
l = len(sequences)
for i in xrange(0, maxlen):
seq = sequences[i % l]
it = seq.next()
yield it
begin = dtstart + delta*N
end = dtstart
query_params = {
"sample": [{ "name": query }],
"output": { "format": "csv" },
"group-by":{ "time": "1s" },
}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}/api/query".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = begin
iterations = 0
expected_tags = [
"tag3=H",
"tag3=G",
"tag3=F",
"tag3=E",
"tag3=D",
]
sequences = [sliding_window(it, 100, fn) for it in expected_values]
exp_values = round_robin(sequences, N)
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = expected_tags[iterations % len(expected_tags)]
exp_value = exp_values.next()
if timestamp != exp_ts:
raise ValueError("Expected {0}, actual {1}".format(exp_ts, timestamp))
if value != exp_value:
raise ValueError("Expected {0}, actual {1}".format(exp_value, value))
if not tagline.endswith(exp_tags):
raise ValueError("Expected {0}, actual {1}".format(exp_tags, tagline))
if (iterations + 1) % 10 == 0:
exp_ts -= datetime.timedelta(seconds=1)
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != 990:
raise ValueError("Expect {0} data points, get {1} data points".format(990, iterations))
def read_in_backward_direction(batch_size):
"""Read all data in backward direction.
All data should be received as expected."""
begin = datetime.datetime(year=2100, month=1, day=1)
end = datetime.datetime(year=1970, month=1, day=1)
query = att.makequery("temp", begin, end, output=dict(format='csv'))
queryurl = "http://{0}:{1}/api/query".format(host, httpport)
response = urllib.urlopen(queryurl, json.dumps(query))
iterations = 0
print("Test #1 - read all data in backward direction")
pivot = None
exp_value = None
val_count = 0
num_off = 0
failcnt = 0
for line in response:
try:
columns = line.split(',')
#tagline = columns[0].strip()
#timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
if exp_value is None and pivot is None:
pivot = int(float(value))
if pivot is not None and exp_value is None:
if pivot != int(float(value)):
print("Off-elements count: %d" % num_off)
exp_value = int(float(value))
else:
num_off += 1
if exp_value:
val_count += 1
if float(exp_value) != value:
failcnt += 1
print("Unexpected value at {0}, actual {1}, expected {2}".
format(iterations, value, exp_value))
exp_value = None
pivot = None
val_count = 0
num_off = 0
if exp_value:
if val_count % batch_size == 0:
exp_value -= 1
if iterations % batch_size == 0:
if iterations % (batch_size * 1000) == 0:
print("Read {0}".format(iterations))
iterations += 1
except:
print("Error at line {0}: `{1}`".format(iterations, line))
raise
# Check that we received all values
if iterations == 0:
raise ValueError("Expect {0} data points, get {1} data points".format(
'--', iterations))
if failcnt != 0:
raise ValueError("Some data was lost")
print("Test passed")
def test_paa_in_backward_direction(testname, dtstart, delta, N, fn, query):
expected_values = [
reversed(range(9, 100000, 10)),
reversed(range(8, 100000, 10)),
reversed(range(7, 100000, 10)),
reversed(range(6, 100000, 10)),
reversed(range(5, 100000, 10)),
reversed(range(4, 100000, 10)),
reversed(range(3, 100000, 10)),
reversed(range(2, 100000, 10)),
reversed(range(1, 100000, 10)),
reversed(range(0, 100000, 10)),
]
def sliding_window(values, winlen, func):
top = [0] * winlen
for ix, it in enumerate(values):
k = ix % winlen
top[k] = it
if (ix + 1) % winlen == 0:
yield func(top)
def round_robin(sequences, maxlen):
l = len(sequences)
for i in xrange(0, maxlen):
seq = sequences[i % l]
it = seq.next()
yield it
begin = dtstart + delta * N
end = dtstart
query_params = {
"sample": [{
"name": query
}],
"output": {
"format": "csv"
},
"group-by": {
"time": "1s"
},
}
query = att.makequery("test", begin, end, **query_params)
queryurl = "http://{0}:{1}".format(HOST, HTTPPORT)
response = urlopen(queryurl, json.dumps(query))
exp_ts = begin
iterations = 0
print(testname)
expected_tags = [
"tag3=H",
"tag3=G",
"tag3=F",
"tag3=E",
"tag3=D",
]
sequences = [sliding_window(it, 100, fn) for it in expected_values]
exp_values = round_robin(sequences, N)
for line in response:
try:
columns = line.split(',')
tagline = columns[0].strip()
timestamp = att.parse_timestamp(columns[1].strip())
value = float(columns[2].strip())
exp_tags = expected_tags[iterations % len(expected_tags)]
exp_value = exp_values.next()
if timestamp != exp_ts:
raise ValueError("Expected {0}, actual {1}".format(
exp_ts, timestamp))
if value != exp_value:
raise ValueError("Expected {0}, actual {1}".format(
exp_value, value))
if not tagline.endswith(exp_tags):
raise ValueError("Expected {0}, actual {1}".format(
exp_tags, tagline))
if (iterations + 1) % 10 == 0:
exp_ts -= datetime.timedelta(seconds=1)
iterations += 1
except:
print("Error at line: {0}".format(line))
raise
# Check that we received all values
if iterations != 990:
raise ValueError("Expect {0} data points, get {1} data points".format(
990, iterations))
print("{0} passed".format(testname[:testname.index(" - ")]))
