def test_system_graphite_retentions(graphite_table):
expected = """
graphite_rollup plain \\\\.count$ sum 0 0 1 0 ['test'] ['graphite']
graphite_rollup plain \\\\.max$ max 0 0 2 0 ['test'] ['graphite']
graphite_rollup plain ^five_min\\\\. 31536000 14400 3 0 ['test'] ['graphite']
graphite_rollup plain ^five_min\\\\. 5184000 3600 3 0 ['test'] ['graphite']
graphite_rollup plain ^five_min\\\\. 0 300 3 0 ['test'] ['graphite']
graphite_rollup plain ^one_min avg 31536000 600 4 0 ['test'] ['graphite']
graphite_rollup plain ^one_min avg 7776000 300 4 0 ['test'] ['graphite']
graphite_rollup plain ^one_min avg 0 60 4 0 ['test'] ['graphite']
graphite_rollup tagged [\\\\?&]retention=one_min(&.*)?$ avg 31536000 600 5 0 ['test'] ['graphite']
graphite_rollup tagged [\\\\?&]retention=one_min(&.*)?$ avg 7776000 300 5 0 ['test'] ['graphite']
graphite_rollup tagged [\\\\?&]retention=one_min(&.*)?$ avg 0 60 5 0 ['test'] ['graphite']
graphite_rollup tagged [\\\\?&]retention=five_min(&.*)?$ avg 31536000 14400 6 0 ['test'] ['graphite']
graphite_rollup tagged [\\\\?&]retention=five_min(&.*)?$ avg 5184000 3600 6 0 ['test'] ['graphite']
graphite_rollup tagged [\\\\?&]retention=five_min(&.*)?$ avg 0 300 6 0 ['test'] ['graphite']
graphite_rollup tagged ^for_taggged avg 31536000 600 7 0 ['test'] ['graphite']
graphite_rollup tagged ^for_taggged avg 7776000 300 7 0 ['test'] ['graphite']
graphite_rollup tagged ^for_taggged avg 0 60 7 0 ['test'] ['graphite']
graphite_rollup all ^ten_min\\\\. sum 31536000 28800 8 0 ['test'] ['graphite']
graphite_rollup all ^ten_min\\\\. sum 5184000 7200 8 0 ['test'] ['graphite']
graphite_rollup all ^ten_min\\\\. sum 0 600 8 0 ['test'] ['graphite']
"""
result = q("SELECT * from system.graphite_retentions")
mismatch = csv_compare(result, expected)
assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected}\ndiff\n{mismatch}\n"
q(
"""
DROP TABLE IF EXISTS test.graphite2;
CREATE TABLE test.graphite2
(metric String, value Float64, timestamp UInt32, date Date, updated UInt32)
ENGINE = GraphiteMergeTree('graphite_rollup')
PARTITION BY toYYYYMM(date)
ORDER BY (metric, timestamp)
SETTINGS index_granularity=8192;
"""
)
expected = """
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
"""
result = q(
"""
SELECT
config_name,
Tables.database,
Tables.table
FROM system.graphite_retentions
"""
)
assert csv_compare(result, expected), f"got\n{result}\nwant\n{expected}"
def test_rollup_versions_all(graphite_table):
timestamp = int(time.time())
rounded_timestamp = timestamp - timestamp % 600
date = datetime.date.today().isoformat()
# Insert rows with timestamps relative to the current time so that the
# first retention clause is active.
# Two parts are created.
q(
"""
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
VALUES ('ten_min.x1', 100, {timestamp}, '{date}', 1);
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
VALUES ('ten_min.x1', 200, {timestamp}, '{date}', 2);
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
VALUES ('ten_min.x1?env=staging', 100, {timestamp}, '{date}', 1);
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
VALUES ('ten_min.x1?env=staging', 200, {timestamp}, '{date}', 2);
""".format(
timestamp=timestamp, date=date
)
)
expected1 = """\
ten_min.x1 100 {timestamp} {date} 1
ten_min.x1 200 {timestamp} {date} 2
ten_min.x1?env=staging 100 {timestamp} {date} 1
ten_min.x1?env=staging 200 {timestamp} {date} 2
""".format(
timestamp=timestamp, date=date
)
result = q("SELECT * FROM test.graphite ORDER BY metric, updated")
mismatch = csv_compare(result, expected1)
assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected1}\ndiff\n{mismatch}\n"
q("OPTIMIZE TABLE test.graphite")
# After rollup only the row with max version is retained.
expected2 = """\
ten_min.x1 200 {timestamp} {date} 2
ten_min.x1?env=staging 200 {timestamp} {date} 2
""".format(
timestamp=rounded_timestamp, date=date
)
result = q("SELECT * FROM test.graphite ORDER BY metric, updated")
mismatch = csv_compare(result, expected2)
assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected2}\ndiff\n{mismatch}\n"
def test_rules_isolation(graphite_table):
to_insert = """\
one_min.x1 100 1000000000 2001-09-09 1
for_taggged 100 1000000001 2001-09-09 1
for_taggged 200 1000000001 2001-09-09 2
one_min?env=staging 100 1000000001 2001-09-09 1
one_min?env=staging 200 1000000001 2001-09-09 2
"""
q("INSERT INTO test.graphite FORMAT TSV", to_insert)
expected = """\
for_taggged 200 1000000001 2001-09-09 2
one_min.x1 100 999999600 2001-09-09 1
one_min?env=staging 200 1000000001 2001-09-09 2
"""
result = q(
"""
OPTIMIZE TABLE test.graphite PARTITION 200109 FINAL;
SELECT * FROM test.graphite;
"""
)
result = q("SELECT * FROM test.graphite ORDER BY metric, updated")
mismatch = csv_compare(result, expected)
assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected}\ndiff\n{mismatch}\n"
def test_rollup_versions_tagged(graphite_table):
timestamp = int(time.time())
rounded_timestamp = timestamp - timestamp % 60
date = datetime.date.today().isoformat()
# Insert rows with timestamps relative to the current time so that the
# first retention clause is active.
# Two parts are created.
q('''
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
VALUES ('x1?retention=one_min', 100, {timestamp}, '{date}', 1);
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
VALUES ('x1?retention=one_min', 200, {timestamp}, '{date}', 2);
'''.format(timestamp=timestamp, date=date))
expected1 = '''\
x1?retention=one_min 100 {timestamp} {date} 1
x1?retention=one_min 200 {timestamp} {date} 2
'''.format(timestamp=timestamp, date=date)
result = q('SELECT * FROM test.graphite ORDER BY metric, updated')
mismatch = csv_compare(result, expected1)
assert len(
mismatch) == 0, f"got\n{result}\nwant\n{expected1}\ndiff\n{mismatch}\n"
q('OPTIMIZE TABLE test.graphite')
# After rollup only the row with max version is retained.
expected2 = '''\
x1?retention=one_min 200 {timestamp} {date} 2
'''.format(timestamp=rounded_timestamp, date=date)
result = q('SELECT * FROM test.graphite ORDER BY metric, updated')
mismatch = csv_compare(result, expected2)
assert len(
mismatch) == 0, f"got\n{result}\nwant\n{expected2}\ndiff\n{mismatch}\n"
def test_system_graphite_retentions(graphite_table):
expected = '''
graphite_rollup all \\\\.count$ sum 0 0 1 0 ['test'] ['graphite']
graphite_rollup all \\\\.max$ max 0 0 2 0 ['test'] ['graphite']
graphite_rollup all ^five_min\\\\. 31536000 14400 3 0 ['test'] ['graphite']
graphite_rollup all ^five_min\\\\. 5184000 3600 3 0 ['test'] ['graphite']
graphite_rollup all ^five_min\\\\. 0 300 3 0 ['test'] ['graphite']
graphite_rollup all ^one_min avg 31536000 600 4 0 ['test'] ['graphite']
graphite_rollup all ^one_min avg 7776000 300 4 0 ['test'] ['graphite']
graphite_rollup all ^one_min avg 0 60 4 0 ['test'] ['graphite']
'''
result = q('SELECT * from system.graphite_retentions')
mismatch = csv_compare(result, expected)
assert len(
mismatch) == 0, f"got\n{result}\nwant\n{expected}\ndiff\n{mismatch}\n"
q('''
DROP TABLE IF EXISTS test.graphite2;
CREATE TABLE test.graphite2
(metric String, value Float64, timestamp UInt32, date Date, updated UInt32)
ENGINE = GraphiteMergeTree('graphite_rollup')
PARTITION BY toYYYYMM(date)
ORDER BY (metric, timestamp)
SETTINGS index_granularity=8192;
''')
expected = '''
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
graphite_rollup ['test','test'] ['graphite','graphite2']
'''
result = q('''
SELECT
config_name,
Tables.database,
Tables.table
FROM system.graphite_retentions
''')
assert TSV(result) == TSV(expected)