def test_system_graphite_retentions(graphite_table): expected = """ graphite_rollup plain \\\\.count$ sum 0 0 1 0 ['test'] ['graphite'] graphite_rollup plain \\\\.max$ max 0 0 2 0 ['test'] ['graphite'] graphite_rollup plain ^five_min\\\\. 31536000 14400 3 0 ['test'] ['graphite'] graphite_rollup plain ^five_min\\\\. 5184000 3600 3 0 ['test'] ['graphite'] graphite_rollup plain ^five_min\\\\. 0 300 3 0 ['test'] ['graphite'] graphite_rollup plain ^one_min avg 31536000 600 4 0 ['test'] ['graphite'] graphite_rollup plain ^one_min avg 7776000 300 4 0 ['test'] ['graphite'] graphite_rollup plain ^one_min avg 0 60 4 0 ['test'] ['graphite'] graphite_rollup tagged [\\\\?&]retention=one_min(&.*)?$ avg 31536000 600 5 0 ['test'] ['graphite'] graphite_rollup tagged [\\\\?&]retention=one_min(&.*)?$ avg 7776000 300 5 0 ['test'] ['graphite'] graphite_rollup tagged [\\\\?&]retention=one_min(&.*)?$ avg 0 60 5 0 ['test'] ['graphite'] graphite_rollup tagged [\\\\?&]retention=five_min(&.*)?$ avg 31536000 14400 6 0 ['test'] ['graphite'] graphite_rollup tagged [\\\\?&]retention=five_min(&.*)?$ avg 5184000 3600 6 0 ['test'] ['graphite'] graphite_rollup tagged [\\\\?&]retention=five_min(&.*)?$ avg 0 300 6 0 ['test'] ['graphite'] graphite_rollup tagged ^for_taggged avg 31536000 600 7 0 ['test'] ['graphite'] graphite_rollup tagged ^for_taggged avg 7776000 300 7 0 ['test'] ['graphite'] graphite_rollup tagged ^for_taggged avg 0 60 7 0 ['test'] ['graphite'] graphite_rollup all ^ten_min\\\\. sum 31536000 28800 8 0 ['test'] ['graphite'] graphite_rollup all ^ten_min\\\\. sum 5184000 7200 8 0 ['test'] ['graphite'] graphite_rollup all ^ten_min\\\\. sum 0 600 8 0 ['test'] ['graphite'] """ result = q("SELECT * from system.graphite_retentions") mismatch = csv_compare(result, expected) assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected}\ndiff\n{mismatch}\n" q( """ DROP TABLE IF EXISTS test.graphite2; CREATE TABLE test.graphite2 (metric String, value Float64, timestamp UInt32, date Date, updated UInt32) ENGINE = GraphiteMergeTree('graphite_rollup') PARTITION BY toYYYYMM(date) ORDER BY (metric, timestamp) SETTINGS index_granularity=8192; """ ) expected = """ graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] """ result = q( """ SELECT config_name, Tables.database, Tables.table FROM system.graphite_retentions """ ) assert csv_compare(result, expected), f"got\n{result}\nwant\n{expected}"
def test_rollup_versions_all(graphite_table): timestamp = int(time.time()) rounded_timestamp = timestamp - timestamp % 600 date = datetime.date.today().isoformat() # Insert rows with timestamps relative to the current time so that the # first retention clause is active. # Two parts are created. q( """ INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('ten_min.x1', 100, {timestamp}, '{date}', 1); INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('ten_min.x1', 200, {timestamp}, '{date}', 2); INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('ten_min.x1?env=staging', 100, {timestamp}, '{date}', 1); INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('ten_min.x1?env=staging', 200, {timestamp}, '{date}', 2); """.format( timestamp=timestamp, date=date ) ) expected1 = """\ ten_min.x1 100 {timestamp} {date} 1 ten_min.x1 200 {timestamp} {date} 2 ten_min.x1?env=staging 100 {timestamp} {date} 1 ten_min.x1?env=staging 200 {timestamp} {date} 2 """.format( timestamp=timestamp, date=date ) result = q("SELECT * FROM test.graphite ORDER BY metric, updated") mismatch = csv_compare(result, expected1) assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected1}\ndiff\n{mismatch}\n" q("OPTIMIZE TABLE test.graphite") # After rollup only the row with max version is retained. expected2 = """\ ten_min.x1 200 {timestamp} {date} 2 ten_min.x1?env=staging 200 {timestamp} {date} 2 """.format( timestamp=rounded_timestamp, date=date ) result = q("SELECT * FROM test.graphite ORDER BY metric, updated") mismatch = csv_compare(result, expected2) assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected2}\ndiff\n{mismatch}\n"
def test_rules_isolation(graphite_table): to_insert = """\ one_min.x1 100 1000000000 2001-09-09 1 for_taggged 100 1000000001 2001-09-09 1 for_taggged 200 1000000001 2001-09-09 2 one_min?env=staging 100 1000000001 2001-09-09 1 one_min?env=staging 200 1000000001 2001-09-09 2 """ q("INSERT INTO test.graphite FORMAT TSV", to_insert) expected = """\ for_taggged 200 1000000001 2001-09-09 2 one_min.x1 100 999999600 2001-09-09 1 one_min?env=staging 200 1000000001 2001-09-09 2 """ result = q( """ OPTIMIZE TABLE test.graphite PARTITION 200109 FINAL; SELECT * FROM test.graphite; """ ) result = q("SELECT * FROM test.graphite ORDER BY metric, updated") mismatch = csv_compare(result, expected) assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected}\ndiff\n{mismatch}\n"
def test_rollup_versions_tagged(graphite_table): timestamp = int(time.time()) rounded_timestamp = timestamp - timestamp % 60 date = datetime.date.today().isoformat() # Insert rows with timestamps relative to the current time so that the # first retention clause is active. # Two parts are created. q(''' INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('x1?retention=one_min', 100, {timestamp}, '{date}', 1); INSERT INTO test.graphite (metric, value, timestamp, date, updated) VALUES ('x1?retention=one_min', 200, {timestamp}, '{date}', 2); '''.format(timestamp=timestamp, date=date)) expected1 = '''\ x1?retention=one_min 100 {timestamp} {date} 1 x1?retention=one_min 200 {timestamp} {date} 2 '''.format(timestamp=timestamp, date=date) result = q('SELECT * FROM test.graphite ORDER BY metric, updated') mismatch = csv_compare(result, expected1) assert len( mismatch) == 0, f"got\n{result}\nwant\n{expected1}\ndiff\n{mismatch}\n" q('OPTIMIZE TABLE test.graphite') # After rollup only the row with max version is retained. expected2 = '''\ x1?retention=one_min 200 {timestamp} {date} 2 '''.format(timestamp=rounded_timestamp, date=date) result = q('SELECT * FROM test.graphite ORDER BY metric, updated') mismatch = csv_compare(result, expected2) assert len( mismatch) == 0, f"got\n{result}\nwant\n{expected2}\ndiff\n{mismatch}\n"
def test_system_graphite_retentions(graphite_table): expected = ''' graphite_rollup all \\\\.count$ sum 0 0 1 0 ['test'] ['graphite'] graphite_rollup all \\\\.max$ max 0 0 2 0 ['test'] ['graphite'] graphite_rollup all ^five_min\\\\. 31536000 14400 3 0 ['test'] ['graphite'] graphite_rollup all ^five_min\\\\. 5184000 3600 3 0 ['test'] ['graphite'] graphite_rollup all ^five_min\\\\. 0 300 3 0 ['test'] ['graphite'] graphite_rollup all ^one_min avg 31536000 600 4 0 ['test'] ['graphite'] graphite_rollup all ^one_min avg 7776000 300 4 0 ['test'] ['graphite'] graphite_rollup all ^one_min avg 0 60 4 0 ['test'] ['graphite'] ''' result = q('SELECT * from system.graphite_retentions') mismatch = csv_compare(result, expected) assert len( mismatch) == 0, f"got\n{result}\nwant\n{expected}\ndiff\n{mismatch}\n" q(''' DROP TABLE IF EXISTS test.graphite2; CREATE TABLE test.graphite2 (metric String, value Float64, timestamp UInt32, date Date, updated UInt32) ENGINE = GraphiteMergeTree('graphite_rollup') PARTITION BY toYYYYMM(date) ORDER BY (metric, timestamp) SETTINGS index_granularity=8192; ''') expected = ''' graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] graphite_rollup ['test','test'] ['graphite','graphite2'] ''' result = q(''' SELECT config_name, Tables.database, Tables.table FROM system.graphite_retentions ''') assert TSV(result) == TSV(expected)