Exemple #1
0
def test_system_graphite_retentions(graphite_table):
    expected = """
graphite_rollup	plain	\\\\.count$	sum	0	0	1	0	['test']	['graphite']
graphite_rollup	plain	\\\\.max$	max	0	0	2	0	['test']	['graphite']
graphite_rollup	plain	^five_min\\\\.		31536000	14400	3	0	['test']	['graphite']
graphite_rollup	plain	^five_min\\\\.		5184000	3600	3	0	['test']	['graphite']
graphite_rollup	plain	^five_min\\\\.		0	300	3	0	['test']	['graphite']
graphite_rollup	plain	^one_min	avg	31536000	600	4	0	['test']	['graphite']
graphite_rollup	plain	^one_min	avg	7776000	300	4	0	['test']	['graphite']
graphite_rollup	plain	^one_min	avg	0	60	4	0	['test']	['graphite']
graphite_rollup	tagged	[\\\\?&]retention=one_min(&.*)?$	avg	31536000	600	5	0	['test']	['graphite']
graphite_rollup	tagged	[\\\\?&]retention=one_min(&.*)?$	avg	7776000	300	5	0	['test']	['graphite']
graphite_rollup	tagged	[\\\\?&]retention=one_min(&.*)?$	avg	0	60	5	0	['test']	['graphite']
graphite_rollup	tagged	[\\\\?&]retention=five_min(&.*)?$	avg	31536000	14400	6	0	['test']	['graphite']
graphite_rollup	tagged	[\\\\?&]retention=five_min(&.*)?$	avg	5184000	3600	6	0	['test']	['graphite']
graphite_rollup	tagged	[\\\\?&]retention=five_min(&.*)?$	avg	0	300	6	0	['test']	['graphite']
graphite_rollup	tagged	^for_taggged	avg	31536000	600	7	0	['test']	['graphite']
graphite_rollup	tagged	^for_taggged	avg	7776000	300	7	0	['test']	['graphite']
graphite_rollup	tagged	^for_taggged	avg	0	60	7	0	['test']	['graphite']
graphite_rollup	all	^ten_min\\\\.	sum	31536000	28800	8	0	['test']	['graphite']
graphite_rollup	all	^ten_min\\\\.	sum	5184000	7200	8	0	['test']	['graphite']
graphite_rollup	all	^ten_min\\\\.	sum	0	600	8	0	['test']	['graphite']
    """
    result = q("SELECT * from system.graphite_retentions")

    mismatch = csv_compare(result, expected)
    assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected}\ndiff\n{mismatch}\n"

    q(
        """
DROP TABLE IF EXISTS test.graphite2;
CREATE TABLE test.graphite2
    (metric String, value Float64, timestamp UInt32, date Date, updated UInt32)
    ENGINE = GraphiteMergeTree('graphite_rollup')
    PARTITION BY toYYYYMM(date)
    ORDER BY (metric, timestamp)
    SETTINGS index_granularity=8192;
    """
    )
    expected = """
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
    """
    result = q(
        """
    SELECT
        config_name,
        Tables.database,
        Tables.table
    FROM system.graphite_retentions
    """
    )
    assert csv_compare(result, expected), f"got\n{result}\nwant\n{expected}"
Exemple #2
0
def test_rollup_versions_all(graphite_table):
    timestamp = int(time.time())
    rounded_timestamp = timestamp - timestamp % 600
    date = datetime.date.today().isoformat()

    # Insert rows with timestamps relative to the current time so that the
    # first retention clause is active.
    # Two parts are created.
    q(
        """
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
      VALUES ('ten_min.x1', 100, {timestamp}, '{date}', 1);
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
      VALUES ('ten_min.x1', 200, {timestamp}, '{date}', 2);
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
      VALUES ('ten_min.x1?env=staging', 100, {timestamp}, '{date}', 1);
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
      VALUES ('ten_min.x1?env=staging', 200, {timestamp}, '{date}', 2);
""".format(
            timestamp=timestamp, date=date
        )
    )

    expected1 = """\
ten_min.x1	100	{timestamp}	{date}	1
ten_min.x1	200	{timestamp}	{date}	2
ten_min.x1?env=staging	100	{timestamp}	{date}	1
ten_min.x1?env=staging	200	{timestamp}	{date}	2
""".format(
        timestamp=timestamp, date=date
    )

    result = q("SELECT * FROM test.graphite ORDER BY metric, updated")
    mismatch = csv_compare(result, expected1)
    assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected1}\ndiff\n{mismatch}\n"

    q("OPTIMIZE TABLE test.graphite")

    # After rollup only the row with max version is retained.
    expected2 = """\
ten_min.x1	200	{timestamp}	{date}	2
ten_min.x1?env=staging	200	{timestamp}	{date}	2
""".format(
        timestamp=rounded_timestamp, date=date
    )

    result = q("SELECT * FROM test.graphite ORDER BY metric, updated")
    mismatch = csv_compare(result, expected2)
    assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected2}\ndiff\n{mismatch}\n"
Exemple #3
0
def test_rules_isolation(graphite_table):
    to_insert = """\
one_min.x1	100	1000000000	2001-09-09	1
for_taggged	100	1000000001	2001-09-09	1
for_taggged	200	1000000001	2001-09-09	2
one_min?env=staging	100	1000000001	2001-09-09	1
one_min?env=staging	200	1000000001	2001-09-09	2
"""

    q("INSERT INTO test.graphite FORMAT TSV", to_insert)

    expected = """\
for_taggged	200	1000000001	2001-09-09	2
one_min.x1	100	999999600	2001-09-09	1
one_min?env=staging	200	1000000001	2001-09-09	2
"""

    result = q(
        """
OPTIMIZE TABLE test.graphite PARTITION 200109 FINAL;

SELECT * FROM test.graphite;
"""
    )

    result = q("SELECT * FROM test.graphite ORDER BY metric, updated")
    mismatch = csv_compare(result, expected)
    assert len(mismatch) == 0, f"got\n{result}\nwant\n{expected}\ndiff\n{mismatch}\n"
Exemple #4
0
def test_rollup_versions_tagged(graphite_table):
    timestamp = int(time.time())
    rounded_timestamp = timestamp - timestamp % 60
    date = datetime.date.today().isoformat()

    # Insert rows with timestamps relative to the current time so that the
    # first retention clause is active.
    # Two parts are created.
    q('''
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
      VALUES ('x1?retention=one_min', 100, {timestamp}, '{date}', 1);
INSERT INTO test.graphite (metric, value, timestamp, date, updated)
      VALUES ('x1?retention=one_min', 200, {timestamp}, '{date}', 2);
'''.format(timestamp=timestamp, date=date))

    expected1 = '''\
x1?retention=one_min	100	{timestamp}	{date}	1
x1?retention=one_min	200	{timestamp}	{date}	2
'''.format(timestamp=timestamp, date=date)

    result = q('SELECT * FROM test.graphite ORDER BY metric, updated')
    mismatch = csv_compare(result, expected1)
    assert len(
        mismatch) == 0, f"got\n{result}\nwant\n{expected1}\ndiff\n{mismatch}\n"

    q('OPTIMIZE TABLE test.graphite')

    # After rollup only the row with max version is retained.
    expected2 = '''\
x1?retention=one_min	200	{timestamp}	{date}	2
'''.format(timestamp=rounded_timestamp, date=date)

    result = q('SELECT * FROM test.graphite ORDER BY metric, updated')
    mismatch = csv_compare(result, expected2)
    assert len(
        mismatch) == 0, f"got\n{result}\nwant\n{expected2}\ndiff\n{mismatch}\n"
Exemple #5
0
def test_system_graphite_retentions(graphite_table):
    expected = '''
graphite_rollup	all	\\\\.count$	sum	0	0	1	0	['test']	['graphite']
graphite_rollup	all	\\\\.max$	max	0	0	2	0	['test']	['graphite']
graphite_rollup	all	^five_min\\\\.		31536000	14400	3	0	['test']	['graphite']
graphite_rollup	all	^five_min\\\\.		5184000	3600	3	0	['test']	['graphite']
graphite_rollup	all	^five_min\\\\.		0	300	3	0	['test']	['graphite']
graphite_rollup	all	^one_min	avg	31536000	600	4	0	['test']	['graphite']
graphite_rollup	all	^one_min	avg	7776000	300	4	0	['test']	['graphite']
graphite_rollup	all	^one_min	avg	0	60	4	0	['test']	['graphite']
    '''
    result = q('SELECT * from system.graphite_retentions')

    mismatch = csv_compare(result, expected)
    assert len(
        mismatch) == 0, f"got\n{result}\nwant\n{expected}\ndiff\n{mismatch}\n"

    q('''
DROP TABLE IF EXISTS test.graphite2;
CREATE TABLE test.graphite2
    (metric String, value Float64, timestamp UInt32, date Date, updated UInt32)
    ENGINE = GraphiteMergeTree('graphite_rollup')
    PARTITION BY toYYYYMM(date)
    ORDER BY (metric, timestamp)
    SETTINGS index_granularity=8192;
    ''')
    expected = '''
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
graphite_rollup	['test','test']	['graphite','graphite2']
    '''
    result = q('''
    SELECT
        config_name,
        Tables.database,
        Tables.table
    FROM system.graphite_retentions
    ''')
    assert TSV(result) == TSV(expected)