def _get_column(column_type: str, default_type: str, default_expr: str,
codec_expr: str) -> ColumnType:
column: ColumnType = Visitor().visit(grammar.parse(column_type))
if default_type == "MATERIALIZED":
column = Materialized(column, _strip_cast(default_expr))
elif default_type == "DEFAULT":
column = WithDefault(column, _strip_cast(default_expr))
if codec_expr:
column = WithCodecs(column, codec_expr.split(", "))
return column
from snuba.clickhouse.columns import Column, DateTime, Enum, String, UInt
from snuba.clusters.storage_sets import StorageSetKey
from snuba.migrations import migration, operations
from snuba.migrations.columns import WithDefault
from snuba.migrations.context import Context
from snuba.migrations.status import Status
from snuba.migrations.table_engines import Distributed, ReplacingMergeTree
columns = [
Column("group", String()),
Column("migration_id", String()),
Column("timestamp", DateTime()),
Column("status", Enum([("completed", 0), ("in_progress", 1), ("not_started", 2)]),),
Column("version", WithDefault(UInt(64), "1")),
]
class Migration(migration.Migration):
"""
This migration extends Migration instead of MultiStepMigration since it is
responsible for bootstrapping the migration system itself. It skips setting
the in progress status in the forwards method and the not started status in
the backwards method. Since the migration table doesn't exist yet, we can't
write any statuses until this migration is completed.
"""
blocking = False
def __forwards_local(self) -> Sequence[operations.Operation]:
tags_col = Column("tags", Nested([("key", String()), ("value", String())]))
columns = [
Column("project_id", UInt(64)),
Column("transaction_id", UUID()),
Column("trace_id", UUID()),
Column("transaction_span_id", UInt(64)),
Column("span_id", UInt(64)),
Column("parent_span_id", Nullable(UInt(64))),
Column("transaction_name", LowCardinality(String())),
Column("description", String()), # description in span
Column("op", LowCardinality(String())),
Column(
"status",
WithDefault(UInt(8), str(UNKNOWN_SPAN_STATUS)),
),
Column("start_ts", DateTime()),
Column("start_ns", UInt(32)),
Column("finish_ts", DateTime()),
Column("finish_ns", UInt(32)),
Column("duration_ms", UInt(32)),
tags_col,
Column("retention_days", UInt(16)),
Column("deleted", UInt(8)),
]
class Migration(migration.MultiStepMigration):
blocking = False
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.ModifyColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("duration", UInt(32)),
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("sdk_name",
WithDefault(LowCardinality(String()), "''")),
after="user_email",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("sdk_version",
WithDefault(LowCardinality(String()), "''")),
after="sdk_name",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("transaction_status",
WithDefault(UInt(8), str(UNKNOWN_SPAN_STATUS))),
after="transaction_op",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("_tags_flattened", String()),
after="tags",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("_contexts_flattened", String()),
after="contexts",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("user_hash",
Materialized(UInt(64), "cityHash64(user)")),
after="user",
),
# The following columns were originally created as non low cardinality strings
operations.ModifyColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("transaction_name", LowCardinality(String())),
),
operations.ModifyColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("release", LowCardinality(Nullable(String()))),
),
operations.ModifyColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("dist", LowCardinality(Nullable(String()))),
),
operations.ModifyColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("sdk_name",
WithDefault(LowCardinality(String()), "''")),
),
operations.ModifyColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("sdk_version",
WithDefault(LowCardinality(String()), "''")),
),
operations.ModifyColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("environment",
LowCardinality(Nullable(String()))),
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("message_timestamp", DateTime()),
after="offset",
),
operations.DropColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column_name="_start_date",
),
operations.DropColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column_name="_finish_date",
),
]
"event_hash",
WithCodecs(
Materialized(
UInt(64),
"cityHash64(toString(event_id))",
),
["NONE"],
),
),
Column("platform", LowCardinality(String())),
Column("environment", LowCardinality(Nullable(String()))),
Column("release", LowCardinality(Nullable(String()))),
Column("dist", LowCardinality(Nullable(String()))),
Column("ip_address_v4", Nullable(IPv4())),
Column("ip_address_v6", Nullable(IPv6())),
Column("user", WithDefault(String(), "''")),
Column(
"user_hash",
Materialized(UInt(64), "cityHash64(user)"),
),
Column("user_id", Nullable(String())),
Column("user_name", Nullable(String())),
Column("user_email", Nullable(String())),
Column("sdk_name", LowCardinality(Nullable(String()))),
Column("sdk_version", LowCardinality(Nullable(String()))),
Column("tags", Nested([("key", String()), ("value", String())])),
Column("_tags_flattened", String()),
Column("contexts", Nested([("key", String()), ("value", String())])),
Column("_contexts_flattened", String()),
Column("transaction_name", WithDefault(LowCardinality(String()), "''")),
Column(
from snuba.clusters.storage_sets import StorageSetKey
from snuba.migrations import migration, operations, table_engines
from snuba.migrations.columns import LowCardinality, Materialized, WithDefault
UNKNOWN_SPAN_STATUS = 2
columns = [
Column("project_id", UInt(64)),
Column("event_id", UUID()),
Column("trace_id", UUID()),
Column("span_id", UInt(64)),
Column("transaction_name", LowCardinality(String())),
Column("transaction_hash",
Materialized(UInt(64), "cityHash64(transaction_name)")),
Column("transaction_op", LowCardinality(String())),
Column("transaction_status", WithDefault(UInt(8),
str(UNKNOWN_SPAN_STATUS))),
Column("start_ts", DateTime()),
Column("start_ms", UInt(16)),
Column("finish_ts", DateTime()),
Column("finish_ms", UInt(16)),
Column("duration", UInt(32)),
Column("platform", LowCardinality(String())),
Column("environment", LowCardinality(Nullable(String()))),
Column("release", LowCardinality(Nullable(String()))),
Column("dist", LowCardinality(Nullable(String()))),
Column("ip_address_v4", Nullable(IPv4())),
Column("ip_address_v6", Nullable(IPv6())),
Column("user", WithDefault(
String(),
"''",
)),
("LowCardinality(Nullable(String))", "", "", ""),
LowCardinality(Nullable(String())),
),
# Materialized
(
("Date", "MATERIALIZED", "toDate(col1)", ""),
Materialized(Date(), "toDate(col1)"),
),
(
("UInt64", "MATERIALIZED", "CAST(cityHash64(col1), 'UInt64')", ""),
Materialized(UInt(64), "cityHash64(col1)"),
),
# Default value
(
("LowCardinality(String)", "DEFAULT", "a", ""),
WithDefault(LowCardinality(String()), "a"),
),
(("UInt8", "DEFAULT", "2", ""), WithDefault(UInt(8), "2")),
# With codecs
(("UUID", "", "", "NONE"), WithCodecs(UUID(), ["NONE"])),
(
("DateTime", "", "", "DoubleDelta, LZ4"),
WithCodecs(DateTime(), ["DoubleDelta", "LZ4"]),
),
]
@pytest.mark.parametrize("input, expected_output", test_data)
def test_parse_column(input, expected_output):
(input_name, input_type, default_expr, codec_expr) = input
assert (
def __forward_migrations(
self, table_name: str) -> Sequence[operations.Operation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.QUERYLOG,
table_name=table_name,
column=Column(
"clickhouse_queries.all_columns",
WithDefault(
Array(Array(LowCardinality(String()))),
"arrayResize([['']], length(clickhouse_queries.sql))",
),
),
after="clickhouse_queries.consistent",
),
operations.AddColumn(
storage_set=StorageSetKey.QUERYLOG,
table_name=table_name,
column=Column(
"clickhouse_queries.or_conditions",
WithDefault(
Array(UInt(8)),
"arrayResize([0], length(clickhouse_queries.sql))",
),
),
after="clickhouse_queries.all_columns",
),
operations.AddColumn(
storage_set=StorageSetKey.QUERYLOG,
table_name=table_name,
column=Column(
"clickhouse_queries.where_columns",
WithDefault(
Array(Array(LowCardinality(String()))),
"arrayResize([['']], length(clickhouse_queries.sql))",
),
),
after="clickhouse_queries.or_conditions",
),
operations.AddColumn(
storage_set=StorageSetKey.QUERYLOG,
table_name=table_name,
column=Column(
"clickhouse_queries.where_mapping_columns",
WithDefault(
Array(Array(LowCardinality(String()))),
"arrayResize([['']], length(clickhouse_queries.sql))",
),
),
after="clickhouse_queries.where_columns",
),
operations.AddColumn(
storage_set=StorageSetKey.QUERYLOG,
table_name=table_name,
column=Column(
"clickhouse_queries.groupby_columns",
WithDefault(
Array(Array(LowCardinality(String()))),
"arrayResize([['']], length(clickhouse_queries.sql))",
),
),
after="clickhouse_queries.where_mapping_columns",
),
operations.AddColumn(
storage_set=StorageSetKey.QUERYLOG,
table_name=table_name,
column=Column(
"clickhouse_queries.array_join_columns",
WithDefault(
Array(Array(LowCardinality(String()))),
"arrayResize([['']], length(clickhouse_queries.sql))",
),
),
after="clickhouse_queries.groupby_columns",
),
]