def test_missing_trace_context(self) -> None:
start, finish = self.__get_timestamps()
message = TransactionEvent(
event_id="e5e062bf2e1d4afd96fd2f90b6770431",
trace_id="7400045b25c443b885914600aa83ad04",
span_id="8841662216cc598b",
transaction_name="/organizations/:orgId/issues/",
status="cancelled",
op="navigation",
timestamp=finish,
start_timestamp=start,
platform="python",
dist="",
user_name="me",
user_id="myself",
user_email="*****@*****.**",
ipv4="127.0.0.1",
ipv6=None,
environment="prod",
release="34a554c14b68285d8a8eb6c5c4c56dfc1db9a83a",
sdk_name="sentry.python",
sdk_version="0.9.0",
http_method="POST",
http_referer="tagstore.something",
geo={"country_code": "XY", "region": "fake_region", "city": "fake_city"},
)
payload = message.serialize()
# Force an invalid event
del payload[2]["data"]["contexts"]
meta = KafkaMessageMetadata(
offset=1, partition=2, timestamp=datetime(1970, 1, 1)
)
processor = TransactionsMessageProcessor()
assert processor.process_message(payload, meta) is None
def test_base_process(self):
message = TransactionEvent(
event_id='e5e062bf2e1d4afd96fd2f90b6770431',
trace_id='7400045b25c443b885914600aa83ad04',
span_id='8841662216cc598b',
transaction_name='/organizations/:orgId/issues/',
op='navigation',
start_timestamp=1565303393.917,
timestamp=1565303392.918,
platform='python',
dist='',
user_name='me',
user_id='myself',
user_email='*****@*****.**',
ipv4='127.0.0.1',
ipv6=None,
environment='prod',
release='34a554c14b68285d8a8eb6c5c4c56dfc1db9a83a',
)
meta = KafkaMessageMetadata(
offset=1,
partition=2,
)
processor = TransactionsMessageProcessor()
ret = processor.process_message(message.serialize(), meta)
assert ret.action == ProcessorAction.INSERT
assert ret.data == [message.build_result(meta)]
def test_missing_trace_context(self):
message = TransactionEvent(
event_id='e5e062bf2e1d4afd96fd2f90b6770431',
trace_id='7400045b25c443b885914600aa83ad04',
span_id='8841662216cc598b',
transaction_name='/organizations/:orgId/issues/',
op='navigation',
start_timestamp=1565303392.917,
timestamp=1565303393.918,
platform='python',
dist='',
user_name='me',
user_id='myself',
user_email='*****@*****.**',
ipv4='127.0.0.1',
ipv6=None,
environment='prod',
release='34a554c14b68285d8a8eb6c5c4c56dfc1db9a83a',
)
payload = message.serialize()
# Force an invalid event
del payload[2]['data']['contexts']
meta = KafkaMessageMetadata(offset=1, partition=2)
processor = TransactionsMessageProcessor()
assert processor.process_message(payload, meta) is None
def test_base_process(self):
start, finish = self.__get_timestamps()
message = TransactionEvent(
event_id="e5e062bf2e1d4afd96fd2f90b6770431",
trace_id="7400045b25c443b885914600aa83ad04",
span_id="8841662216cc598b",
transaction_name="/organizations/:orgId/issues/",
status="cancelled",
op="navigation",
timestamp=finish,
start_timestamp=start,
platform="python",
dist="",
user_name="me",
user_id="myself",
user_email="*****@*****.**",
ipv4="127.0.0.1",
ipv6=None,
environment="prod",
release="34a554c14b68285d8a8eb6c5c4c56dfc1db9a83a",
sdk_name="sentry.python",
sdk_version="0.9.0",
geo={
"country_code": "XY",
"region": "fake_region",
"city": "fake_city"
},
)
meta = KafkaMessageMetadata(
offset=1,
partition=2,
)
processor = TransactionsMessageProcessor()
ret = processor.process_message(message.serialize(), meta)
assert ret.action == ProcessorAction.INSERT
assert ret.data == [message.build_result(meta)]
def test_base_process(self) -> None:
old_skip_context = settings.TRANSACT_SKIP_CONTEXT_STORE
settings.TRANSACT_SKIP_CONTEXT_STORE = {1: {"experiments"}}
start, finish = self.__get_timestamps()
message = TransactionEvent(
event_id="e5e062bf2e1d4afd96fd2f90b6770431",
trace_id="7400045b25c443b885914600aa83ad04",
span_id="8841662216cc598b",
transaction_name="/organizations/:orgId/issues/",
status="cancelled",
op="navigation",
timestamp=finish,
start_timestamp=start,
platform="python",
dist="",
user_name="me",
user_id="myself",
user_email="*****@*****.**",
ipv4="127.0.0.1",
ipv6=None,
environment="prod",
release="34a554c14b68285d8a8eb6c5c4c56dfc1db9a83a",
sdk_name="sentry.python",
sdk_version="0.9.0",
http_method="POST",
http_referer="tagstore.something",
geo={"country_code": "XY", "region": "fake_region", "city": "fake_city"},
)
meta = KafkaMessageMetadata(
offset=1, partition=2, timestamp=datetime(1970, 1, 1)
)
assert TransactionsMessageProcessor().process_message(
message.serialize(), meta
) == InsertBatch([message.build_result(meta)], None)
settings.TRANSACT_SKIP_CONTEXT_STORE = old_skip_context
# during create statement
# (https://github.com/ClickHouse/ClickHouse/issues/12586), so the
# materialization is added with a migration.
skipped_cols_on_creation={"_tags_hash_map"},
)
storage = WritableTableStorage(
storage_key=StorageKey.TRANSACTIONS,
storage_set_key=StorageSetKey.TRANSACTIONS,
schema=schema,
query_processors=[
NestedFieldConditionOptimizer(
"contexts",
"_contexts_flattened",
{"start_ts", "finish_ts"},
BEGINNING_OF_TIME,
),
MappingOptimizer("tags", "_tags_hash_map", "tags_hash_map_enabled"),
TransactionColumnProcessor(),
ArrayJoinKeyValueOptimizer("tags"),
ArrayJoinKeyValueOptimizer("measurements"),
PrewhereProcessor(),
],
stream_loader=KafkaStreamLoader(
processor=TransactionsMessageProcessor(), default_topic="events",
),
query_splitters=[TimeSplitQueryStrategy(timestamp_col="finish_ts")],
writer_options={"insert_allow_materialized_columns": 1},
)
# during create statement
# (https://github.com/ClickHouse/ClickHouse/issues/12586), so the
# materialization is added with a migration.
skipped_cols_on_creation={"_tags_hash_map"},
)
storage = WritableTableStorage(
storage_key=StorageKey.TRANSACTIONS,
storage_set_key=StorageSetKey.TRANSACTIONS,
schema=schema,
query_processors=[
NestedFieldConditionOptimizer(
"contexts",
"_contexts_flattened",
{"start_ts", "finish_ts"},
BEGINNING_OF_TIME,
),
MappingOptimizer("tags", "_tags_hash_map", "tags_hash_map_enabled"),
TransactionColumnProcessor(),
ArrayJoinKeyValueOptimizer("tags"),
ArrayJoinKeyValueOptimizer("measurements"),
PrewhereProcessor(),
],
stream_loader=KafkaStreamLoader(
processor=TransactionsMessageProcessor(),
default_topic="events",
),
query_splitters=[TimeSplitQueryStrategy(timestamp_col="finish_ts")],
writer_options={"insert_allow_materialized_columns": 1},
)
def __init__(self) -> None:
columns = ColumnSet(
[
("project_id", UInt(64)),
("event_id", UUID()),
("trace_id", UUID()),
("span_id", UInt(64)),
("transaction_name", LowCardinality(String())),
(
"transaction_hash",
Materialized(UInt(64), "cityHash64(transaction_name)",),
),
("transaction_op", LowCardinality(String())),
("transaction_status", WithDefault(UInt(8), UNKNOWN_SPAN_STATUS)),
("start_ts", DateTime()),
("start_ms", UInt(16)),
("_start_date", Materialized(Date(), "toDate(start_ts)"),),
("finish_ts", DateTime()),
("finish_ms", UInt(16)),
("_finish_date", Materialized(Date(), "toDate(finish_ts)"),),
("duration", UInt(32)),
("platform", LowCardinality(String())),
("environment", LowCardinality(Nullable(String()))),
("release", LowCardinality(Nullable(String()))),
("dist", LowCardinality(Nullable(String()))),
("ip_address_v4", Nullable(IPv4())),
("ip_address_v6", Nullable(IPv6())),
("user", WithDefault(String(), "''",)),
("user_hash", Materialized(UInt(64), "cityHash64(user)"),),
("user_id", Nullable(String())),
("user_name", Nullable(String())),
("user_email", Nullable(String())),
("sdk_name", WithDefault(LowCardinality(String()), "''")),
("sdk_version", WithDefault(LowCardinality(String()), "''")),
("tags", Nested([("key", String()), ("value", String())])),
("_tags_flattened", String()),
("contexts", Nested([("key", String()), ("value", String())])),
("_contexts_flattened", String()),
("partition", UInt(16)),
("offset", UInt(64)),
("retention_days", UInt(16)),
("deleted", UInt(8)),
]
)
schema = ReplacingMergeTreeSchema(
columns=columns,
local_table_name="transactions_local",
dist_table_name="transactions_dist",
mandatory_conditions=[],
prewhere_candidates=["event_id", "project_id"],
order_by="(project_id, _finish_date, transaction_name, cityHash64(span_id))",
partition_by="(retention_days, toMonday(_finish_date))",
version_column="deleted",
sample_expr=None,
migration_function=transactions_migrations,
)
dataset_schemas = DatasetSchemas(read_schema=schema, write_schema=schema,)
self.__tags_processor = TagColumnProcessor(
columns=columns,
promoted_columns=self._get_promoted_columns(),
column_tag_map=self._get_column_tag_map(),
)
super().__init__(
dataset_schemas=dataset_schemas,
table_writer=TransactionsTableWriter(
write_schema=schema,
stream_loader=KafkaStreamLoader(
processor=TransactionsMessageProcessor(), default_topic="events",
),
),
time_group_columns={
"bucketed_start": "start_ts",
"bucketed_end": "finish_ts",
},
time_parse_columns=("start_ts", "finish_ts"),
)
def __init__(self):
columns = ColumnSet([
('project_id', UInt(64)),
('event_id', UUID()),
('trace_id', UUID()),
('span_id', UInt(64)),
('transaction_name', String()),
('transaction_hash',
Materialized(
UInt(64),
'cityHash64(transaction_name)',
)),
('transaction_op', LowCardinality(String())),
('start_ts', DateTime()),
('start_ms', UInt(16)),
('finish_ts', DateTime()),
('finish_ms', UInt(16)),
('duration',
Materialized(
UInt(32),
'((finish_ts - start_ts) * 1000) + (finish_ms - start_ms)',
)),
('platform', LowCardinality(String())),
('environment', Nullable(String())),
('release', Nullable(String())),
('dist', Nullable(String())),
('ip_address_v4', Nullable(IPv4())),
('ip_address_v6', Nullable(IPv6())),
('user', WithDefault(
String(),
"''",
)),
('user_id', Nullable(String())),
('user_name', Nullable(String())),
('user_email', Nullable(String())),
('tags', Nested([
('key', String()),
('value', String()),
])),
('contexts', Nested([
('key', String()),
('value', String()),
])),
('partition', UInt(16)),
('offset', UInt(64)),
('retention_days', UInt(16)),
('deleted', UInt(8)),
])
schema = ReplacingMergeTreeSchema(
columns=columns,
local_table_name='transactions_local',
dist_table_name='transactions_dist',
order_by=
'(project_id, toStartOfDay(start_ts), transaction_hash, start_ts, start_ms, trace_id, span_id)',
partition_by='(retention_days, toMonday(start_ts))',
version_column='deleted',
sample_expr=None,
)
dataset_schemas = DatasetSchemas(
read_schema=schema,
write_schema=schema,
)
super().__init__(
dataset_schemas=dataset_schemas,
processor=TransactionsMessageProcessor(),
default_topic="events",
time_group_columns={
'bucketed_start': 'start_ts',
'bucketed_end': 'finish_ts',
},
)