def __init__(
self,
topic: str,
broker_config: KafkaBrokerConfig,
group_id: str,
storage: CdcStorage,
):
self.__consumer = StrictConsumer(
topic=topic,
broker_config=broker_config,
group_id=group_id,
initial_auto_offset_reset="earliest",
partition_assignment_timeout=settings.SNAPSHOT_CONTROL_TOPIC_INIT_TIMEOUT,
on_message=self.__handle_msg,
)
self.__recovery_state = RecoveryState(storage.get_postgres_table())
("user_id", UInt(64, Modifiers(nullable=True))),
("team_id", UInt(64, Modifiers(nullable=True))),
])
schema = WritableTableSchema(
columns=columns,
local_table_name="groupassignee_local",
dist_table_name="groupassignee_dist",
storage_set_key=StorageSetKey.CDC,
)
POSTGRES_TABLE = "sentry_groupasignee"
storage = CdcStorage(
storage_key=StorageKey.GROUPASSIGNEES,
storage_set_key=StorageSetKey.CDC,
schema=schema,
query_processors=[
PrewhereProcessor(["project_id"]),
ConsistencyEnforcerProcessor(),
],
stream_loader=build_kafka_stream_loader_from_settings(
processor=GroupAssigneeProcessor(POSTGRES_TABLE),
default_topic=Topic.CDC,
pre_filter=CdcTableNameMessageFilter(POSTGRES_TABLE),
),
default_control_topic="cdc_control",
postgres_table=POSTGRES_TABLE,
row_processor=lambda row: GroupAssigneeRow.from_bulk(row).to_clickhouse(),
)
local_table_name="groupedmessage_local",
dist_table_name="groupedmessage_dist",
storage_set_key=StorageSetKey.EVENTS,
mandatory_conditions=[
binary_condition(
ConditionFunctions.EQ,
Column(None, None, "record_deleted"),
Literal(None, 0),
),
],
prewhere_candidates=["project_id", "id"],
)
POSTGRES_TABLE = "sentry_groupedmessage"
storage = CdcStorage(
storage_key=StorageKey.GROUPEDMESSAGES,
storage_set_key=StorageSetKey.EVENTS,
schema=schema,
query_processors=[],
stream_loader=build_kafka_stream_loader_from_settings(
StorageKey.GROUPEDMESSAGES,
processor=GroupedMessageProcessor(POSTGRES_TABLE),
default_topic_name="cdc",
pre_filter=CdcTableNameMessageFilter(POSTGRES_TABLE),
),
default_control_topic="cdc_control",
postgres_table=POSTGRES_TABLE,
row_processor=lambda row: GroupedMessageRow.from_bulk(row).to_clickhouse(),
)
("team_id", Nullable(UInt(64))),
]
)
schema = ReplacingMergeTreeSchema(
columns=columns,
local_table_name="groupassignee_local",
dist_table_name="groupassignee_dist",
order_by="(project_id, group_id)",
partition_by=None,
version_column="offset",
)
POSTGRES_TABLE = "sentry_groupasignee"
storage = CdcStorage(
schemas=StorageSchemas(read_schema=schema, write_schema=schema),
table_writer=GroupAssigneeTableWriter(
write_schema=schema,
stream_loader=KafkaStreamLoader(
processor=GroupAssigneeProcessor(POSTGRES_TABLE),
default_topic="cdc",
pre_filter=CdcTableNameMessageFilter(POSTGRES_TABLE),
),
postgres_table=POSTGRES_TABLE,
),
query_processors=[PrewhereProcessor()],
default_control_topic="cdc_control",
postgres_table=POSTGRES_TABLE,
)
("project_id", UInt(64)),
("group_id", UInt(64)),
("date_added", DateTime(Modifiers(nullable=True))),
("user_id", UInt(64, Modifiers(nullable=True))),
("team_id", UInt(64, Modifiers(nullable=True))),
])
schema = WritableTableSchema(
columns=columns,
local_table_name="groupassignee_local",
dist_table_name="groupassignee_dist",
storage_set_key=StorageSetKey.EVENTS,
)
POSTGRES_TABLE = "sentry_groupasignee"
storage = CdcStorage(
storage_key=StorageKey.GROUPASSIGNEES,
storage_set_key=StorageSetKey.EVENTS,
schema=schema,
query_processors=[PrewhereProcessor()],
stream_loader=KafkaStreamLoader(
processor=GroupAssigneeProcessor(POSTGRES_TABLE),
default_topic="cdc",
pre_filter=CdcTableNameMessageFilter(POSTGRES_TABLE),
),
default_control_topic="cdc_control",
postgres_table=POSTGRES_TABLE,
row_processor=lambda row: GroupAssigneeRow.from_bulk(row).to_clickhouse(),
)