def __init__(
self,
topic: str,
broker_config: KafkaBrokerConfig,
group_id: str,
storage: CdcStorage,
):
self.__consumer = StrictConsumer(
topic=topic,
broker_config=broker_config,
group_id=group_id,
initial_auto_offset_reset="earliest",
partition_assignment_timeout=settings.SNAPSHOT_CONTROL_TOPIC_INIT_TIMEOUT,
on_message=self.__handle_msg,
)
self.__recovery_state = RecoveryState(storage.get_postgres_table())
def __init__(
self,
topic: str,
bootstrap_servers: Sequence[str],
group_id: str,
):
self.__consumer = StrictConsumer(
topic=topic,
bootstrap_servers=bootstrap_servers,
group_id=group_id,
initial_auto_offset_reset="earliest",
partition_assignment_timeout=settings.
SNAPSHOT_CONTROL_TOPIC_INIT_TIMEOUT,
on_message=self.__handle_msg,
)
self.__recovery_state = RecoveryState()
def __consumer(self, on_message) -> StrictConsumer:
return StrictConsumer(
topic="my_topic",
group_id="something",
broker_config=self.broker_config,
initial_auto_offset_reset="earliest",
partition_assignment_timeout=1,
on_partitions_assigned=None,
on_partitions_revoked=None,
on_message=on_message,
)
def __consumer(self, on_message) -> StrictConsumer:
return StrictConsumer(
topic="topic",
bootstrap_servers="somewhere",
group_id="something",
auto_offset_reset="earliest",
partition_assignment_timeout=1,
on_partitions_assigned=None,
on_partitions_revoked=None,
on_message=on_message,
)
class BootstrapState(State[ConsumerStateCompletionEvent,
Optional[ConsumerStateData]]):
"""
This is the state the consumer starts into.
Its job is to either transition to normal operation or to recover a
previously running snapshot if the conumer was restarted while the
process was on going.
The recovery process is done by consuming the whole control topic.
"""
def __init__(
self,
topic: str,
bootstrap_servers: Sequence[str],
group_id: str,
storage: CdcStorage,
):
self.__consumer = StrictConsumer(
topic=topic,
bootstrap_servers=bootstrap_servers,
group_id=group_id,
initial_auto_offset_reset="earliest",
partition_assignment_timeout=settings.
SNAPSHOT_CONTROL_TOPIC_INIT_TIMEOUT,
on_message=self.__handle_msg,
)
self.__recovery_state = RecoveryState(storage.get_postgres_table())
def __handle_msg(self, message: Message) -> CommitDecision:
value = json.loads(message.value())
parsed_message = parse_control_message(value)
if isinstance(parsed_message, SnapshotInit):
commit_decision = self.__recovery_state.process_init(
parsed_message)
elif isinstance(parsed_message, SnapshotAbort):
commit_decision = self.__recovery_state.process_abort(
parsed_message)
elif isinstance(parsed_message, SnapshotLoaded):
commit_decision = self.__recovery_state.process_snapshot_loaded(
parsed_message, )
else:
logger.warning("Received an unrecognized message: %r",
parsed_message)
commit_decision = CommitDecision.DO_NOT_COMMIT
return commit_decision
def signal_shutdown(self) -> None:
self.__consumer.signal_shutdown()
def handle(
self,
state_data: Optional[ConsumerStateData],
) -> Tuple[ConsumerStateCompletionEvent, Optional[ConsumerStateData]]:
logger.info("Running %r", self.__consumer)
self.__consumer.run()
snapshot = self.__recovery_state.get_active_snapshot()
if snapshot and snapshot[1] is not None:
state_data = ConsumerStateData.snapshot_ready_state(
snapshot_id=snapshot[0],
transaction_data=snapshot[1],
)
else:
state_data = None
logger.info("Caught up on the control topic")
return (
self.__recovery_state.get_completion_event(),
state_data,
)