def __init__(self, app: AppT, tables: TableManagerT, **kwargs: Any) -> None: self.app = app self.tables = cast(TableManager, tables) self.standby_tps = set() self.active_tps = set() self.tp_to_table = {} self.active_offsets = Counter() self.standby_offsets = Counter() self.active_highwaters = Counter() self.standby_highwaters = Counter() self.completed = Event() self.buffers = defaultdict(list) self.buffer_sizes = {} self.recovery_delay = self.app.conf.stream_recovery_delay self.actives_for_table = defaultdict(set) self.standbys_for_table = defaultdict(set) super().__init__(**kwargs)
def standby_remaining(self) -> Counter[TP]: highwaters = self.standby_highwaters offsets = self.standby_offsets return Counter({ tp: highwater - offsets[tp] for tp, highwater in highwaters.items() if highwater >= 0 and offsets[tp] >= 0 })
def active_remaining(self) -> Counter[TP]: highwaters = self.active_highwaters offsets = self.active_offsets return Counter({ tp: highwater - offsets[tp] for tp, highwater in highwaters.items() if highwater is not None and offsets[tp] is not None })
def __init__(self, table: CollectionT, channel: ChannelT, app: AppT, tps: Set[TP], offsets: Counter[TP] = None, stats_interval: Seconds = 5.0, **kwargs: Any) -> None: super().__init__(**kwargs) self.table = table self.channel = channel self.app = app self.tps = tps self.offsets = Counter() if offsets is None else offsets self.stats_interval = stats_interval for tp in self.tps: self.offsets.setdefault(tp, -1) self._highwaters = Counter() self._stop_event = asyncio.Event(loop=self.loop)
def __init__(self, app: AppT, **kwargs: Any) -> None: super().__init__(**kwargs) self.app = app self.data: MutableMapping = {} self._channels = {} self._changelogs = {} self._table_offsets = Counter() self._standbys = {} self._recovery_started = asyncio.Event(loop=self.loop) self.recovery_completed = asyncio.Event(loop=self.loop)
def __init__(self, *, max_avg_history: int = MAX_AVG_HISTORY, max_commit_latency_history: int = MAX_COMMIT_LATENCY_HISTORY, max_send_latency_history: int = MAX_SEND_LATENCY_HISTORY, messages_sent: int = 0, tables: MutableMapping[str, TableState] = None, messages_active: int = 0, events_active: int = 0, messages_received_total: int = 0, messages_received_by_topic: Counter[str] = None, events_total: int = 0, events_by_stream: Counter[StreamT] = None, events_by_task: Counter[asyncio.Task] = None, events_runtime: List[float] = None, commit_latency: List[float] = None, send_latency: List[float] = None, events_s: int = 0, messages_s: int = 0, events_runtime_avg: float = 0.0, topic_buffer_full: Counter[TopicT] = None, **kwargs: Any) -> None: self.max_avg_history = max_avg_history self.max_commit_latency_history = max_commit_latency_history self.max_send_latency_history = max_send_latency_history self.tables = {} if tables is None else tables self.commit_latency = [] if commit_latency is None else commit_latency self.send_latency = [] if send_latency is None else send_latency self.messages_active = messages_active self.messages_received_total = messages_received_total self.messages_received_by_topic = Counter() self.messages_sent = messages_sent self.messages_sent_by_topic = Counter() self.messages_s = messages_s self.events_active = events_active self.events_total = events_total self.events_by_task = Counter() self.events_by_stream = Counter() self.events_s = events_s self.events_runtime_avg = events_runtime_avg self.events_runtime = [] if events_runtime is None else events_runtime self.topic_buffer_full = Counter() self.time: Callable[[], float] = monotonic self.metric_counts = Counter() self.tp_committed_offsets = {} self.tp_read_offsets = {} self.tp_end_offsets = {} Service.__init__(self, **kwargs)
def _create_reviver(self, table: CollectionT, tps: Set[TP]) -> ChangelogReaderT: table = cast(Table, table) offsets = self._table_offsets table_tps = { tp for tp in tps if tp.topic == table._changelog_topic_name() } self._sync_persisted_offsets(table, table_tps) tp_offsets: Counter[TP] = Counter( {tp: offsets[tp] for tp in table_tps if tp in offsets}) channel = self._channels[table] return ChangelogReader( table, channel, self.app, table_tps, tp_offsets, loop=self.loop, beacon=self.beacon, )
async def _start_standbys(self, tps: Set[TP]) -> None: self.log.info('Attempting to start standbys') assert not self._standbys table_standby_tps = self._group_table_tps(tps) offsets = self._table_offsets for table, table_tps in table_standby_tps.items(): self.log.info('Starting standbys for tps: %s', tps) self._sync_persisted_offsets(table, table_tps) tp_offsets: Counter[TP] = Counter( {tp: offsets[tp] for tp in table_tps if tp in offsets}) channel = self._channels[table] standby = StandbyReader( table, channel, self.app, table_tps, tp_offsets, loop=self.loop, beacon=self.beacon, ) self._standbys[table] = standby await standby.start()
def _assigned_partition_counts(self, active: bool) -> Counter[int]: return Counter( partition for copartitioned in self._client_assignments.values() for partition in copartitioned.get_assigned_partitions(active))