def __init__(self,
app: AppT,
tables: TableManagerT,
**kwargs: Any) -> None:
self.app = app
self.tables = cast(TableManager, tables)
self.standby_tps = set()
self.active_tps = set()
self.tp_to_table = {}
self.active_offsets = Counter()
self.standby_offsets = Counter()
self.active_highwaters = Counter()
self.standby_highwaters = Counter()
self.completed = Event()
self.buffers = defaultdict(list)
self.buffer_sizes = {}
self.recovery_delay = self.app.conf.stream_recovery_delay
self.actives_for_table = defaultdict(set)
self.standbys_for_table = defaultdict(set)
super().__init__(**kwargs)
def standby_remaining(self) -> Counter[TP]:
highwaters = self.standby_highwaters
offsets = self.standby_offsets
return Counter({
tp: highwater - offsets[tp]
for tp, highwater in highwaters.items()
if highwater >= 0 and offsets[tp] >= 0
})
def active_remaining(self) -> Counter[TP]:
highwaters = self.active_highwaters
offsets = self.active_offsets
return Counter({
tp: highwater - offsets[tp]
for tp, highwater in highwaters.items()
if highwater is not None and offsets[tp] is not None
})
def __init__(self,
table: CollectionT,
channel: ChannelT,
app: AppT,
tps: Set[TP],
offsets: Counter[TP] = None,
stats_interval: Seconds = 5.0,
**kwargs: Any) -> None:
super().__init__(**kwargs)
self.table = table
self.channel = channel
self.app = app
self.tps = tps
self.offsets = Counter() if offsets is None else offsets
self.stats_interval = stats_interval
for tp in self.tps:
self.offsets.setdefault(tp, -1)
self._highwaters = Counter()
self._stop_event = asyncio.Event(loop=self.loop)
def __init__(self, app: AppT, **kwargs: Any) -> None:
super().__init__(**kwargs)
self.app = app
self.data: MutableMapping = {}
self._channels = {}
self._changelogs = {}
self._table_offsets = Counter()
self._standbys = {}
self._recovery_started = asyncio.Event(loop=self.loop)
self.recovery_completed = asyncio.Event(loop=self.loop)
def __init__(self,
*,
max_avg_history: int = MAX_AVG_HISTORY,
max_commit_latency_history: int = MAX_COMMIT_LATENCY_HISTORY,
max_send_latency_history: int = MAX_SEND_LATENCY_HISTORY,
messages_sent: int = 0,
tables: MutableMapping[str, TableState] = None,
messages_active: int = 0,
events_active: int = 0,
messages_received_total: int = 0,
messages_received_by_topic: Counter[str] = None,
events_total: int = 0,
events_by_stream: Counter[StreamT] = None,
events_by_task: Counter[asyncio.Task] = None,
events_runtime: List[float] = None,
commit_latency: List[float] = None,
send_latency: List[float] = None,
events_s: int = 0,
messages_s: int = 0,
events_runtime_avg: float = 0.0,
topic_buffer_full: Counter[TopicT] = None,
**kwargs: Any) -> None:
self.max_avg_history = max_avg_history
self.max_commit_latency_history = max_commit_latency_history
self.max_send_latency_history = max_send_latency_history
self.tables = {} if tables is None else tables
self.commit_latency = [] if commit_latency is None else commit_latency
self.send_latency = [] if send_latency is None else send_latency
self.messages_active = messages_active
self.messages_received_total = messages_received_total
self.messages_received_by_topic = Counter()
self.messages_sent = messages_sent
self.messages_sent_by_topic = Counter()
self.messages_s = messages_s
self.events_active = events_active
self.events_total = events_total
self.events_by_task = Counter()
self.events_by_stream = Counter()
self.events_s = events_s
self.events_runtime_avg = events_runtime_avg
self.events_runtime = [] if events_runtime is None else events_runtime
self.topic_buffer_full = Counter()
self.time: Callable[[], float] = monotonic
self.metric_counts = Counter()
self.tp_committed_offsets = {}
self.tp_read_offsets = {}
self.tp_end_offsets = {}
Service.__init__(self, **kwargs)
def _create_reviver(self, table: CollectionT,
tps: Set[TP]) -> ChangelogReaderT:
table = cast(Table, table)
offsets = self._table_offsets
table_tps = {
tp
for tp in tps if tp.topic == table._changelog_topic_name()
}
self._sync_persisted_offsets(table, table_tps)
tp_offsets: Counter[TP] = Counter(
{tp: offsets[tp]
for tp in table_tps if tp in offsets})
channel = self._channels[table]
return ChangelogReader(
table,
channel,
self.app,
table_tps,
tp_offsets,
loop=self.loop,
beacon=self.beacon,
)
async def _start_standbys(self, tps: Set[TP]) -> None:
self.log.info('Attempting to start standbys')
assert not self._standbys
table_standby_tps = self._group_table_tps(tps)
offsets = self._table_offsets
for table, table_tps in table_standby_tps.items():
self.log.info('Starting standbys for tps: %s', tps)
self._sync_persisted_offsets(table, table_tps)
tp_offsets: Counter[TP] = Counter(
{tp: offsets[tp]
for tp in table_tps if tp in offsets})
channel = self._channels[table]
standby = StandbyReader(
table,
channel,
self.app,
table_tps,
tp_offsets,
loop=self.loop,
beacon=self.beacon,
)
self._standbys[table] = standby
await standby.start()
def _assigned_partition_counts(self, active: bool) -> Counter[int]:
return Counter(
partition for copartitioned in self._client_assignments.values()
for partition in copartitioned.get_assigned_partitions(active))
