def __init__(self, app: AppT, tables: TableManagerT,
**kwargs: Any) -> None:
self.app = app
self.tables = cast(_TableManager, tables)
self.standby_tps = set()
self.active_tps = set()
self.tp_to_table = {}
self.active_offsets = Counter()
self.standby_offsets = Counter()
self.active_highwaters = Counter()
self.standby_highwaters = Counter()
self.completed = Event()
self.buffers = defaultdict(list)
self.buffer_sizes = {}
self.recovery_delay = self.app.conf.stream_recovery_delay
self.actives_for_table = defaultdict(set)
self.standbys_for_table = defaultdict(set)
self._active_events_received_at = {}
self._standby_events_received_at = {}
self._processing_times = deque()
super().__init__(**kwargs)
def standby_remaining(self) -> Counter[TP]:
highwaters = self.standby_highwaters
offsets = self.standby_offsets
return Counter({
tp: highwater - offsets[tp]
for tp, highwater in highwaters.items()
if highwater >= 0 and offsets[tp] >= 0
})
def active_remaining(self) -> Counter[TP]:
highwaters = self.active_highwaters
offsets = self.active_offsets
return Counter({
tp: highwater - offsets[tp]
for tp, highwater in highwaters.items()
if highwater is not None and offsets[tp] is not None
})
def standby_remaining(self) -> Counter[TP]:
"""Return counter of remaining changes by standby partition."""
highwaters = self.standby_highwaters
offsets = self.standby_offsets
return Counter({
tp: highwater - offsets[tp]
for tp, highwater in highwaters.items()
if highwater >= 0 and offsets[tp] >= 0
})
def active_remaining(self) -> Counter[TP]:
"""Return counter of remaining changes by active partition."""
highwaters = self.active_highwaters
offsets = self.active_offsets
return Counter({
tp: highwater - offsets[tp]
for tp, highwater in highwaters.items()
if highwater is not None and offsets[tp] is not None
})
def __init__(self, expr: Type, root: 'RootNode' = None, *,
user_types: CoercionMapping = None,
date_parser: Callable[[Any], datetime] = None) -> None:
assert self.type == NodeType.ROOT
self.type_stats = Counter()
self.user_types = user_types or {}
self.extra_locals = {}
self.date_parser: Callable[[Any], datetime]
if date_parser is not None:
self.date_parser = date_parser
else:
self.date_parser = parse_iso8601
self.found_types = defaultdict(set)
super().__init__(expr, root=self)
def __init__(self,
*,
max_avg_history: int = MAX_AVG_HISTORY,
max_commit_latency_history: int = MAX_COMMIT_LATENCY_HISTORY,
max_send_latency_history: int = MAX_SEND_LATENCY_HISTORY,
messages_sent: int = 0,
tables: MutableMapping[str, TableState] = None,
messages_active: int = 0,
events_active: int = 0,
messages_received_total: int = 0,
messages_received_by_topic: Counter[str] = None,
events_total: int = 0,
events_by_stream: Counter[StreamT] = None,
events_by_task: Counter[asyncio.Task] = None,
events_runtime: List[float] = None,
commit_latency: List[float] = None,
send_latency: List[float] = None,
events_s: int = 0,
messages_s: int = 0,
events_runtime_avg: float = 0.0,
topic_buffer_full: Counter[TopicT] = None,
**kwargs: Any) -> None:
self.max_avg_history = max_avg_history
self.max_commit_latency_history = max_commit_latency_history
self.max_send_latency_history = max_send_latency_history
self.tables = {} if tables is None else tables
self.commit_latency = [] if commit_latency is None else commit_latency
self.send_latency = [] if send_latency is None else send_latency
self.messages_active = messages_active
self.messages_received_total = messages_received_total
self.messages_received_by_topic = Counter()
self.messages_sent = messages_sent
self.messages_sent_by_topic = Counter()
self.messages_s = messages_s
self.events_active = events_active
self.events_total = events_total
self.events_by_task = Counter()
self.events_by_stream = Counter()
self.events_s = events_s
self.events_runtime_avg = events_runtime_avg
self.events_runtime = [] if events_runtime is None else events_runtime
self.topic_buffer_full = Counter()
self.time: Callable[[], float] = monotonic
self.metric_counts = Counter()
self.tp_committed_offsets = {}
self.tp_read_offsets = {}
self.tp_end_offsets = {}
Service.__init__(self, **kwargs)
def __init__(self,
transport: TransportT,
callback: ConsumerCallback,
on_partitions_revoked: PartitionsRevokedCallback,
on_partitions_assigned: PartitionsAssignedCallback,
*,
commit_interval: float = None,
commit_livelock_soft_timeout: float = None,
loop: asyncio.AbstractEventLoop = None,
**kwargs: Any) -> None:
assert callback is not None
self.transport = transport
self.app = self.transport.app
self.in_transaction = self.app.in_transaction
self.callback = callback
self._on_message_in = self.app.sensors.on_message_in
self._on_partitions_revoked = on_partitions_revoked
self._on_partitions_assigned = on_partitions_assigned
self._commit_every = self.app.conf.broker_commit_every
self.scheduler = self.app.conf.ConsumerScheduler()
self.commit_interval = (commit_interval
or self.app.conf.broker_commit_interval)
self.commit_livelock_soft_timeout = (
commit_livelock_soft_timeout
or self.app.conf.broker_commit_livelock_soft_timeout)
self._gap = defaultdict(list)
self._acked = defaultdict(list)
self._acked_index = defaultdict(set)
self._read_offset = defaultdict(lambda: None)
self._committed_offset = defaultdict(lambda: None)
self._unacked_messages = WeakSet()
self._waiting_for_ack = None
self._time_start = monotonic()
self._last_batch = Counter()
self._end_offset_monitor_interval = self.commit_interval * 2
self.randomly_assigned_topics = set()
self.can_resume_flow = Event()
self._reset_state()
super().__init__(loop=loop or self.transport.loop, **kwargs)
self.transactions = self.transport.create_transaction_manager(
consumer=self,
producer=self.app.producer,
beacon=self.beacon,
loop=self.loop,
)
def _khan62(self) -> Sequence:
"""Perform Khan's simple topological sort algorithm from '62.
See https://en.wikipedia.org/wiki/Topological_sorting
"""
count: Counter[Any] = Counter()
result = []
for node in self:
for successor in self[node]:
count[successor] += 1
ready = [node for node in self if not count[node]]
while ready:
node = ready.pop()
result.append(node)
for successor in self[node]:
count[successor] -= 1
if count[successor] == 0:
ready.append(successor)
result.reverse()
return result
def __init__(
self,
*,
max_avg_history: int = None,
max_commit_latency_history: int = None,
max_send_latency_history: int = None,
max_assignment_latency_history: int = None,
messages_sent: int = 0,
tables: MutableMapping[str, TableState] = None,
messages_active: int = 0,
events_active: int = 0,
messages_received_total: int = 0,
messages_received_by_topic: Counter[str] = None,
events_total: int = 0,
events_by_stream: Counter[StreamT] = None,
events_by_task: Counter[asyncio.Task] = None,
events_runtime: Deque[float] = None,
commit_latency: Deque[float] = None,
send_latency: Deque[float] = None,
assignment_latency: Deque[float] = None,
events_s: int = 0,
messages_s: int = 0,
events_runtime_avg: float = 0.0,
topic_buffer_full: Counter[TP] = None,
rebalances: int = None,
rebalance_return_latency: Deque[float] = None,
rebalance_end_latency: Deque[float] = None,
rebalance_return_avg: float = 0.0,
rebalance_end_avg: float = 0.0,
time: Callable[[], float] = monotonic,
http_response_codes: Counter[HTTPStatus] = None,
http_response_latency: Deque[float] = None,
http_response_latency_avg: float = 0.0,
**kwargs: Any
) -> None:
if max_avg_history is not None:
self.max_avg_history = max_avg_history
if max_commit_latency_history is not None:
self.max_commit_latency_history = max_commit_latency_history
if max_send_latency_history is not None:
self.max_send_latency_history = max_send_latency_history
if max_assignment_latency_history is not None:
self.max_assignment_latency_history = max_assignment_latency_history
if rebalances is not None:
self.rebalances = rebalances
self.tables = {} if tables is None else tables
self.commit_latency = deque() if commit_latency is None else commit_latency
self.send_latency = deque() if send_latency is None else send_latency
self.assignment_latency = (
deque() if assignment_latency is None else assignment_latency
)
self.rebalance_return_latency = (
deque() if rebalance_return_latency is None else rebalance_return_latency
)
self.rebalance_end_latency = (
deque() if rebalance_end_latency is None else rebalance_end_latency
)
self.rebalance_return_avg = rebalance_return_avg
self.rebalance_end_avg = rebalance_end_avg
self.messages_active = messages_active
self.messages_received_total = messages_received_total
self.messages_received_by_topic = Counter()
self.messages_sent = messages_sent
self.messages_sent_by_topic = Counter()
self.messages_s = messages_s
self.events_active = events_active
self.events_total = events_total
self.events_by_task = Counter()
self.events_by_stream = Counter()
self.events_s = events_s
self.events_runtime_avg = events_runtime_avg
self.events_runtime = deque() if events_runtime is None else events_runtime
self.topic_buffer_full = Counter()
self.time: Callable[[], float] = time
self.http_response_codes = Counter()
self.http_response_latency = deque()
self.http_response_latency_avg = http_response_latency_avg
self.metric_counts = Counter()
self.tp_committed_offsets = {}
self.tp_read_offsets = {}
self.tp_end_offsets = {}
self.stream_inbound_time = {}
Service.__init__(self, **kwargs)
def _assigned_partition_counts(self, active: bool) -> Counter[int]:
return Counter(
partition
for copartitioned in self._client_assignments.values()
for partition in copartitioned.get_assigned_partitions(active)
)
