def __init__(self, app: AppT, tables: TableManagerT, **kwargs: Any) -> None: self.app = app self.tables = cast(_TableManager, tables) self.standby_tps = set() self.active_tps = set() self.tp_to_table = {} self.active_offsets = Counter() self.standby_offsets = Counter() self.active_highwaters = Counter() self.standby_highwaters = Counter() self.completed = Event() self.buffers = defaultdict(list) self.buffer_sizes = {} self.recovery_delay = self.app.conf.stream_recovery_delay self.actives_for_table = defaultdict(set) self.standbys_for_table = defaultdict(set) self._active_events_received_at = {} self._standby_events_received_at = {} self._processing_times = deque() super().__init__(**kwargs)
def standby_remaining(self) -> Counter[TP]: highwaters = self.standby_highwaters offsets = self.standby_offsets return Counter({ tp: highwater - offsets[tp] for tp, highwater in highwaters.items() if highwater >= 0 and offsets[tp] >= 0 })
def active_remaining(self) -> Counter[TP]: highwaters = self.active_highwaters offsets = self.active_offsets return Counter({ tp: highwater - offsets[tp] for tp, highwater in highwaters.items() if highwater is not None and offsets[tp] is not None })
def standby_remaining(self) -> Counter[TP]: """Return counter of remaining changes by standby partition.""" highwaters = self.standby_highwaters offsets = self.standby_offsets return Counter({ tp: highwater - offsets[tp] for tp, highwater in highwaters.items() if highwater >= 0 and offsets[tp] >= 0 })
def active_remaining(self) -> Counter[TP]: """Return counter of remaining changes by active partition.""" highwaters = self.active_highwaters offsets = self.active_offsets return Counter({ tp: highwater - offsets[tp] for tp, highwater in highwaters.items() if highwater is not None and offsets[tp] is not None })
def __init__(self, expr: Type, root: 'RootNode' = None, *, user_types: CoercionMapping = None, date_parser: Callable[[Any], datetime] = None) -> None: assert self.type == NodeType.ROOT self.type_stats = Counter() self.user_types = user_types or {} self.extra_locals = {} self.date_parser: Callable[[Any], datetime] if date_parser is not None: self.date_parser = date_parser else: self.date_parser = parse_iso8601 self.found_types = defaultdict(set) super().__init__(expr, root=self)
def __init__(self, *, max_avg_history: int = MAX_AVG_HISTORY, max_commit_latency_history: int = MAX_COMMIT_LATENCY_HISTORY, max_send_latency_history: int = MAX_SEND_LATENCY_HISTORY, messages_sent: int = 0, tables: MutableMapping[str, TableState] = None, messages_active: int = 0, events_active: int = 0, messages_received_total: int = 0, messages_received_by_topic: Counter[str] = None, events_total: int = 0, events_by_stream: Counter[StreamT] = None, events_by_task: Counter[asyncio.Task] = None, events_runtime: List[float] = None, commit_latency: List[float] = None, send_latency: List[float] = None, events_s: int = 0, messages_s: int = 0, events_runtime_avg: float = 0.0, topic_buffer_full: Counter[TopicT] = None, **kwargs: Any) -> None: self.max_avg_history = max_avg_history self.max_commit_latency_history = max_commit_latency_history self.max_send_latency_history = max_send_latency_history self.tables = {} if tables is None else tables self.commit_latency = [] if commit_latency is None else commit_latency self.send_latency = [] if send_latency is None else send_latency self.messages_active = messages_active self.messages_received_total = messages_received_total self.messages_received_by_topic = Counter() self.messages_sent = messages_sent self.messages_sent_by_topic = Counter() self.messages_s = messages_s self.events_active = events_active self.events_total = events_total self.events_by_task = Counter() self.events_by_stream = Counter() self.events_s = events_s self.events_runtime_avg = events_runtime_avg self.events_runtime = [] if events_runtime is None else events_runtime self.topic_buffer_full = Counter() self.time: Callable[[], float] = monotonic self.metric_counts = Counter() self.tp_committed_offsets = {} self.tp_read_offsets = {} self.tp_end_offsets = {} Service.__init__(self, **kwargs)
def __init__(self, transport: TransportT, callback: ConsumerCallback, on_partitions_revoked: PartitionsRevokedCallback, on_partitions_assigned: PartitionsAssignedCallback, *, commit_interval: float = None, commit_livelock_soft_timeout: float = None, loop: asyncio.AbstractEventLoop = None, **kwargs: Any) -> None: assert callback is not None self.transport = transport self.app = self.transport.app self.in_transaction = self.app.in_transaction self.callback = callback self._on_message_in = self.app.sensors.on_message_in self._on_partitions_revoked = on_partitions_revoked self._on_partitions_assigned = on_partitions_assigned self._commit_every = self.app.conf.broker_commit_every self.scheduler = self.app.conf.ConsumerScheduler() self.commit_interval = (commit_interval or self.app.conf.broker_commit_interval) self.commit_livelock_soft_timeout = ( commit_livelock_soft_timeout or self.app.conf.broker_commit_livelock_soft_timeout) self._gap = defaultdict(list) self._acked = defaultdict(list) self._acked_index = defaultdict(set) self._read_offset = defaultdict(lambda: None) self._committed_offset = defaultdict(lambda: None) self._unacked_messages = WeakSet() self._waiting_for_ack = None self._time_start = monotonic() self._last_batch = Counter() self._end_offset_monitor_interval = self.commit_interval * 2 self.randomly_assigned_topics = set() self.can_resume_flow = Event() self._reset_state() super().__init__(loop=loop or self.transport.loop, **kwargs) self.transactions = self.transport.create_transaction_manager( consumer=self, producer=self.app.producer, beacon=self.beacon, loop=self.loop, )
def _khan62(self) -> Sequence: """Perform Khan's simple topological sort algorithm from '62. See https://en.wikipedia.org/wiki/Topological_sorting """ count: Counter[Any] = Counter() result = [] for node in self: for successor in self[node]: count[successor] += 1 ready = [node for node in self if not count[node]] while ready: node = ready.pop() result.append(node) for successor in self[node]: count[successor] -= 1 if count[successor] == 0: ready.append(successor) result.reverse() return result
def __init__( self, *, max_avg_history: int = None, max_commit_latency_history: int = None, max_send_latency_history: int = None, max_assignment_latency_history: int = None, messages_sent: int = 0, tables: MutableMapping[str, TableState] = None, messages_active: int = 0, events_active: int = 0, messages_received_total: int = 0, messages_received_by_topic: Counter[str] = None, events_total: int = 0, events_by_stream: Counter[StreamT] = None, events_by_task: Counter[asyncio.Task] = None, events_runtime: Deque[float] = None, commit_latency: Deque[float] = None, send_latency: Deque[float] = None, assignment_latency: Deque[float] = None, events_s: int = 0, messages_s: int = 0, events_runtime_avg: float = 0.0, topic_buffer_full: Counter[TP] = None, rebalances: int = None, rebalance_return_latency: Deque[float] = None, rebalance_end_latency: Deque[float] = None, rebalance_return_avg: float = 0.0, rebalance_end_avg: float = 0.0, time: Callable[[], float] = monotonic, http_response_codes: Counter[HTTPStatus] = None, http_response_latency: Deque[float] = None, http_response_latency_avg: float = 0.0, **kwargs: Any ) -> None: if max_avg_history is not None: self.max_avg_history = max_avg_history if max_commit_latency_history is not None: self.max_commit_latency_history = max_commit_latency_history if max_send_latency_history is not None: self.max_send_latency_history = max_send_latency_history if max_assignment_latency_history is not None: self.max_assignment_latency_history = max_assignment_latency_history if rebalances is not None: self.rebalances = rebalances self.tables = {} if tables is None else tables self.commit_latency = deque() if commit_latency is None else commit_latency self.send_latency = deque() if send_latency is None else send_latency self.assignment_latency = ( deque() if assignment_latency is None else assignment_latency ) self.rebalance_return_latency = ( deque() if rebalance_return_latency is None else rebalance_return_latency ) self.rebalance_end_latency = ( deque() if rebalance_end_latency is None else rebalance_end_latency ) self.rebalance_return_avg = rebalance_return_avg self.rebalance_end_avg = rebalance_end_avg self.messages_active = messages_active self.messages_received_total = messages_received_total self.messages_received_by_topic = Counter() self.messages_sent = messages_sent self.messages_sent_by_topic = Counter() self.messages_s = messages_s self.events_active = events_active self.events_total = events_total self.events_by_task = Counter() self.events_by_stream = Counter() self.events_s = events_s self.events_runtime_avg = events_runtime_avg self.events_runtime = deque() if events_runtime is None else events_runtime self.topic_buffer_full = Counter() self.time: Callable[[], float] = time self.http_response_codes = Counter() self.http_response_latency = deque() self.http_response_latency_avg = http_response_latency_avg self.metric_counts = Counter() self.tp_committed_offsets = {} self.tp_read_offsets = {} self.tp_end_offsets = {} self.stream_inbound_time = {} Service.__init__(self, **kwargs)
def _assigned_partition_counts(self, active: bool) -> Counter[int]: return Counter( partition for copartitioned in self._client_assignments.values() for partition in copartitioned.get_assigned_partitions(active) )