def table_cleanup_interval(self, value: Seconds) -> None: self._table_cleanup_interval = want_seconds(value)
def broker_commit_interval(self, value: Seconds) -> None: self._broker_commit_interval = want_seconds(value)
def broker_commit_livelock_soft_timeout(self, value: Seconds) -> None: self._broker_commit_livelock_soft_timeout = want_seconds(value)
def broker_session_timeout(self, value: Seconds) -> None: self._broker_session_timeout = want_seconds(value)
def broker_heartbeat_interval(self, value: Seconds) -> None: self._broker_heartbeat_interval = want_seconds(value)
#: Used as the default value for :setting:`broker_commit_every`. BROKER_COMMIT_EVERY = 10_000 #: How often we commit acknowledged messages on a timer. #: Used as the default value for :setting:`broker_commit_interval`. BROKER_COMMIT_INTERVAL = 2.8 #: Kafka consumer session timeout (``session_timeout_ms``). BROKER_SESSION_TIMEOUT = 30.0 #: Kafka consumer heartbeat (``heartbeat_interval_ms``). BROKER_HEARTBEAT_INTERVAL = 3.0 #: How long time it takes before we warn that the commit offset has #: not advanced. BROKER_LIVELOCK_SOFT = want_seconds(timedelta(minutes=5)) #: How often we clean up expired items in windowed tables. #: Used as the default value for :setting:`table_cleanup_interval`. TABLE_CLEANUP_INTERVAL = 30.0 #: Prefix used for reply topics. REPLY_TO_PREFIX = 'f-reply-' #: Default expiry time for replies, in seconds (float). REPLY_EXPIRES = want_seconds(timedelta(days=1)) #: Max number of messages channels/streams/topics can "prefetch". STREAM_BUFFER_MAXSIZE = 4096 #: We buffer up sending messages until the
def __init__( # noqa: C901 self, id: str, *, version: int = None, broker: Union[str, URL] = None, broker_client_id: str = None, broker_commit_every: int = None, broker_commit_interval: Seconds = None, broker_commit_livelock_soft_timeout: Seconds = None, broker_session_timeout: Seconds = None, broker_heartbeat_interval: Seconds = None, broker_check_crcs: bool = None, agent_supervisor: SymbolArg[Type[SupervisorStrategyT]] = None, store: Union[str, URL] = None, autodiscover: AutodiscoverArg = None, origin: str = None, canonical_url: Union[str, URL] = None, datadir: Union[Path, str] = None, tabledir: Union[Path, str] = None, key_serializer: CodecArg = None, value_serializer: CodecArg = None, loghandlers: List[logging.StreamHandler] = None, table_cleanup_interval: Seconds = None, table_standby_replicas: int = None, topic_replication_factor: int = None, topic_partitions: int = None, id_format: str = None, reply_to: str = None, reply_to_prefix: str = None, reply_create_topic: bool = None, reply_expires: Seconds = None, ssl_context: ssl.SSLContext = None, stream_buffer_maxsize: int = None, stream_wait_empty: bool = None, stream_ack_cancelled_tasks: bool = None, stream_ack_exceptions: bool = None, stream_publish_on_commit: bool = None, producer_linger_ms: int = None, producer_max_batch_size: int = None, producer_acks: int = None, producer_max_request_size: int = None, producer_compression_type: str = None, worker_redirect_stdouts: bool = None, worker_redirect_stdouts_level: Severity = None, Agent: SymbolArg[Type[AgentT]] = None, Stream: SymbolArg[Type[StreamT]] = None, Table: SymbolArg[Type[TableT]] = None, TableManager: SymbolArg[Type[TableManagerT]] = None, Serializers: SymbolArg[Type[RegistryT]] = None, Worker: SymbolArg[Type[WorkerT]] = None, PartitionAssignor: SymbolArg[Type[PartitionAssignorT]] = None, LeaderAssignor: SymbolArg[Type[LeaderAssignorT]] = None, Router: SymbolArg[Type[RouterT]] = None, Topic: SymbolArg[Type[TopicT]] = None, HttpClient: SymbolArg[Type[HttpClientT]] = None, Monitor: SymbolArg[Type[SensorT]] = None, # XXX backward compat (remove for Faust 1.0) url: Union[str, URL] = None, **kwargs: Any) -> None: self.version = version if version is not None else self._version self.id_format = id_format if id_format is not None else self.id_format self.origin = origin if origin is not None else self.origin self.id = id self.broker = url or broker or BROKER_URL self.ssl_context = ssl_context self.store = store or STORE_URL if autodiscover is not None: self.autodiscover = autodiscover if broker_client_id is not None: self.broker_client_id = broker_client_id self.canonical_url = canonical_url or '' # datadir is a format string that can contain e.g. {conf.id} self.datadir = datadir or DATADIR self.tabledir = tabledir or TABLEDIR self.broker_commit_interval = (broker_commit_interval or self._broker_commit_interval) self.broker_commit_livelock_soft_timeout = ( broker_commit_livelock_soft_timeout or self._broker_commit_livelock_soft_timeout) if broker_session_timeout is not None: self.broker_session_timeout = want_seconds(broker_session_timeout) if broker_heartbeat_interval is not None: self.broker_heartbeat_interval = want_seconds( broker_heartbeat_interval) self.table_cleanup_interval = (table_cleanup_interval or self._table_cleanup_interval) if broker_commit_every is not None: self.broker_commit_every = broker_commit_every if broker_check_crcs is not None: self.broker_check_crcs = broker_check_crcs if key_serializer is not None: self.key_serializer = key_serializer if value_serializer is not None: self.value_serializer = value_serializer if table_standby_replicas is not None: self.table_standby_replicas = table_standby_replicas if topic_replication_factor is not None: self.topic_replication_factor = topic_replication_factor if topic_partitions is not None: self.topic_partitions = topic_partitions if reply_create_topic is not None: self.reply_create_topic = reply_create_topic self.loghandlers = loghandlers if loghandlers is not None else [] if stream_buffer_maxsize is not None: self.stream_buffer_maxsize = stream_buffer_maxsize if stream_wait_empty is not None: self.stream_wait_empty = stream_wait_empty if stream_ack_cancelled_tasks is not None: self.stream_ack_cancelled_tasks = stream_ack_cancelled_tasks if stream_ack_exceptions is not None: self.stream_ack_exceptions = stream_ack_exceptions if stream_publish_on_commit is not None: self.stream_publish_on_commit = stream_publish_on_commit if producer_linger_ms is not None: self.producer_linger_ms = producer_linger_ms if producer_max_batch_size is not None: self.producer_max_batch_size = producer_max_batch_size if producer_acks is not None: self.producer_acks = producer_acks if producer_max_request_size is not None: self.producer_max_request_size = producer_max_request_size if producer_compression_type is not None: self.producer_compression_type = producer_compression_type if worker_redirect_stdouts is not None: self.worker_redirect_stdouts = worker_redirect_stdouts if worker_redirect_stdouts_level is not None: self.worker_redirect_stdouts_level = worker_redirect_stdouts_level if reply_to_prefix is not None: self.reply_to_prefix = reply_to_prefix if reply_to is not None: self.reply_to = reply_to else: self.reply_to = f'{self.reply_to_prefix}{uuid4()}' if reply_expires is not None: self.reply_expires = reply_expires self.agent_supervisor = agent_supervisor or AGENT_SUPERVISOR_TYPE self.Agent = Agent or AGENT_TYPE self.Stream = Stream or STREAM_TYPE self.Table = Table or TABLE_TYPE self.Set = Set or SET_TYPE self.TableManager = TableManager or TABLE_MANAGER_TYPE self.Serializers = Serializers or REGISTRY_TYPE self.Worker = Worker or WORKER_TYPE self.PartitionAssignor = PartitionAssignor or PARTITION_ASSIGNOR_TYPE self.LeaderAssignor = LeaderAssignor or LEADER_ASSIGNOR_TYPE self.Router = Router or ROUTER_TYPE self.Topic = Topic or TOPIC_TYPE self.HttpClient = HttpClient or HTTP_CLIENT_TYPE self.Monitor = Monitor or MONITOR_TYPE self.__dict__.update(kwargs) # arbitrary configuration
def to_python(self, conf: _Settings, value: _Seconds) -> float: return want_seconds(value)
async def take(self, max_: int, within: Seconds) -> AsyncIterable[Sequence[T_co]]: """Buffer n values at a time and yield a list of buffered values. Arguments: within: Timeout for when we give up waiting for another value, and process the values we have. Warning: If there's no timeout (i.e. `timeout=None`), the agent is likely to stall and block buffered events for an unreasonable length of time(!). """ buffer: List[T_co] = [] events: List[EventT] = [] buffer_add = buffer.append event_add = events.append buffer_size = buffer.__len__ buffer_full = asyncio.Event(loop=self.loop) buffer_consumed = asyncio.Event(loop=self.loop) timeout = want_seconds(within) if within else None stream_enable_acks: bool = self.enable_acks buffer_consuming: Optional[asyncio.Future] = None channel_it = aiter(self.channel) # We add this processor to populate the buffer, and the stream # is passively consumed in the background (enable_passive below). async def add_to_buffer(value: T) -> T: try: # buffer_consuming is set when consuming buffer after timeout. nonlocal buffer_consuming if buffer_consuming is not None: try: await buffer_consuming finally: buffer_consuming = None buffer_add(cast(T_co, value)) event = self.current_event if event is not None: event_add(event) if buffer_size() >= max_: # signal that the buffer is full and should be emptied. buffer_full.set() # strict wait for buffer to be consumed after buffer full. # If max is 1000, we are not allowed to return 1001 values. buffer_consumed.clear() await self.wait(buffer_consumed) except Exception as exc: self.log.exception('Error adding to take buffer: %r', exc) await self.crash(exc) return value # Disable acks to ensure this method acks manually # events only after they are consumed by the user self.enable_acks = False self.add_processor(add_to_buffer) self._enable_passive(cast(ChannelT, channel_it)) try: while not self.should_stop: # wait until buffer full, or timeout await self.wait_for_stopped(buffer_full, timeout=timeout) if buffer: # make sure background thread does not add new times to # budfer while we read. buffer_consuming = self.loop.create_future() try: yield list(buffer) finally: buffer.clear() for event in events: await self.ack(event) events.clear() # allow writing to buffer again notify(buffer_consuming) buffer_full.clear() buffer_consumed.set() finally: # Restore last behaviour of "enable_acks" self.enable_acks = stream_enable_acks self._processors.remove(add_to_buffer)
async def get(self, *, timeout: Seconds = None) -> Any: timeout_: float = want_seconds(timeout) if timeout_: return await asyncio.wait_for(self.queue.get(), timeout=timeout_) return await self.queue.get()
def stream_recovery_delay(self, delay: Seconds) -> None: self._stream_recovery_delay = want_seconds(delay)
def reply_expires(self, reply_expires: Seconds) -> None: self._reply_expires = want_seconds(reply_expires)