def __init__(self, app: AppT, *, name: str = None, default: Callable[[], Any] = None, store: Union[str, URL] = None, schema: SchemaT = None, key_type: ModelArg = None, value_type: ModelArg = None, partitions: int = None, window: WindowT = None, changelog_topic: TopicT = None, help: str = None, on_recover: RecoverCallback = None, on_changelog_event: ChangelogEventCallback = None, recovery_buffer_size: int = 1000, standby_buffer_size: int = None, extra_topic_configs: Mapping[str, Any] = None, recover_callbacks: Set[RecoverCallback] = None, options: Mapping[str, Any] = None, **kwargs: Any) -> None: Service.__init__(self, **kwargs) self.app = app self.name = cast(str, name) # set lazily so CAN BE NONE! self.default = default self._store = URL(store) if store else None self.schema = schema self.key_type = key_type self.value_type = value_type self.partitions = partitions self.window = window self._changelog_topic = changelog_topic self.extra_topic_configs = extra_topic_configs or {} self.help = help or '' self._on_changelog_event = on_changelog_event self.recovery_buffer_size = recovery_buffer_size self.standby_buffer_size = standby_buffer_size or recovery_buffer_size assert self.recovery_buffer_size > 0 and self.standby_buffer_size > 0 self.options = options # Setting Serializers from key_type and value_type # Possible values json and raw # Fallback to json self.key_serializer = self._serializer_from_type(self.key_type) self.value_serializer = self._serializer_from_type(self.value_type) # Table key expiration self._partition_timestamp_keys = defaultdict(set) self._partition_timestamps = defaultdict(list) self._partition_latest_timestamp = defaultdict(int) self._recover_callbacks = set(recover_callbacks or []) if on_recover: self.on_recover(on_recover) # Aliases self._sensor_on_get = self.app.sensors.on_table_get self._sensor_on_set = self.app.sensors.on_table_set self._sensor_on_del = self.app.sensors.on_table_del
def __init__(self, app: AppT, url: Union[URL, str] = 'memory://', **kwargs: Any) -> None: self.app = app self.url = URL(url) Service.__init__(self, **kwargs)
def __init__(self, dataPath: Path = None, runtimeConfig=None, dotPath=None, app=None, config=None, parent=None): self.app = app if app else runningApp() self.rtCfg = runtimeConfig self.dotPath = dotPath self._config = config self.sServiceStarted = AsyncSignal() if self.dotPath is None and parent and parent.dotPath: self.dotPath = f'{parent.dotPath}.{self.name}' if dataPath: self.rootPath = dataPath else: self.rootPath = self.app.dataPathForService( self.dotPath if self.dotPath else 'tmp') Service.__init__(self) KeyListener.__init__(self) self.pubsubServices = [] self.serviceConfigBind()
def __init__(self, app: AppT, **kwargs: Any) -> None: self.app = app self.views = {} self.reverse_names = {} blueprints = list(self.default_blueprints) if self.app.conf.debug: blueprints += self.debug_blueprints self.blueprints = BlueprintManager(blueprints) Service.__init__(self, **kwargs)
def __init__(self, *, max_avg_history: int = MAX_AVG_HISTORY, max_commit_latency_history: int = MAX_COMMIT_LATENCY_HISTORY, max_send_latency_history: int = MAX_SEND_LATENCY_HISTORY, messages_sent: int = 0, tables: MutableMapping[str, TableState] = None, messages_active: int = 0, events_active: int = 0, messages_received_total: int = 0, messages_received_by_topic: Counter[str] = None, events_total: int = 0, events_by_stream: Counter[StreamT] = None, events_by_task: Counter[asyncio.Task] = None, events_runtime: List[float] = None, commit_latency: List[float] = None, send_latency: List[float] = None, events_s: int = 0, messages_s: int = 0, events_runtime_avg: float = 0.0, topic_buffer_full: Counter[TopicT] = None, **kwargs: Any) -> None: self.max_avg_history = max_avg_history self.max_commit_latency_history = max_commit_latency_history self.max_send_latency_history = max_send_latency_history self.tables = {} if tables is None else tables self.commit_latency = [] if commit_latency is None else commit_latency self.send_latency = [] if send_latency is None else send_latency self.messages_active = messages_active self.messages_received_total = messages_received_total self.messages_received_by_topic = Counter() self.messages_sent = messages_sent self.messages_sent_by_topic = Counter() self.messages_s = messages_s self.events_active = events_active self.events_total = events_total self.events_by_task = Counter() self.events_by_stream = Counter() self.events_s = events_s self.events_runtime_avg = events_runtime_avg self.events_runtime = [] if events_runtime is None else events_runtime self.topic_buffer_full = Counter() self.time: Callable[[], float] = monotonic self.metric_counts = Counter() self.tp_committed_offsets = {} self.tp_read_offsets = {} self.tp_end_offsets = {} Service.__init__(self, **kwargs)
def __init__(self, agent: AgentT, stream: StreamT, it: _T, index: int = None, active_partitions: Set[TP] = None, **kwargs: Any) -> None: self.agent = agent self.stream = stream self.it = it self.index = index self.active_partitions = active_partitions self.actor_task = None Service.__init__(self, **kwargs)
def __init__( self, fun: AgentFun, *, app: AppT, name: str = None, channel: Union[str, ChannelT] = None, concurrency: int = 1, sink: Iterable[SinkT] = None, on_error: AgentErrorHandler = None, supervisor_strategy: Type[SupervisorStrategyT] = None, help: str = None, schema: SchemaT = None, key_type: ModelArg = None, value_type: ModelArg = None, isolated_partitions: bool = False, use_reply_headers: bool = None, **kwargs: Any, ) -> None: self.app = app self.fun: AgentFun = fun self.name = name or canonshortname(self.fun) # key-type/value_type arguments only apply when a channel # is not set if schema is not None: assert channel is None or isinstance(channel, str) if key_type is not None: assert channel is None or isinstance(channel, str) self._key_type = key_type if value_type is not None: assert channel is None or isinstance(channel, str) self._schema = schema self._value_type = value_type self._channel_arg = channel self._channel_kwargs = kwargs self.concurrency = concurrency or 1 self.isolated_partitions = isolated_partitions self.help = help or "" self._sinks = list(sink) if sink is not None else [] self._on_error: Optional[AgentErrorHandler] = on_error self.supervisor_strategy = supervisor_strategy self._actors = WeakSet() self._actor_by_partition = WeakValueDictionary() if self.isolated_partitions and self.concurrency > 1: raise ImproperlyConfigured( "Agent concurrency must be 1 when using isolated partitions" ) self.use_reply_headers = use_reply_headers Service.__init__(self)
def __init__( self, channel: AsyncIterator[T_co], *, app: AppT, processors: Iterable[Processor[T]] = None, combined: List[JoinableT] = None, on_start: Callable = None, join_strategy: JoinT = None, beacon: NodeT = None, concurrency_index: int = None, prev: StreamT = None, active_partitions: Set[TP] = None, enable_acks: bool = True, prefix: str = "", loop: asyncio.AbstractEventLoop = None, ) -> None: Service.__init__(self, loop=loop, beacon=beacon) self.app = app self.channel = channel self.outbox = self.app.FlowControlQueue( maxsize=self.app.conf.stream_buffer_maxsize, loop=self.loop, clear_on_resume=True, ) self._passive_started = asyncio.Event(loop=self.loop) self.join_strategy = join_strategy self.combined = combined if combined is not None else [] self.concurrency_index = concurrency_index self._prev = prev self.active_partitions = active_partitions self.enable_acks = enable_acks self.prefix = prefix self._processors = list(processors) if processors else [] self._on_start = on_start # attach beacon to channel, or if iterable attach to current task. task = current_task(loop=self.loop) if task is not None: self.task_owner = task # Generate message handler self._on_stream_event_in = self.app.sensors.on_stream_event_in self._on_stream_event_out = self.app.sensors.on_stream_event_out self._on_message_in = self.app.sensors.on_message_in self._on_message_out = self.app.sensors.on_message_out self._skipped_value = object()
def __init__(self, app: AppT, **kwargs: Any) -> None: Service.__init__(self, **kwargs) self.app = app self._topics = set() self._topic_name_index = defaultdict(set) self._tp_index = defaultdict(set) self._tp_to_callback = {} self._acking_topics = set() self._subscription_changed = None self._subscription_done = None self._compiler = ConductorCompiler() # we compile the closure used for receive messages # (this just optimizes symbol lookups, localizing variables etc). self.on_message: ConsumerCallback self.on_message = self._compile_message_handler()
def as_service(self, loop: asyncio.AbstractEventLoop, *args: Any, **kwargs: Any) -> Service: return Service.from_awaitable( self.execute(*args, **kwargs), name=type(self).__name__, loop=loop or asyncio.get_event_loop())
def as_service(self, loop: asyncio.AbstractEventLoop, *args: Any, **kwargs: Any) -> Service: """Wrap command in a :class:`mode.Service` object.""" return Service.from_awaitable( self.execute(*args, **kwargs), name=type(self).__name__, loop=loop or asyncio.get_event_loop())
def __init__(self, app: AppT, **kwargs: Any) -> None: Service.__init__(self, **kwargs) self.app = app self._topics = set() self._topic_name_index = defaultdict(set) self._tp_index = defaultdict(set) self._tp_to_callback = {} self._acking_topics = set() self._subscription_changed = None self._subscription_done = None self._compiler = ConductorCompiler() # This callback is called whenever the Consumer has # fetched a new record from Kafka. # We compile this down to a closure having variables already # localized as an optimization. self.on_message: ConsumerCallback self.on_message = self._compile_message_handler()
def __init__(self, url: Union[str, URL], app: AppT, *, table_name: str = '', key_type: ModelArg = None, value_type: ModelArg = None, key_serializer: CodecArg = 'json', value_serializer: CodecArg = 'json', **kwargs: Any) -> None: Service.__init__(self, **kwargs) self.url = URL(url) self.app = app self.table_name = table_name self.key_type = key_type self.value_type = value_type self.key_serializer = key_serializer self.value_serializer = value_serializer
async def test_interface(): s = Service() s.on_init() s.__post_init__() await s.on_start() await s.on_stop() await s.on_shutdown()
def __init__(self, url: Union[str, URL], app: AppT, table: CollectionT, *, table_name: str = '', key_type: ModelArg = None, value_type: ModelArg = None, key_serializer: CodecArg = None, value_serializer: CodecArg = None, options: Mapping[str, Any] = None, **kwargs: Any) -> None: Service.__init__(self, **kwargs) self.url = URL(url) self.app = app self.table = table self.table_name = table_name or self.table.name self.key_type = key_type self.value_type = value_type self.key_serializer = key_serializer self.value_serializer = value_serializer self.options = options
def __init__( self, *, max_avg_history: int = None, max_commit_latency_history: int = None, max_send_latency_history: int = None, max_assignment_latency_history: int = None, messages_sent: int = 0, tables: MutableMapping[str, TableState] = None, messages_active: int = 0, events_active: int = 0, messages_received_total: int = 0, messages_received_by_topic: Counter[str] = None, events_total: int = 0, events_by_stream: Counter[StreamT] = None, events_by_task: Counter[asyncio.Task] = None, events_runtime: Deque[float] = None, commit_latency: Deque[float] = None, send_latency: Deque[float] = None, assignment_latency: Deque[float] = None, events_s: int = 0, messages_s: int = 0, events_runtime_avg: float = 0.0, topic_buffer_full: Counter[TP] = None, rebalances: int = None, rebalance_return_latency: Deque[float] = None, rebalance_end_latency: Deque[float] = None, rebalance_return_avg: float = 0.0, rebalance_end_avg: float = 0.0, time: Callable[[], float] = monotonic, http_response_codes: Counter[HTTPStatus] = None, http_response_latency: Deque[float] = None, http_response_latency_avg: float = 0.0, **kwargs: Any ) -> None: if max_avg_history is not None: self.max_avg_history = max_avg_history if max_commit_latency_history is not None: self.max_commit_latency_history = max_commit_latency_history if max_send_latency_history is not None: self.max_send_latency_history = max_send_latency_history if max_assignment_latency_history is not None: self.max_assignment_latency_history = max_assignment_latency_history if rebalances is not None: self.rebalances = rebalances self.tables = {} if tables is None else tables self.commit_latency = deque() if commit_latency is None else commit_latency self.send_latency = deque() if send_latency is None else send_latency self.assignment_latency = ( deque() if assignment_latency is None else assignment_latency ) self.rebalance_return_latency = ( deque() if rebalance_return_latency is None else rebalance_return_latency ) self.rebalance_end_latency = ( deque() if rebalance_end_latency is None else rebalance_end_latency ) self.rebalance_return_avg = rebalance_return_avg self.rebalance_end_avg = rebalance_end_avg self.messages_active = messages_active self.messages_received_total = messages_received_total self.messages_received_by_topic = Counter() self.messages_sent = messages_sent self.messages_sent_by_topic = Counter() self.messages_s = messages_s self.events_active = events_active self.events_total = events_total self.events_by_task = Counter() self.events_by_stream = Counter() self.events_s = events_s self.events_runtime_avg = events_runtime_avg self.events_runtime = deque() if events_runtime is None else events_runtime self.topic_buffer_full = Counter() self.time: Callable[[], float] = time self.http_response_codes = Counter() self.http_response_latency = deque() self.http_response_latency_avg = http_response_latency_avg self.metric_counts = Counter() self.tp_committed_offsets = {} self.tp_read_offsets = {} self.tp_end_offsets = {} self.stream_inbound_time = {} Service.__init__(self, **kwargs)
def __init__(self, app: AppT, **kwargs: Any) -> None: Service.__init__(self, **kwargs) self.app = app
def __init__(self, table: Table, **kwargs: Any) -> None: self.table = table self.table_name = self.table.name self.data = {} self._dirty = set() Service.__init__(self, **kwargs)
def __init__(self, app: AppT, **kwargs: Any) -> None: self.app = app self.views = {} self.reverse_names = {} self.blueprints = BlueprintManager(self.default_blueprints) Service.__init__(self, **kwargs)
def __init__(self, app: AppT, **kwargs: Any) -> None: self.app = app self.data = OrderedDict() self._by_topic = defaultdict(WeakSet) self._agents_started = Event() Service.__init__(self, **kwargs)
def __init__(self, app: AppT, **kwargs: Any) -> None: self.app = app self.checks = {} Service.__init__(self, **kwargs)
def test_constructor_aligns_beacons(self): x = Service() y = Service() Worker(x, y)