def __init__(self,
app: AppT,
*,
name: str = None,
default: Callable[[], Any] = None,
store: Union[str, URL] = None,
schema: SchemaT = None,
key_type: ModelArg = None,
value_type: ModelArg = None,
partitions: int = None,
window: WindowT = None,
changelog_topic: TopicT = None,
help: str = None,
on_recover: RecoverCallback = None,
on_changelog_event: ChangelogEventCallback = None,
recovery_buffer_size: int = 1000,
standby_buffer_size: int = None,
extra_topic_configs: Mapping[str, Any] = None,
recover_callbacks: Set[RecoverCallback] = None,
options: Mapping[str, Any] = None,
**kwargs: Any) -> None:
Service.__init__(self, **kwargs)
self.app = app
self.name = cast(str, name) # set lazily so CAN BE NONE!
self.default = default
self._store = URL(store) if store else None
self.schema = schema
self.key_type = key_type
self.value_type = value_type
self.partitions = partitions
self.window = window
self._changelog_topic = changelog_topic
self.extra_topic_configs = extra_topic_configs or {}
self.help = help or ''
self._on_changelog_event = on_changelog_event
self.recovery_buffer_size = recovery_buffer_size
self.standby_buffer_size = standby_buffer_size or recovery_buffer_size
assert self.recovery_buffer_size > 0 and self.standby_buffer_size > 0
self.options = options
# Setting Serializers from key_type and value_type
# Possible values json and raw
# Fallback to json
self.key_serializer = self._serializer_from_type(self.key_type)
self.value_serializer = self._serializer_from_type(self.value_type)
# Table key expiration
self._partition_timestamp_keys = defaultdict(set)
self._partition_timestamps = defaultdict(list)
self._partition_latest_timestamp = defaultdict(int)
self._recover_callbacks = set(recover_callbacks or [])
if on_recover:
self.on_recover(on_recover)
# Aliases
self._sensor_on_get = self.app.sensors.on_table_get
self._sensor_on_set = self.app.sensors.on_table_set
self._sensor_on_del = self.app.sensors.on_table_del
def __init__(self,
app: AppT,
url: Union[URL, str] = 'memory://',
**kwargs: Any) -> None:
self.app = app
self.url = URL(url)
Service.__init__(self, **kwargs)
def __init__(self,
dataPath: Path = None,
runtimeConfig=None,
dotPath=None,
app=None,
config=None,
parent=None):
self.app = app if app else runningApp()
self.rtCfg = runtimeConfig
self.dotPath = dotPath
self._config = config
self.sServiceStarted = AsyncSignal()
if self.dotPath is None and parent and parent.dotPath:
self.dotPath = f'{parent.dotPath}.{self.name}'
if dataPath:
self.rootPath = dataPath
else:
self.rootPath = self.app.dataPathForService(
self.dotPath if self.dotPath else 'tmp')
Service.__init__(self)
KeyListener.__init__(self)
self.pubsubServices = []
self.serviceConfigBind()
def __init__(self, app: AppT, **kwargs: Any) -> None:
self.app = app
self.views = {}
self.reverse_names = {}
blueprints = list(self.default_blueprints)
if self.app.conf.debug:
blueprints += self.debug_blueprints
self.blueprints = BlueprintManager(blueprints)
Service.__init__(self, **kwargs)
def __init__(self,
*,
max_avg_history: int = MAX_AVG_HISTORY,
max_commit_latency_history: int = MAX_COMMIT_LATENCY_HISTORY,
max_send_latency_history: int = MAX_SEND_LATENCY_HISTORY,
messages_sent: int = 0,
tables: MutableMapping[str, TableState] = None,
messages_active: int = 0,
events_active: int = 0,
messages_received_total: int = 0,
messages_received_by_topic: Counter[str] = None,
events_total: int = 0,
events_by_stream: Counter[StreamT] = None,
events_by_task: Counter[asyncio.Task] = None,
events_runtime: List[float] = None,
commit_latency: List[float] = None,
send_latency: List[float] = None,
events_s: int = 0,
messages_s: int = 0,
events_runtime_avg: float = 0.0,
topic_buffer_full: Counter[TopicT] = None,
**kwargs: Any) -> None:
self.max_avg_history = max_avg_history
self.max_commit_latency_history = max_commit_latency_history
self.max_send_latency_history = max_send_latency_history
self.tables = {} if tables is None else tables
self.commit_latency = [] if commit_latency is None else commit_latency
self.send_latency = [] if send_latency is None else send_latency
self.messages_active = messages_active
self.messages_received_total = messages_received_total
self.messages_received_by_topic = Counter()
self.messages_sent = messages_sent
self.messages_sent_by_topic = Counter()
self.messages_s = messages_s
self.events_active = events_active
self.events_total = events_total
self.events_by_task = Counter()
self.events_by_stream = Counter()
self.events_s = events_s
self.events_runtime_avg = events_runtime_avg
self.events_runtime = [] if events_runtime is None else events_runtime
self.topic_buffer_full = Counter()
self.time: Callable[[], float] = monotonic
self.metric_counts = Counter()
self.tp_committed_offsets = {}
self.tp_read_offsets = {}
self.tp_end_offsets = {}
Service.__init__(self, **kwargs)
def __init__(self,
agent: AgentT,
stream: StreamT,
it: _T,
index: int = None,
active_partitions: Set[TP] = None,
**kwargs: Any) -> None:
self.agent = agent
self.stream = stream
self.it = it
self.index = index
self.active_partitions = active_partitions
self.actor_task = None
Service.__init__(self, **kwargs)
def __init__(
self,
fun: AgentFun,
*,
app: AppT,
name: str = None,
channel: Union[str, ChannelT] = None,
concurrency: int = 1,
sink: Iterable[SinkT] = None,
on_error: AgentErrorHandler = None,
supervisor_strategy: Type[SupervisorStrategyT] = None,
help: str = None,
schema: SchemaT = None,
key_type: ModelArg = None,
value_type: ModelArg = None,
isolated_partitions: bool = False,
use_reply_headers: bool = None,
**kwargs: Any,
) -> None:
self.app = app
self.fun: AgentFun = fun
self.name = name or canonshortname(self.fun)
# key-type/value_type arguments only apply when a channel
# is not set
if schema is not None:
assert channel is None or isinstance(channel, str)
if key_type is not None:
assert channel is None or isinstance(channel, str)
self._key_type = key_type
if value_type is not None:
assert channel is None or isinstance(channel, str)
self._schema = schema
self._value_type = value_type
self._channel_arg = channel
self._channel_kwargs = kwargs
self.concurrency = concurrency or 1
self.isolated_partitions = isolated_partitions
self.help = help or ""
self._sinks = list(sink) if sink is not None else []
self._on_error: Optional[AgentErrorHandler] = on_error
self.supervisor_strategy = supervisor_strategy
self._actors = WeakSet()
self._actor_by_partition = WeakValueDictionary()
if self.isolated_partitions and self.concurrency > 1:
raise ImproperlyConfigured(
"Agent concurrency must be 1 when using isolated partitions"
)
self.use_reply_headers = use_reply_headers
Service.__init__(self)
def __init__(
self,
channel: AsyncIterator[T_co],
*,
app: AppT,
processors: Iterable[Processor[T]] = None,
combined: List[JoinableT] = None,
on_start: Callable = None,
join_strategy: JoinT = None,
beacon: NodeT = None,
concurrency_index: int = None,
prev: StreamT = None,
active_partitions: Set[TP] = None,
enable_acks: bool = True,
prefix: str = "",
loop: asyncio.AbstractEventLoop = None,
) -> None:
Service.__init__(self, loop=loop, beacon=beacon)
self.app = app
self.channel = channel
self.outbox = self.app.FlowControlQueue(
maxsize=self.app.conf.stream_buffer_maxsize,
loop=self.loop,
clear_on_resume=True,
)
self._passive_started = asyncio.Event(loop=self.loop)
self.join_strategy = join_strategy
self.combined = combined if combined is not None else []
self.concurrency_index = concurrency_index
self._prev = prev
self.active_partitions = active_partitions
self.enable_acks = enable_acks
self.prefix = prefix
self._processors = list(processors) if processors else []
self._on_start = on_start
# attach beacon to channel, or if iterable attach to current task.
task = current_task(loop=self.loop)
if task is not None:
self.task_owner = task
# Generate message handler
self._on_stream_event_in = self.app.sensors.on_stream_event_in
self._on_stream_event_out = self.app.sensors.on_stream_event_out
self._on_message_in = self.app.sensors.on_message_in
self._on_message_out = self.app.sensors.on_message_out
self._skipped_value = object()
def __init__(self, app: AppT, **kwargs: Any) -> None:
Service.__init__(self, **kwargs)
self.app = app
self._topics = set()
self._topic_name_index = defaultdict(set)
self._tp_index = defaultdict(set)
self._tp_to_callback = {}
self._acking_topics = set()
self._subscription_changed = None
self._subscription_done = None
self._compiler = ConductorCompiler()
# we compile the closure used for receive messages
# (this just optimizes symbol lookups, localizing variables etc).
self.on_message: ConsumerCallback
self.on_message = self._compile_message_handler()
def as_service(self,
loop: asyncio.AbstractEventLoop,
*args: Any, **kwargs: Any) -> Service:
return Service.from_awaitable(
self.execute(*args, **kwargs),
name=type(self).__name__,
loop=loop or asyncio.get_event_loop())
def as_service(self,
loop: asyncio.AbstractEventLoop,
*args: Any, **kwargs: Any) -> Service:
"""Wrap command in a :class:`mode.Service` object."""
return Service.from_awaitable(
self.execute(*args, **kwargs),
name=type(self).__name__,
loop=loop or asyncio.get_event_loop())
def __init__(self, app: AppT, **kwargs: Any) -> None:
Service.__init__(self, **kwargs)
self.app = app
self._topics = set()
self._topic_name_index = defaultdict(set)
self._tp_index = defaultdict(set)
self._tp_to_callback = {}
self._acking_topics = set()
self._subscription_changed = None
self._subscription_done = None
self._compiler = ConductorCompiler()
# This callback is called whenever the Consumer has
# fetched a new record from Kafka.
# We compile this down to a closure having variables already
# localized as an optimization.
self.on_message: ConsumerCallback
self.on_message = self._compile_message_handler()
def __init__(self,
url: Union[str, URL],
app: AppT,
*,
table_name: str = '',
key_type: ModelArg = None,
value_type: ModelArg = None,
key_serializer: CodecArg = 'json',
value_serializer: CodecArg = 'json',
**kwargs: Any) -> None:
Service.__init__(self, **kwargs)
self.url = URL(url)
self.app = app
self.table_name = table_name
self.key_type = key_type
self.value_type = value_type
self.key_serializer = key_serializer
self.value_serializer = value_serializer
async def test_interface():
s = Service()
s.on_init()
s.__post_init__()
await s.on_start()
await s.on_stop()
await s.on_shutdown()
def __init__(self,
url: Union[str, URL],
app: AppT,
table: CollectionT,
*,
table_name: str = '',
key_type: ModelArg = None,
value_type: ModelArg = None,
key_serializer: CodecArg = None,
value_serializer: CodecArg = None,
options: Mapping[str, Any] = None,
**kwargs: Any) -> None:
Service.__init__(self, **kwargs)
self.url = URL(url)
self.app = app
self.table = table
self.table_name = table_name or self.table.name
self.key_type = key_type
self.value_type = value_type
self.key_serializer = key_serializer
self.value_serializer = value_serializer
self.options = options
def __init__(
self,
*,
max_avg_history: int = None,
max_commit_latency_history: int = None,
max_send_latency_history: int = None,
max_assignment_latency_history: int = None,
messages_sent: int = 0,
tables: MutableMapping[str, TableState] = None,
messages_active: int = 0,
events_active: int = 0,
messages_received_total: int = 0,
messages_received_by_topic: Counter[str] = None,
events_total: int = 0,
events_by_stream: Counter[StreamT] = None,
events_by_task: Counter[asyncio.Task] = None,
events_runtime: Deque[float] = None,
commit_latency: Deque[float] = None,
send_latency: Deque[float] = None,
assignment_latency: Deque[float] = None,
events_s: int = 0,
messages_s: int = 0,
events_runtime_avg: float = 0.0,
topic_buffer_full: Counter[TP] = None,
rebalances: int = None,
rebalance_return_latency: Deque[float] = None,
rebalance_end_latency: Deque[float] = None,
rebalance_return_avg: float = 0.0,
rebalance_end_avg: float = 0.0,
time: Callable[[], float] = monotonic,
http_response_codes: Counter[HTTPStatus] = None,
http_response_latency: Deque[float] = None,
http_response_latency_avg: float = 0.0,
**kwargs: Any
) -> None:
if max_avg_history is not None:
self.max_avg_history = max_avg_history
if max_commit_latency_history is not None:
self.max_commit_latency_history = max_commit_latency_history
if max_send_latency_history is not None:
self.max_send_latency_history = max_send_latency_history
if max_assignment_latency_history is not None:
self.max_assignment_latency_history = max_assignment_latency_history
if rebalances is not None:
self.rebalances = rebalances
self.tables = {} if tables is None else tables
self.commit_latency = deque() if commit_latency is None else commit_latency
self.send_latency = deque() if send_latency is None else send_latency
self.assignment_latency = (
deque() if assignment_latency is None else assignment_latency
)
self.rebalance_return_latency = (
deque() if rebalance_return_latency is None else rebalance_return_latency
)
self.rebalance_end_latency = (
deque() if rebalance_end_latency is None else rebalance_end_latency
)
self.rebalance_return_avg = rebalance_return_avg
self.rebalance_end_avg = rebalance_end_avg
self.messages_active = messages_active
self.messages_received_total = messages_received_total
self.messages_received_by_topic = Counter()
self.messages_sent = messages_sent
self.messages_sent_by_topic = Counter()
self.messages_s = messages_s
self.events_active = events_active
self.events_total = events_total
self.events_by_task = Counter()
self.events_by_stream = Counter()
self.events_s = events_s
self.events_runtime_avg = events_runtime_avg
self.events_runtime = deque() if events_runtime is None else events_runtime
self.topic_buffer_full = Counter()
self.time: Callable[[], float] = time
self.http_response_codes = Counter()
self.http_response_latency = deque()
self.http_response_latency_avg = http_response_latency_avg
self.metric_counts = Counter()
self.tp_committed_offsets = {}
self.tp_read_offsets = {}
self.tp_end_offsets = {}
self.stream_inbound_time = {}
Service.__init__(self, **kwargs)
def __init__(self, app: AppT, **kwargs: Any) -> None:
Service.__init__(self, **kwargs)
self.app = app
def __init__(self, table: Table, **kwargs: Any) -> None:
self.table = table
self.table_name = self.table.name
self.data = {}
self._dirty = set()
Service.__init__(self, **kwargs)
def __init__(self, app: AppT, **kwargs: Any) -> None:
self.app = app
self.views = {}
self.reverse_names = {}
self.blueprints = BlueprintManager(self.default_blueprints)
Service.__init__(self, **kwargs)
def __init__(self, app: AppT, **kwargs: Any) -> None:
self.app = app
self.data = OrderedDict()
self._by_topic = defaultdict(WeakSet)
self._agents_started = Event()
Service.__init__(self, **kwargs)
def __init__(self, app: AppT, **kwargs: Any) -> None:
self.app = app
self.checks = {}
Service.__init__(self, **kwargs)
def test_constructor_aligns_beacons(self):
x = Service()
y = Service()
Worker(x, y)
