def __init__(
self,
_callable: Callable,
deployment_name: str,
replica_tag: ReplicaTag,
deployment_config: DeploymentConfig,
user_config: Any,
version: DeploymentVersion,
is_function: bool,
controller_handle: ActorHandle,
) -> None:
self.deployment_config = deployment_config
self.deployment_name = deployment_name
self.replica_tag = replica_tag
self.callable = _callable
self.is_function = is_function
self.user_config = user_config
self.version = version
self.rwlock = aiorwlock.RWLock()
user_health_check = getattr(_callable, HEALTH_CHECK_METHOD, None)
if not callable(user_health_check):
def user_health_check():
pass
self.user_health_check = sync_to_async(user_health_check)
self.num_ongoing_requests = 0
self.request_counter = metrics.Counter(
"serve_deployment_request_counter",
description=
("The number of queries that have been processed in this replica."
),
tag_keys=("deployment", "replica"),
)
self.request_counter.set_default_tags({
"deployment": self.deployment_name,
"replica": self.replica_tag
})
self.error_counter = metrics.Counter(
"serve_deployment_error_counter",
description=(
"The number of exceptions that have occurred in this replica."
),
tag_keys=("deployment", "replica"),
)
self.error_counter.set_default_tags({
"deployment": self.deployment_name,
"replica": self.replica_tag
})
self.restart_counter = metrics.Counter(
"serve_deployment_replica_starts",
description=
("The number of times this replica has been restarted due to failure."
),
tag_keys=("deployment", "replica"),
)
self.restart_counter.set_default_tags({
"deployment": self.deployment_name,
"replica": self.replica_tag
})
self.processing_latency_tracker = metrics.Histogram(
"serve_deployment_processing_latency_ms",
description="The latency for queries to be processed.",
boundaries=DEFAULT_LATENCY_BUCKET_MS,
tag_keys=("deployment", "replica"),
)
self.processing_latency_tracker.set_default_tags({
"deployment":
self.deployment_name,
"replica":
self.replica_tag
})
self.num_processing_items = metrics.Gauge(
"serve_replica_processing_queries",
description="The current number of queries being processed.",
tag_keys=("deployment", "replica"),
)
self.num_processing_items.set_default_tags({
"deployment": self.deployment_name,
"replica": self.replica_tag
})
self.restart_counter.inc()
self._shutdown_wait_loop_s = deployment_config.graceful_shutdown_wait_loop_s
if deployment_config.autoscaling_config:
process_remote_func = controller_handle.record_autoscaling_metrics.remote
config = deployment_config.autoscaling_config
start_metrics_pusher(
interval_s=config.metrics_interval_s,
collection_callback=self._collect_autoscaling_metrics,
metrics_process_func=process_remote_func,
)
# NOTE(edoakes): we used to recommend that users use the "ray" logger
# and tagged the logs with metadata as below. We now recommend using
# the "ray.serve" 'component logger' (as of Ray 1.13). This is left to
# maintain backwards compatibility with users who were using the
# existing logger. We can consider removing it in Ray 2.0.
ray_logger = logging.getLogger("ray")
for handler in ray_logger.handlers:
handler.setFormatter(
logging.Formatter(
handler.formatter._fmt +
f" component=serve deployment={self.deployment_name} "
f"replica={self.replica_tag}"))
def __init__(self, _callable: Callable, backend_config: BackendConfig,
is_function: bool, controller_handle: ActorHandle) -> None:
self.backend_tag = ray.serve.api.get_replica_context().backend_tag
self.replica_tag = ray.serve.api.get_replica_context().replica_tag
self.callable = _callable
self.is_function = is_function
self.config = backend_config
self.batch_queue = _BatchQueue(self.config.max_batch_size or 1,
self.config.batch_wait_timeout)
self.reconfigure(self.config.user_config)
self.num_ongoing_requests = 0
self.request_counter = metrics.Counter(
"serve_backend_request_counter",
description=("The number of queries that have been "
"processed in this replica."),
tag_keys=("backend", ))
self.request_counter.set_default_tags({"backend": self.backend_tag})
self.long_poll_client = LongPollAsyncClient(controller_handle, {
LongPollKey.BACKEND_CONFIGS: self._update_backend_configs,
})
self.error_counter = metrics.Counter(
"serve_backend_error_counter",
description=("The number of exceptions that have "
"occurred in the backend."),
tag_keys=("backend", ))
self.error_counter.set_default_tags({"backend": self.backend_tag})
self.restart_counter = metrics.Counter(
"serve_backend_replica_starts",
description=("The number of times this replica "
"has been restarted due to failure."),
tag_keys=("backend", "replica"))
self.restart_counter.set_default_tags({
"backend": self.backend_tag,
"replica": self.replica_tag
})
self.queuing_latency_tracker = metrics.Histogram(
"serve_backend_queuing_latency_ms",
description=("The latency for queries in the replica's queue "
"waiting to be processed or batched."),
boundaries=DEFAULT_LATENCY_BUCKET_MS,
tag_keys=("backend", "replica"))
self.queuing_latency_tracker.set_default_tags({
"backend": self.backend_tag,
"replica": self.replica_tag
})
self.processing_latency_tracker = metrics.Histogram(
"serve_backend_processing_latency_ms",
description="The latency for queries to be processed.",
boundaries=DEFAULT_LATENCY_BUCKET_MS,
tag_keys=("backend", "replica", "batch_size"))
self.processing_latency_tracker.set_default_tags({
"backend": self.backend_tag,
"replica": self.replica_tag
})
self.num_queued_items = metrics.Gauge(
"serve_replica_queued_queries",
description=("The current number of queries queued in "
"the backend replicas."),
tag_keys=("backend", "replica"))
self.num_queued_items.set_default_tags({
"backend": self.backend_tag,
"replica": self.replica_tag
})
self.num_processing_items = metrics.Gauge(
"serve_replica_processing_queries",
description="The current number of queries being processed.",
tag_keys=("backend", "replica"))
self.num_processing_items.set_default_tags({
"backend": self.backend_tag,
"replica": self.replica_tag
})
self.restart_counter.inc()
ray_logger = logging.getLogger("ray")
for handler in ray_logger.handlers:
handler.setFormatter(
logging.Formatter(
handler.formatter._fmt +
f" component=serve backend={self.backend_tag} "
f"replica={self.replica_tag}"))
asyncio.get_event_loop().create_task(self.main_loop())
def __init__(self, _callable: Callable, backend_config: BackendConfig,
is_function: bool, controller_handle: ActorHandle) -> None:
self.backend_tag = ray.serve.api.get_replica_context().deployment
self.replica_tag = ray.serve.api.get_replica_context().replica_tag
self.callable = _callable
self.is_function = is_function
self.config = backend_config
self.num_ongoing_requests = 0
self.request_counter = metrics.Counter(
"serve_deployment_request_counter",
description=("The number of queries that have been "
"processed in this replica."),
tag_keys=("deployment", "replica"))
self.request_counter.set_default_tags({
"deployment": self.backend_tag,
"replica": self.replica_tag
})
self.loop = asyncio.get_event_loop()
self.long_poll_client = LongPollClient(
controller_handle,
{
(LongPollNamespace.BACKEND_CONFIGS, self.backend_tag):
self._update_backend_configs,
},
call_in_event_loop=self.loop,
)
self.error_counter = metrics.Counter(
"serve_deployment_error_counter",
description=("The number of exceptions that have "
"occurred in this replica."),
tag_keys=("deployment", "replica"))
self.error_counter.set_default_tags({
"deployment": self.backend_tag,
"replica": self.replica_tag
})
self.restart_counter = metrics.Counter(
"serve_deployment_replica_starts",
description=("The number of times this replica "
"has been restarted due to failure."),
tag_keys=("deployment", "replica"))
self.restart_counter.set_default_tags({
"deployment": self.backend_tag,
"replica": self.replica_tag
})
self.processing_latency_tracker = metrics.Histogram(
"serve_deployment_processing_latency_ms",
description="The latency for queries to be processed.",
boundaries=DEFAULT_LATENCY_BUCKET_MS,
tag_keys=("deployment", "replica"))
self.processing_latency_tracker.set_default_tags({
"deployment":
self.backend_tag,
"replica":
self.replica_tag
})
self.num_processing_items = metrics.Gauge(
"serve_replica_processing_queries",
description="The current number of queries being processed.",
tag_keys=("deployment", "replica"))
self.num_processing_items.set_default_tags({
"deployment": self.backend_tag,
"replica": self.replica_tag
})
self.restart_counter.inc()
ray_logger = logging.getLogger("ray")
for handler in ray_logger.handlers:
handler.setFormatter(
logging.Formatter(
handler.formatter._fmt +
f" component=serve deployment={self.backend_tag} "
f"replica={self.replica_tag}"))
def __init__(
self,
_callable: Callable,
deployment_name: str,
replica_tag: ReplicaTag,
deployment_config: DeploymentConfig,
user_config: Any,
version: DeploymentVersion,
is_function: bool,
controller_handle: ActorHandle,
) -> None:
self.deployment_config = deployment_config
self.deployment_name = deployment_name
self.replica_tag = replica_tag
self.callable = _callable
self.is_function = is_function
self.user_config = user_config
self.version = version
self.rwlock = aiorwlock.RWLock()
user_health_check = getattr(_callable, HEALTH_CHECK_METHOD, None)
if not callable(user_health_check):
def user_health_check():
pass
self.user_health_check = sync_to_async(user_health_check)
self.num_ongoing_requests = 0
self.request_counter = metrics.Counter(
"serve_deployment_request_counter",
description=("The number of queries that have been "
"processed in this replica."),
tag_keys=("deployment", "replica"),
)
self.request_counter.set_default_tags({
"deployment": self.deployment_name,
"replica": self.replica_tag
})
self.error_counter = metrics.Counter(
"serve_deployment_error_counter",
description=("The number of exceptions that have "
"occurred in this replica."),
tag_keys=("deployment", "replica"),
)
self.error_counter.set_default_tags({
"deployment": self.deployment_name,
"replica": self.replica_tag
})
self.restart_counter = metrics.Counter(
"serve_deployment_replica_starts",
description=("The number of times this replica "
"has been restarted due to failure."),
tag_keys=("deployment", "replica"),
)
self.restart_counter.set_default_tags({
"deployment": self.deployment_name,
"replica": self.replica_tag
})
self.processing_latency_tracker = metrics.Histogram(
"serve_deployment_processing_latency_ms",
description="The latency for queries to be processed.",
boundaries=DEFAULT_LATENCY_BUCKET_MS,
tag_keys=("deployment", "replica"),
)
self.processing_latency_tracker.set_default_tags({
"deployment":
self.deployment_name,
"replica":
self.replica_tag
})
self.num_processing_items = metrics.Gauge(
"serve_replica_processing_queries",
description="The current number of queries being processed.",
tag_keys=("deployment", "replica"),
)
self.num_processing_items.set_default_tags({
"deployment": self.deployment_name,
"replica": self.replica_tag
})
self.restart_counter.inc()
self._shutdown_wait_loop_s = deployment_config.graceful_shutdown_wait_loop_s
if deployment_config.autoscaling_config:
config = deployment_config.autoscaling_config
start_metrics_pusher(
interval_s=config.metrics_interval_s,
collection_callback=self._collect_autoscaling_metrics,
controller_handle=controller_handle,
)
ray_logger = logging.getLogger("ray")
for handler in ray_logger.handlers:
handler.setFormatter(
logging.Formatter(
handler.formatter._fmt +
f" component=serve deployment={self.deployment_name} "
f"replica={self.replica_tag}"))
def __init__(self, backend_tag: str, replica_tag: str, _callable: Callable,
backend_config: BackendConfig, is_function: bool) -> None:
self.backend_tag = backend_tag
self.replica_tag = replica_tag
self.callable = _callable
self.is_function = is_function
self.config = backend_config
self.batch_queue = BatchQueue(self.config.max_batch_size or 1,
self.config.batch_wait_timeout)
self.num_ongoing_requests = 0
self.request_counter = metrics.Count(
"backend_request_counter",
description=("Number of queries that have been "
"processed in this replica"),
tag_keys=("backend", ))
self.request_counter.set_default_tags({"backend": self.backend_tag})
self.error_counter = metrics.Count(
"backend_error_counter",
description=("Number of exceptions that have "
"occurred in the backend"),
tag_keys=("backend", ))
self.error_counter.set_default_tags({"backend": self.backend_tag})
self.restart_counter = metrics.Count(
"backend_worker_starts",
description=("The number of time this replica workers "
"has been restarted due to failure."),
tag_keys=("backend", "replica_tag"))
self.restart_counter.set_default_tags({
"backend": self.backend_tag,
"replica_tag": self.replica_tag
})
self.queuing_latency_tracker = metrics.Histogram(
"backend_queuing_latency_ms",
description=(
"The latency for queries waiting in the replica's queue "
"waiting to be processed or batched."),
boundaries=DEFAULT_LATENCY_BUCKET_MS,
tag_keys=("backend", "replica_tag"))
self.queuing_latency_tracker.set_default_tags({
"backend": self.backend_tag,
"replica_tag": self.replica_tag
})
self.processing_latency_tracker = metrics.Histogram(
"backend_processing_latency_ms",
description="The latency for queries to be processed",
boundaries=DEFAULT_LATENCY_BUCKET_MS,
tag_keys=("backend", "replica_tag", "batch_size"))
self.processing_latency_tracker.set_default_tags({
"backend": self.backend_tag,
"replica_tag": self.replica_tag
})
self.num_queued_items = metrics.Gauge(
"replica_queued_queries",
description=("Current number of queries queued in the "
"the backend replicas"),
tag_keys=("backend", "replica_tag"))
self.num_queued_items.set_default_tags({
"backend": self.backend_tag,
"replica_tag": self.replica_tag
})
self.num_processing_items = metrics.Gauge(
"replica_processing_queries",
description="Current number of queries being processed",
tag_keys=("backend", "replica_tag"))
self.num_processing_items.set_default_tags({
"backend": self.backend_tag,
"replica_tag": self.replica_tag
})
self.restart_counter.record(1)
asyncio.get_event_loop().create_task(self.main_loop())
