def test_custom_metrics_edge_cases(metric_mock):
# None or empty boundaries are not allowed.
with pytest.raises(ValueError):
Histogram("hist")
with pytest.raises(ValueError):
Histogram("hist", boundaries=[])
# Empty name is not allowed.
with pytest.raises(ValueError):
Counter("")
# The tag keys must be a tuple type.
with pytest.raises(TypeError):
Counter("name", tag_keys=("a"))
def f():
counter = Counter("test_counter", description="desc")
counter.inc()
counter = ray.get(ray.put(counter)) # Test serialization.
counter.inc()
counter.inc(2)
ray.get(worker_should_exit.wait.remote())
def test_basic_custom_metrics(metric_mock):
# Make sure each of metric works as expected.
# -- Counter --
count = Counter("count", tag_keys=("a", ))
with pytest.raises(TypeError):
count.inc("hi")
with pytest.raises(ValueError):
count.inc(0)
with pytest.raises(ValueError):
count.inc(-1)
count._metric = metric_mock
count.inc(1, {"a": "1"})
metric_mock.record.assert_called_with(1, tags={"a": "1"})
# -- Gauge --
gauge = Gauge("gauge", description="gauge")
gauge._metric = metric_mock
gauge.record(4)
metric_mock.record.assert_called_with(4, tags={})
# -- Histogram
histogram = Histogram(
"hist", description="hist", boundaries=[1.0, 3.0], tag_keys=("a", "b"))
histogram._metric = metric_mock
tags = {"a": "10", "b": "b"}
histogram.observe(8, tags=tags)
metric_mock.record.assert_called_with(8, tags=tags)
def _setup_cluster_for_test(ray_start_cluster):
NUM_NODES = 2
cluster = ray_start_cluster
# Add a head node.
cluster.add_node(_system_config={"metrics_report_interval_ms": 1000})
# Add worker nodes.
[cluster.add_node() for _ in range(NUM_NODES - 1)]
cluster.wait_for_nodes()
ray.init(address=cluster.address)
worker_should_exit = SignalActor.remote()
# Generate a metric in the driver.
counter = Counter("test_driver_counter", description="desc")
counter.inc()
# Generate some metrics from actor & tasks.
@ray.remote
def f():
counter = Counter("test_counter", description="desc")
counter.inc()
counter = ray.get(ray.put(counter)) # Test serialization.
counter.inc()
counter.inc(2)
ray.get(worker_should_exit.wait.remote())
# Generate some metrics for the placement group.
pg = ray.util.placement_group(bundles=[{"CPU": 1}])
ray.get(pg.ready())
print(ray.util.placement_group_table())
ray.util.remove_placement_group(pg)
@ray.remote
class A:
async def ping(self):
histogram = Histogram("test_histogram",
description="desc",
boundaries=[0.1, 1.6])
histogram = ray.get(ray.put(histogram)) # Test serialization.
histogram.record(1.5)
ray.get(worker_should_exit.wait.remote())
a = A.remote()
obj_refs = [f.remote(), a.ping.remote()]
node_info_list = ray.nodes()
prom_addresses = []
for node_info in node_info_list:
metrics_export_port = node_info["MetricsExportPort"]
addr = node_info["NodeManagerAddress"]
prom_addresses.append(f"{addr}:{metrics_export_port}")
autoscaler_export_addr = "{}:{}".format(cluster.head_node.node_ip_address,
AUTOSCALER_METRIC_PORT)
yield prom_addresses, autoscaler_export_addr
ray.get(worker_should_exit.send.remote())
ray.get(obj_refs)
ray.shutdown()
cluster.shutdown()
def test_custom_metrics_validation(shutdown_only):
ray.init()
# Missing tag(s) from tag_keys.
metric = Counter("name", tag_keys=("a", "b"))
metric.set_default_tags({"a": "1"})
metric.inc(1.0, {"b": "2"})
metric.inc(1.0, {"a": "1", "b": "2"})
with pytest.raises(ValueError):
metric.inc(1.0)
with pytest.raises(ValueError):
metric.inc(1.0, {"a": "2"})
# Extra tag not in tag_keys.
metric = Counter("name", tag_keys=("a", ))
with pytest.raises(ValueError):
metric.inc(1.0, {"a": "1", "b": "2"})
# tag_keys must be tuple.
with pytest.raises(TypeError):
Counter("name", tag_keys="a")
# tag_keys must be strs.
with pytest.raises(TypeError):
Counter("name", tag_keys=(1, ))
metric = Counter("name", tag_keys=("a", ))
# Set default tag that isn't in tag_keys.
with pytest.raises(ValueError):
metric.set_default_tags({"a": "1", "c": "2"})
# Default tag value must be str.
with pytest.raises(TypeError):
metric.set_default_tags({"a": 1})
# Tag value must be str.
with pytest.raises(TypeError):
metric.inc(1.0, {"a": 1})
def __init__(self, name):
self._curr_count = 0
self.counter = Counter(
"num_requests",
description="Number of requests processed by the actor.",
tag_keys=("actor_name", ))
self.counter.set_default_tags({"actor_name": name})
self.gauge = Gauge(
"curr_count",
description="Current count held by the actor. Goes up and down.",
tag_keys=("actor_name", ))
self.gauge.set_default_tags({"actor_name": name})
self.histogram = Histogram("request_latency",
description="Latencies of requests in ms.",
boundaries=[0.1, 1],
tag_keys=("actor_name", ))
self.histogram.set_default_tags({"actor_name": name})
def override():
a = Counter("num_count", description="")
b = Counter("num_count", description="")
a.inc(1)
b.inc(1)
