def test_basic_custom_metrics(metric_mock):
# Make sure each of metric works as expected.
# -- Count --
count = Count("count", tag_keys=("a", ))
count._metric = metric_mock
count.record(1)
metric_mock.record.assert_called_with(1, tags={})
# -- Gauge --
gauge = Gauge("gauge", description="gauge")
gauge._metric = metric_mock
gauge.record(4)
metric_mock.record.assert_called_with(4, tags={})
# -- Histogram
histogram = Histogram("hist",
description="hist",
boundaries=[1.0, 3.0],
tag_keys=("a", "b"))
histogram._metric = metric_mock
histogram.record(4)
metric_mock.record.assert_called_with(4, tags={})
tags = {"a": "3"}
histogram.record(10, tags=tags)
metric_mock.record.assert_called_with(10, tags=tags)
tags = {"a": "10", "b": "b"}
histogram.record(8, tags=tags)
metric_mock.record.assert_called_with(8, tags=tags)
def _setup_cluster_for_test(ray_start_cluster):
NUM_NODES = 2
cluster = ray_start_cluster
# Add a head node.
cluster.add_node(_system_config={"metrics_report_interval_ms": 1000})
# Add worker nodes.
[cluster.add_node() for _ in range(NUM_NODES - 1)]
cluster.wait_for_nodes()
ray.init(address=cluster.address)
worker_should_exit = SignalActor.remote()
# Generate a metric in the driver.
counter = Count("test_driver_counter", description="desc")
counter.inc()
# Generate some metrics from actor & tasks.
@ray.remote
def f():
counter = Count("test_counter", description="desc")
counter.inc()
counter = ray.get(ray.put(counter)) # Test serialization.
counter.inc()
ray.get(worker_should_exit.wait.remote())
@ray.remote
class A:
async def ping(self):
histogram = Histogram("test_histogram",
description="desc",
boundaries=[0.1, 1.6])
histogram = ray.get(ray.put(histogram)) # Test serialization.
histogram.record(1.5)
ray.get(worker_should_exit.wait.remote())
a = A.remote()
obj_refs = [f.remote(), a.ping.remote()]
node_info_list = ray.nodes()
prom_addresses = []
for node_info in node_info_list:
metrics_export_port = node_info["MetricsExportPort"]
addr = node_info["NodeManagerAddress"]
prom_addresses.append(f"{addr}:{metrics_export_port}")
yield prom_addresses
ray.get(worker_should_exit.send.remote())
ray.get(obj_refs)
ray.shutdown()
cluster.shutdown()
def test_custom_metrics_edge_cases(metric_mock):
# None or empty boundaries are not allowed.
with pytest.raises(ValueError):
Histogram("hist")
with pytest.raises(ValueError):
Histogram("hist", boundaries=[])
# Empty name is not allowed.
with pytest.raises(ValueError):
Count("")
# The tag keys must be a tuple type.
with pytest.raises(TypeError):
Count("name", tag_keys=("a"))
def f():
counter = Count("test_counter", description="desc")
counter.inc()
counter = ray.get(ray.put(counter)) # Test serialization.
counter.inc()
ray.get(worker_should_exit.wait.remote())
def test_custom_metrics_validation(ray_start_regular_shared):
# Missing tag(s) from tag_keys.
metric = Count("name", tag_keys=("a", "b"))
metric.set_default_tags({"a": "1"})
metric.record(1.0, {"b": "2"})
metric.record(1.0, {"a": "1", "b": "2"})
with pytest.raises(ValueError):
metric.record(1.0)
with pytest.raises(ValueError):
metric.record(1.0, {"a": "2"})
# Extra tag not in tag_keys.
metric = Count("name", tag_keys=("a", ))
with pytest.raises(ValueError):
metric.record(1.0, {"a": "1", "b": "2"})
# tag_keys must be tuple.
with pytest.raises(TypeError):
Count("name", tag_keys="a")
# tag_keys must be strs.
with pytest.raises(TypeError):
Count("name", tag_keys=(1, ))
metric = Count("name", tag_keys=("a", ))
# Set default tag that isn't in tag_keys.
with pytest.raises(ValueError):
metric.set_default_tags({"a": "1", "c": "2"})
# Default tag value must be str.
with pytest.raises(TypeError):
metric.set_default_tags({"a": 1})
# Tag value must be str.
with pytest.raises(TypeError):
metric.record(1.0, {"a": 1})
def override():
a = Count("num_count", description="")
b = Count("num_count", description="")
a.record(1)
b.record(1)
def test_basic_custom_metrics(metric_mock):
# Make sure each of metric works as expected.
# -- Count --
count = Count("count", tag_keys=("a", ))
with pytest.raises(TypeError):
count.inc("hi")
with pytest.raises(ValueError):
count.inc(0)
count.inc(-1)
count._metric = metric_mock
count.record(1, {"a": "1"})
metric_mock.record.assert_called_with(1, tags={"a": "1"})
# -- Gauge --
gauge = Gauge("gauge", description="gauge")
gauge._metric = metric_mock
gauge.record(4)
metric_mock.record.assert_called_with(4, tags={})
# -- Histogram
histogram = Histogram(
"hist", description="hist", boundaries=[1.0, 3.0], tag_keys=("a", "b"))
histogram._metric = metric_mock
tags = {"a": "10", "b": "b"}
histogram.record(8, tags=tags)
metric_mock.record.assert_called_with(8, tags=tags)
def f():
counter = Count("test_counter", description="desc")
counter.record(1)
ray.get(worker_should_exit.wait.remote())