def test_benchmark_lookup_by_uri():
da = MockDataset("benchmark://foo-v0")
db = MockDataset("benchmark://bar-v0")
ba = MockBenchmark(uri="benchmark://foo-v0/abc")
da.benchmark_values.append(ba)
datasets = Datasets([da, db])
assert datasets.benchmark("benchmark://foo-v0/abc") == ba
def test_enumerate_deprecated_datasets():
da = MockDataset("benchmark://a")
db = MockDataset("benchmark://b")
datasets = Datasets((da, db))
db.deprecated = True
assert list(datasets) == [da]
assert list(datasets.datasets(with_deprecated=True)) == [da, db]
def test_enumerate_datasets_deprecated_at_construction_time():
da = MockDataset("benchmark://a")
db = MockDataset("benchmark://b")
db.deprecated = True
datasets = Datasets((da, db))
assert list(datasets) == [da]
assert list(datasets.datasets(with_deprecated=True)) == [da, db]
def test_datasets_get_item_lookup_miss():
da = MockDataset("benchmark://foo-v0")
datasets = Datasets([da])
with pytest.raises(LookupError,
match=r"^Dataset not found: benchmark://bar-v0$"):
datasets.dataset("benchmark://bar-v0")
with pytest.raises(LookupError,
match=r"^Dataset not found: benchmark://bar-v0$"):
_ = datasets["benchmark://bar-v0"]
def test_dataset_proto_scheme(tmpdir):
"""Test the proto:// scheme handler."""
tmpdir = Path(tmpdir)
datasets = Datasets(datasets={})
proto = BenchmarkProto(uri="hello world")
with open(tmpdir / "file.pb", "wb") as f:
f.write(proto.SerializeToString())
benchmark = datasets.benchmark(f"proto://{tmpdir}/file.pb")
assert benchmark.proto.uri == "hello world"
assert benchmark.uri == "benchmark://hello world"
def test_dataset_file_scheme(tmpdir):
"""Test the file:// scheme handler."""
tmpdir = Path(tmpdir)
datasets = Datasets(datasets={})
with open(tmpdir / "file.dat", "w") as f:
f.write("hello, world")
benchmark = datasets.benchmark(f"file://{tmpdir}/file.dat")
assert benchmark.proto.uri == f"file://{tmpdir}/file.dat"
assert benchmark.proto.program.contents == b"hello, world"
assert benchmark.uri == f"file://{tmpdir}/file.dat"
def test_benchmarks_iter_deprecated():
da = MockDataset("benchmark://foo-v0")
db = MockDataset("benchmark://bar-v0")
db.deprecated = True
ba = MockBenchmark(uri="benchmark://foo-v0/abc")
bb = MockBenchmark(uri="benchmark://foo-v0/123")
bc = MockBenchmark(uri="benchmark://bar-v0/abc")
bd = MockBenchmark(uri="benchmark://bar-v0/123")
da.benchmark_values.append(ba)
da.benchmark_values.append(bb)
db.benchmark_values.append(bc)
db.benchmark_values.append(bd)
datasets = Datasets([da, db])
# Iterate over the benchmarks. The deprecated dataset is not included.
assert list(
datasets.benchmark_uris()) == [b.uri for b in datasets.benchmarks()]
assert list(datasets.benchmark_uris()) == [
"benchmark://foo-v0/abc",
"benchmark://foo-v0/123",
]
# Repeat the above, but include the deprecated datasets.
assert list(datasets.benchmark_uris(with_deprecated=True)) == [
b.uri for b in datasets.benchmarks(with_deprecated=True)
]
assert list(datasets.benchmark_uris(with_deprecated=True)) == [
"benchmark://bar-v0/abc",
"benchmark://foo-v0/abc",
"benchmark://bar-v0/123",
"benchmark://foo-v0/123",
]
def test_datasets_add_dataset():
datasets = Datasets([])
da = MockDataset("benchmark://foo-v0")
datasets["benchmark://foo-v0"] = da
assert list(datasets) == [da]
def test_enumerate_datasets_with_custom_sort_order():
da = MockDataset("benchmark://a")
db = MockDataset("benchmark://b")
db.sort_order = -1
datasets = Datasets((da, db))
assert list(datasets) == [db, da]
def test_datasets_add_deprecated_dataset():
datasets = Datasets([])
da = MockDataset("benchmark://a")
da.deprecated = True
datasets["benchmark://foo-v0"] = da
assert list(datasets) == []
def test_random_benchmark(mocker, weighted: bool):
da = MockDataset("benchmark://foo-v0")
da.size = 10
ba = MockBenchmark(uri="benchmark://foo-v0/abc")
da.benchmark_values.append(ba)
datasets = Datasets([da])
mocker.spy(da, "random_benchmark")
num_benchmarks = 5
rng = np.random.default_rng(0)
random_benchmarks = {
b.uri
for b in (datasets.random_benchmark(rng, weighted=weighted)
for _ in range(num_benchmarks))
}
assert da.random_benchmark.call_count == num_benchmarks
assert len(random_benchmarks) == 1
assert next(iter(random_benchmarks)) == "benchmark://foo-v0/abc"
def test_benchmark_uris_order():
da = MockDataset("benchmark://foo-v0")
db = MockDataset("benchmark://bar-v0")
ba = MockBenchmark(uri="benchmark://foo-v0/abc")
bb = MockBenchmark(uri="benchmark://foo-v0/123")
bc = MockBenchmark(uri="benchmark://bar-v0/abc")
bd = MockBenchmark(uri="benchmark://bar-v0/123")
da.benchmark_values.append(ba)
da.benchmark_values.append(bb)
db.benchmark_values.append(bc)
db.benchmark_values.append(bd)
datasets = Datasets([da, db])
assert list(
datasets.benchmark_uris()) == [b.uri for b in datasets.benchmarks()]
# Datasets are ordered by name, so bar-v0 before foo-v0.
assert list(datasets.benchmark_uris()) == [
"benchmark://bar-v0/abc",
"benchmark://foo-v0/abc",
"benchmark://bar-v0/123",
"benchmark://foo-v0/123",
]
def test_datasets_remove():
da = MockDataset("benchmark://foo-v0")
datasets = Datasets([da])
del datasets["benchmark://foo-v0"]
assert list(datasets) == []
def test_enumerate_datasets():
da = MockDataset("benchmark://a")
db = MockDataset("benchmark://b")
datasets = Datasets((da, db))
assert list(datasets) == [da, db]
def test_enumerate_datasets_empty():
datasets = Datasets([])
assert list(datasets) == []
def test_datasets_get_item_default_scheme():
da = MockDataset("benchmark://foo-v0")
datasets = Datasets([da])
assert datasets.dataset("foo-v0") == da
assert datasets["foo-v0"] == da
def test_dataset_proto_scheme_file_not_found(tmpdir):
tmpdir = Path(tmpdir)
datasets = Datasets(datasets={})
with pytest.raises(FileNotFoundError):
datasets.benchmark(f"proto://{tmpdir}/not_a_file")
