def test_benchmark_lookup_by_uri(): da = MockDataset("benchmark://foo-v0") db = MockDataset("benchmark://bar-v0") ba = MockBenchmark(uri="benchmark://foo-v0/abc") da.benchmark_values.append(ba) datasets = Datasets([da, db]) assert datasets.benchmark("benchmark://foo-v0/abc") == ba
def test_enumerate_deprecated_datasets(): da = MockDataset("benchmark://a") db = MockDataset("benchmark://b") datasets = Datasets((da, db)) db.deprecated = True assert list(datasets) == [da] assert list(datasets.datasets(with_deprecated=True)) == [da, db]
def test_enumerate_datasets_deprecated_at_construction_time(): da = MockDataset("benchmark://a") db = MockDataset("benchmark://b") db.deprecated = True datasets = Datasets((da, db)) assert list(datasets) == [da] assert list(datasets.datasets(with_deprecated=True)) == [da, db]
def test_datasets_get_item_lookup_miss(): da = MockDataset("benchmark://foo-v0") datasets = Datasets([da]) with pytest.raises(LookupError, match=r"^Dataset not found: benchmark://bar-v0$"): datasets.dataset("benchmark://bar-v0") with pytest.raises(LookupError, match=r"^Dataset not found: benchmark://bar-v0$"): _ = datasets["benchmark://bar-v0"]
def test_dataset_proto_scheme(tmpdir): """Test the proto:// scheme handler.""" tmpdir = Path(tmpdir) datasets = Datasets(datasets={}) proto = BenchmarkProto(uri="hello world") with open(tmpdir / "file.pb", "wb") as f: f.write(proto.SerializeToString()) benchmark = datasets.benchmark(f"proto://{tmpdir}/file.pb") assert benchmark.proto.uri == "hello world" assert benchmark.uri == "benchmark://hello world"
def test_dataset_file_scheme(tmpdir): """Test the file:// scheme handler.""" tmpdir = Path(tmpdir) datasets = Datasets(datasets={}) with open(tmpdir / "file.dat", "w") as f: f.write("hello, world") benchmark = datasets.benchmark(f"file://{tmpdir}/file.dat") assert benchmark.proto.uri == f"file://{tmpdir}/file.dat" assert benchmark.proto.program.contents == b"hello, world" assert benchmark.uri == f"file://{tmpdir}/file.dat"
def test_benchmarks_iter_deprecated(): da = MockDataset("benchmark://foo-v0") db = MockDataset("benchmark://bar-v0") db.deprecated = True ba = MockBenchmark(uri="benchmark://foo-v0/abc") bb = MockBenchmark(uri="benchmark://foo-v0/123") bc = MockBenchmark(uri="benchmark://bar-v0/abc") bd = MockBenchmark(uri="benchmark://bar-v0/123") da.benchmark_values.append(ba) da.benchmark_values.append(bb) db.benchmark_values.append(bc) db.benchmark_values.append(bd) datasets = Datasets([da, db]) # Iterate over the benchmarks. The deprecated dataset is not included. assert list( datasets.benchmark_uris()) == [b.uri for b in datasets.benchmarks()] assert list(datasets.benchmark_uris()) == [ "benchmark://foo-v0/abc", "benchmark://foo-v0/123", ] # Repeat the above, but include the deprecated datasets. assert list(datasets.benchmark_uris(with_deprecated=True)) == [ b.uri for b in datasets.benchmarks(with_deprecated=True) ] assert list(datasets.benchmark_uris(with_deprecated=True)) == [ "benchmark://bar-v0/abc", "benchmark://foo-v0/abc", "benchmark://bar-v0/123", "benchmark://foo-v0/123", ]
def test_datasets_add_dataset(): datasets = Datasets([]) da = MockDataset("benchmark://foo-v0") datasets["benchmark://foo-v0"] = da assert list(datasets) == [da]
def test_enumerate_datasets_with_custom_sort_order(): da = MockDataset("benchmark://a") db = MockDataset("benchmark://b") db.sort_order = -1 datasets = Datasets((da, db)) assert list(datasets) == [db, da]
def test_datasets_add_deprecated_dataset(): datasets = Datasets([]) da = MockDataset("benchmark://a") da.deprecated = True datasets["benchmark://foo-v0"] = da assert list(datasets) == []
def test_random_benchmark(mocker, weighted: bool): da = MockDataset("benchmark://foo-v0") da.size = 10 ba = MockBenchmark(uri="benchmark://foo-v0/abc") da.benchmark_values.append(ba) datasets = Datasets([da]) mocker.spy(da, "random_benchmark") num_benchmarks = 5 rng = np.random.default_rng(0) random_benchmarks = { b.uri for b in (datasets.random_benchmark(rng, weighted=weighted) for _ in range(num_benchmarks)) } assert da.random_benchmark.call_count == num_benchmarks assert len(random_benchmarks) == 1 assert next(iter(random_benchmarks)) == "benchmark://foo-v0/abc"
def test_benchmark_uris_order(): da = MockDataset("benchmark://foo-v0") db = MockDataset("benchmark://bar-v0") ba = MockBenchmark(uri="benchmark://foo-v0/abc") bb = MockBenchmark(uri="benchmark://foo-v0/123") bc = MockBenchmark(uri="benchmark://bar-v0/abc") bd = MockBenchmark(uri="benchmark://bar-v0/123") da.benchmark_values.append(ba) da.benchmark_values.append(bb) db.benchmark_values.append(bc) db.benchmark_values.append(bd) datasets = Datasets([da, db]) assert list( datasets.benchmark_uris()) == [b.uri for b in datasets.benchmarks()] # Datasets are ordered by name, so bar-v0 before foo-v0. assert list(datasets.benchmark_uris()) == [ "benchmark://bar-v0/abc", "benchmark://foo-v0/abc", "benchmark://bar-v0/123", "benchmark://foo-v0/123", ]
def test_datasets_remove(): da = MockDataset("benchmark://foo-v0") datasets = Datasets([da]) del datasets["benchmark://foo-v0"] assert list(datasets) == []
def test_enumerate_datasets(): da = MockDataset("benchmark://a") db = MockDataset("benchmark://b") datasets = Datasets((da, db)) assert list(datasets) == [da, db]
def test_enumerate_datasets_empty(): datasets = Datasets([]) assert list(datasets) == []
def test_datasets_get_item_default_scheme(): da = MockDataset("benchmark://foo-v0") datasets = Datasets([da]) assert datasets.dataset("foo-v0") == da assert datasets["foo-v0"] == da
def test_dataset_proto_scheme_file_not_found(tmpdir): tmpdir = Path(tmpdir) datasets = Datasets(datasets={}) with pytest.raises(FileNotFoundError): datasets.benchmark(f"proto://{tmpdir}/not_a_file")