def test_dataset_deprecation_message(tmpwd: Path):
"""Test that a deprecation warning is emitted on install()."""
dataset = Dataset(
name="benchmark://test-v0",
description="A test dataset",
license="MIT",
site_data_base="test",
deprecated="The cat sat on the mat",
)
with pytest.warns(DeprecationWarning, match="The cat sat on the mat"):
dataset.install()
def test_with_site_data():
"""Test the dataset property values."""
dataset = Dataset(
name="benchmark://test-v0",
description="A test dataset",
license="MIT",
site_data_base="test",
)
assert dataset.has_site_data
def test_dataset_inferred_properties():
"""Test the values of inferred dataset properties."""
dataset = Dataset(
name="benchmark://test-v2",
description="A test dataset",
license="MIT",
site_data_base="test",
)
assert dataset.scheme == "benchmark"
assert dataset.version == 2
def test_dataset_default_version():
"""Test the dataset property values."""
dataset = Dataset(
name="benchmark://test",
description="A test dataset",
license="MIT",
site_data_base="test",
)
assert dataset.name == "benchmark://test"
assert dataset.scheme == "benchmark"
assert dataset.version == 0
def test_without_site_data():
"""Test the dataset property values."""
dataset = Dataset(
name="benchmark://test-v0",
description="A test dataset",
license="MIT",
)
assert not dataset.has_site_data
with pytest.raises(
ValueError,
match=r"^Dataset has no site data path: benchmark://test-v0$"):
dataset.site_data_path # noqa
def test_dataset_optional_properties():
"""Test the default values of optional dataset properties."""
dataset = Dataset(
name="benchmark://test-v0",
description="A test dataset",
license="MIT",
site_data_base="test",
)
assert dataset.references == {} # Default value.
assert not dataset.deprecated
assert dataset.sort_order == 0
assert dataset.validatable == "No"
def test_dataset_properties():
"""Test the dataset property values."""
dataset = Dataset(
name="benchmark://test-v0",
description="A test dataset",
license="MIT",
site_data_base="test",
)
assert dataset.name == "benchmark://test-v0"
assert dataset.protocol == "benchmark"
assert dataset.description == "A test dataset"
assert dataset.license == "MIT"
def test_dataset_equality_and_sorting():
"""Test comparison operators between datasets."""
a = Dataset(
name="benchmark://a-v0",
description="A test dataset",
license="MIT",
site_data_base="test",
)
a2 = Dataset(
name="benchmark://a-v0",
description="A test dataset",
license="MIT",
site_data_base="test",
)
b = Dataset(
name="benchmark://b-v0",
description="A test dataset",
license="MIT",
site_data_base="test",
)
assert a == a2
assert a != b
assert a < b
assert a <= b
assert b > a
assert b >= a
# String comparisons
assert a == "benchmark://a-v0"
assert a != "benchmark://b-v0"
assert a < "benchmark://b-v0"
# Sorting
assert sorted([a2, b, a]) == [
"benchmark://a-v0",
"benchmark://a-v0",
"benchmark://b-v0",
]
def test_dataset_site_data_directory(tmpwd: Path):
"""Test the path generated for site data."""
dataset = Dataset(
name="benchmark://test-v0",
description="A test dataset",
license="MIT",
site_data_base="test",
)
# Use endswith() since tmpwd on macOS may have a '/private' prefix.
assert str(dataset.site_data_path).endswith(
str(tmpwd / "test" / "benchmark" / "test-v0"))
assert not dataset.site_data_path.is_dir(
) # Dir is not created until needed.
def enumerate_directory(name: str, path: Path):
rows = []
for path in path.iterdir():
if not path.is_file() or not path.name.endswith(".json"):
continue
dataset = Dataset.from_json_file(path)
rows.append((dataset.name, dataset.license, dataset.file_count,
dataset.size_bytes))
rows.append(
("Total", "", sum(r[2] for r in rows), sum(r[3] for r in rows)))
return tabulate(
[(n, l, humanize.intcomma(f), humanize.naturalsize(s))
for n, l, f, s in rows],
headers=(name, "License", "#. Benchmarks", "Size on disk"),
)
def test_dataset__invalid_name(invalid_name: str):
"""Test that invalid dataset names raise an error on init."""
with pytest.raises(
ValueError,
match=
(f"Invalid dataset name: '{invalid_name}'. "
"Dataset name must be in the form: '{{protocol}}://{{name}}-v{{version}}'"
),
):
Dataset(
name=invalid_name,
description="A test dataset",
license="MIT",
site_data_base="test",
)
def test_dataset_optional_properties_explicit_values():
"""Test the non-default values of optional dataset properties."""
dataset = Dataset(
name="benchmark://test-v0",
description="A test dataset",
license="MIT",
site_data_base="test",
references={
"GitHub": "https://github.com/facebookresearch/CompilerGym"
},
deprecated="Deprecation message",
sort_order=10,
validatable="Yes",
)
assert dataset.references == {
"GitHub": "https://github.com/facebookresearch/CompilerGym"
}
assert dataset.deprecated
assert dataset.sort_order == 10
assert dataset.validatable == "Yes"
def test_dataset_properties_read_only(tmpwd: Path):
"""Test that dataset properties are read-only."""
dataset = Dataset(
name="benchmark://test-v0",
description="A test dataset",
license="MIT",
site_data_base="test",
)
with pytest.raises(AttributeError):
dataset.name = "benchmark://test-v1"
with pytest.raises(AttributeError):
dataset.description = "A test dataset"
with pytest.raises(AttributeError):
dataset.license = "MIT"
with pytest.raises(AttributeError):
dataset.site_data_path = tmpwd
if sys.platform == "darwin":
_COMPILE_ARGS = [
"-L",
"/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib",
]
else:
_COMPILE_ARGS = []
LLVM_DATASETS = [
Dataset(
name="blas-v0",
url=
"https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-blas-v0.tar.bz2",
license="BSD 3-Clause",
description="https://github.com/spcl/ncc/tree/master/data",
compiler="llvm-10.0.0",
file_count=300,
size_bytes=3969036,
sha256=
"e724a8114709f8480adeb9873d48e426e8d9444b00cddce48e342b9f0f2b096d",
),
Dataset(
name="cBench-v0",
url=
"https://dl.fbaipublicfiles.com/compiler_gym/llvm_bitcodes-10.0.0-cBench-v0-macos.tar.bz2",
license="BSD 3-Clause",
description="https://github.com/ctuning/ctuning-programs",
compiler="llvm-10.0.0",
file_count=23,
size_bytes=7154448,
sha256=