def mock_get_benchmark_result(requests_mock,
benchmark_result: BenchmarkResult):
"""Mock the get benchmark result endpoint."""
requests_mock.get(
BenchmarkResult._get_endpoint(
project_id=benchmark_result.project_id,
study_id=benchmark_result.study_id,
model_id=benchmark_result.id,
),
text=benchmark_result.json(),
)
def create_benchmark_result(
project_id: str,
study_id: str,
benchmark_id: str,
data_sets: Union[DataSet, List[DataSet], DataSetCollection],
) -> BenchmarkResult:
"""Creates a benchmark result.
Parameters
----------
project_id
The id of the parent project.
study_id
The id of the parent study.
benchmark_id
The id of the benchmark which the result belongs to.
data_sets
The data sets the benchmark is targeting.
"""
results_entries = _results_entries_from_data_sets(data_sets)
statistics_entries = _statistics_from_result_entries(results_entries)
return BenchmarkResult(
id=benchmark_id,
study_id=study_id,
project_id=project_id,
calculation_environment={"openff-evaluator": "1.0.0"},
analysis_environment={"nonbonded": "0.0.01a5"},
data_set_result=DataSetResult(statistic_entries=statistics_entries,
result_entries=results_entries),
)
def _load_sub_study(cls, directory):
benchmark = Benchmark.parse_file(
os.path.join(directory, "benchmark.json"))
benchmark_result = BenchmarkResult.parse_file(
os.path.join(directory, "analysis", "benchmark-results.json"))
return benchmark, benchmark_result
def test_benchmark_analysis(caplog, monkeypatch, dummy_conda_env):
from openff.evaluator.client import RequestResult
from openff.evaluator.datasets import PhysicalPropertyDataSet
benchmark = create_benchmark(
"project-1", "study-1", "benchmark-1", ["data-set-1"], "optimization-1", None
)
# Create a reference data set.
reference_data_set = create_data_set("data-set-1")
reference_data_set.entries.append(reference_data_set.entries[0].copy())
reference_data_set.entries[0].id = 1
reference_data_set.entries[1].id = 2
# Create a set of evaluator results
estimated_data_set = PhysicalPropertyDataSet()
estimated_data_set.add_properties(reference_data_set.entries[0].to_evaluator())
unsuccessful_properties = PhysicalPropertyDataSet()
unsuccessful_properties.add_properties(reference_data_set.entries[1].to_evaluator())
results = RequestResult()
results.estimated_properties = estimated_data_set
results.unsuccessful_properties = unsuccessful_properties
with temporary_cd(os.path.dirname(dummy_conda_env)):
# Save the expected input files.
with open("benchmark.json", "w") as file:
file.write(benchmark.json())
with open("test-set-collection.json", "w") as file:
file.write(DataSetCollection(data_sets=[reference_data_set]).json())
results.json("results.json")
with caplog.at_level(logging.WARNING):
BenchmarkAnalysisFactory.analyze(True)
assert (
"1 properties could not be estimated and so were not analyzed"
in caplog.text
)
assert os.path.isdir("analysis")
assert os.path.isfile(os.path.join("analysis", "benchmark-results.json"))
results_object = BenchmarkResult.parse_file(
os.path.join("analysis", "benchmark-results.json")
)
assert len(results_object.calculation_environment) > 0
assert len(results_object.analysis_environment) > 0
def analyze(cls, reindex):
from openff.evaluator.client import RequestResult
# Load in the definition of the benchmark to optimize.
benchmark = Benchmark.parse_file("benchmark.json")
# Create a directory to store the results in
output_directory = "analysis"
os.makedirs(output_directory, exist_ok=True)
# Load the reference data set
reference_data_sets = DataSetCollection.parse_file(
"test-set-collection.json")
# Load in the request results.
request_results: RequestResult = RequestResult.from_json(
"results.json")
if reindex:
request_results = reindex_results(request_results,
reference_data_sets)
if len(request_results.unsuccessful_properties) > 0:
logger.warning(
f"{len(request_results.unsuccessful_properties)} properties could "
f"not be estimated and so were not analyzed:")
for unsuccessful_property in request_results.unsuccessful_properties:
logger.warning(
f"{unsuccessful_property.id} could not be estimated.")
estimated_data_set = request_results.estimated_properties
# Generate statistics for the estimated properties.
benchmark_results = BenchmarkResult.from_evaluator(
project_id=benchmark.project_id,
study_id=benchmark.study_id,
benchmark_id=benchmark.id,
reference_data_set=reference_data_sets,
estimated_data_set=estimated_data_set,
analysis_environments=benchmark.analysis_environments,
)
benchmark_results.calculation_environment = cls._parse_calculation_environment(
)
benchmark_results.analysis_environment = summarise_current_versions()
# Save the results
with open(os.path.join(output_directory, "benchmark-results.json"),
"w") as file:
file.write(benchmark_results.json())
def test_retrieve_results(self, benchmark, requests_mock):
result = create_benchmark_result(
benchmark.project_id,
benchmark.study_id,
benchmark.id,
create_data_set("data-set-1", 1),
)
mock_get_benchmark_result(requests_mock, result)
with temporary_cd():
BenchmarkInputFactory._retrieve_results(benchmark)
stored_result = BenchmarkResult.parse_file(
os.path.join("analysis", "benchmark-results.json"))
assert stored_result.json() == result.json()
def test_benchmark_result_from_evaluator(estimated_reference_sets):
"""Tests the `BenchmarkResult.from_evaluator` function."""
estimated_data_set, reference_data_set = estimated_reference_sets
benchmark_result = BenchmarkResult.from_evaluator(
project_id="a",
study_id="b",
benchmark_id="c",
reference_data_set=reference_data_set,
estimated_data_set=estimated_data_set,
analysis_environments=[
ChemicalEnvironment.Aqueous,
ChemicalEnvironment.Aldehyde,
],
)
assert benchmark_result.data_set_result is not None
assert len(benchmark_result.data_set_result.result_entries) == len(
estimated_data_set)
# 3 statistic types x 2 properties x 2 categories
assert len(benchmark_result.data_set_result.statistic_entries) == 12
def _retrieve_results(cls, benchmark: Benchmark):
"""Retrieves the full results for a benchmark.
Parameters
----------
benchmark
The benchmark to retrieve the results for.
"""
results = BenchmarkResult.from_rest(
project_id=benchmark.project_id,
study_id=benchmark.study_id,
model_id=benchmark.id,
)
output_directory = "analysis"
os.makedirs(output_directory, exist_ok=True)
with open(os.path.join(output_directory, "benchmark-results.json"),
"w") as file:
file.write(results.json())