def test_retrieve_results(self, benchmark, requests_mock):
result = create_benchmark_result(
benchmark.project_id,
benchmark.study_id,
benchmark.id,
create_data_set("data-set-1", 1),
)
mock_get_benchmark_result(requests_mock, result)
with temporary_cd():
BenchmarkInputFactory._retrieve_results(benchmark)
stored_result = BenchmarkResult.parse_file(
os.path.join("analysis", "benchmark-results.json"))
assert stored_result.json() == result.json()
def commit_benchmark_result(
db: Session, for_optimization: bool
) -> Tuple[Project, Study, Benchmark, BenchmarkResult, DataSetCollection,
Optional[Optimization], Optional[OptimizationResult], ]:
"""Creates a new benchmark result (and all of the required parents such as
a parent project, study etc.).
Parameters
----------
db
The current data base session.
for_optimization
Whether the results should be generated for a benchmark against an
optimization.
"""
# Create the parent optimization
(
project,
study,
benchmark,
data_sets,
optimization,
optimization_result,
) = commit_benchmark(db, for_optimization)
result = create_benchmark_result(project.id, study.id, benchmark.id,
data_sets)
db_result = BenchmarkResultCRUD.create(db, result)
db.add(db_result)
db.commit()
# noinspection PyTypeChecker
result = BenchmarkResultCRUD.db_to_model(db_result)
return (
project,
study,
benchmark,
result,
data_sets,
optimization,
optimization_result,
)
def test_delete_with_dependent(self, db: Session, create_results: bool,
expected_error):
"""Test that a benchmark cannot be deleted until its results have
also been deleted.
"""
create_dependencies(db, self.dependencies())
model = self.create_model()
db.add(self.crud_class().create(db, model))
db.commit()
# Create the benchmark results
if create_results:
data_set = create_data_set("data-set-1")
data_set.entries[0].id = 1
db_result = BenchmarkResultCRUD.create(
db,
create_benchmark_result(model.project_id, model.study_id,
model.id, data_set),
)
db.add(db_result)
db.commit()
# Delete the model.
with expected_error:
self.crud_class().delete(db, model.project_id, model.study_id,
model.id)
if not create_results:
return
BenchmarkResultCRUD.delete(db, model.project_id, model.study_id,
model.id)
db.commit()
self.crud_class().delete(db, model.project_id, model.study_id,
model.id)
db.commit()
self.check_has_deleted(db)
def test_plot(force_field, monkeypatch):
from nonbonded.library.plotting.seaborn import benchmark as benchmark_module
# Mock the required file inputs
data_set = create_data_set("data-set-1", 1)
data_set_collection = DataSetCollection(data_sets=[data_set])
benchmark = create_benchmark(
"project-1",
"study-1",
"benchmark-1",
["data-set-1"],
None,
force_field,
)
benchmark_result = create_benchmark_result(
"project-1", "study-1", "benchmark-1", [create_data_set("data-set-1", 1)]
)
# Mock the already tested plotting methods.
monkeypatch.setattr(benchmark_module, "plot_categorized_rmse", lambda *args: None)
monkeypatch.setattr(benchmark_module, "plot_overall_statistics", lambda *args: None)
monkeypatch.setattr(benchmark_module, "plot_scatter_results", lambda *args: None)
if "nonbonded.library.factories.plots.benchmark" in sys.modules:
sys.modules.pop("nonbonded.library.factories.plots.benchmark")
from nonbonded.library.factories.plots.benchmark import BenchmarkPlotFactory
with temporary_cd():
# Save the inputs in their expected locations.
data_set_collection.to_file("test-set-collection.json")
benchmark.to_file("benchmark.json")
os.makedirs("analysis")
benchmark_result.to_file(os.path.join("analysis", "benchmark-results.json"))
BenchmarkPlotFactory.plot([""], "png")
assert os.path.isdir("plots")
def _create_model(cls, db, create_dependencies=True):
project_id = "project-1"
study_id = "study-1"
benchmark_id = "benchmark-1"
data_sets = create_data_set("data-set-1")
for index, entry in enumerate(data_sets.entries):
entry.id = index + 1
if create_dependencies:
project, study, benchmark, data_set, _, _ = commit_benchmark(
db, False)
project_id = project.id
study_id = study.id
benchmark_id = benchmark.id
data_sets = data_set
benchmark_result = create_benchmark_result(
project_id,
study_id,
benchmark_id,
data_sets,
)
return (
benchmark_result,
{
"project_id": project_id,
"study_id": study_id,
"model_id": benchmark_id
},
)
def benchmarks_and_results(
force_field: ForceField,
) -> Tuple[List[Benchmark], List[BenchmarkResult], List[DataSet]]:
benchmarks = []
benchmark_results = []
data_sets = [create_data_set("data-set-1", 1)]
for index in range(2):
benchmark = create_benchmark(
"project-1",
"study-1",
f"benchmark-{index + 1}",
["data-set-1"],
None,
force_field,
)
benchmark.name = f"Benchmark {index + 1}"
benchmarks.append(benchmark)
benchmark_result = create_benchmark_result(
"project-1", "study-1", "benchmark-1", data_sets
)
for statistic_entry in benchmark_result.data_set_result.statistic_entries:
statistic_entry.value /= index + 1
statistic_entry.lower_95_ci /= index + 1
statistic_entry.upper_95_ci /= index + 1
benchmark_results.append(benchmark_result)
# benchmarks = [
# Benchmark.from_rest(
# project_id="binary-mixture",
# study_id="expanded",
# sub_study_id="openff-1-0-0",
# ),
# Benchmark.from_rest(
# project_id="binary-mixture",
# study_id="expanded",
# sub_study_id="h-mix-rho-x-rho",
# ),
# Benchmark.from_rest(
# project_id="binary-mixture", study_id="expanded", sub_study_id="h-mix-rho-x"
# ),
# ]
# benchmark_results = [
# BenchmarkResult.from_rest(
# project_id="binary-mixture", study_id="expanded", model_id="openff-1-0-0"
# ),
# BenchmarkResult.from_rest(
# project_id="binary-mixture", study_id="expanded", model_id="h-mix-rho-x-rho"
# ),
# BenchmarkResult.from_rest(
# project_id="binary-mixture", study_id="expanded", model_id="h-mix-rho-x"
# ),
# ]
#
# data_set_ids = {
# test_set_id
# for benchmark in benchmarks
# for test_set_id in benchmark.test_set_ids
# }
# data_sets = [
# DataSet.from_rest(data_set_id=data_set_id) for data_set_id in data_set_ids
# ]
return benchmarks, benchmark_results, data_sets