def test_generate_evaluator_target(self, requests_mock):
data_set = create_data_set("data-set-1")
mock_get_data_set(requests_mock, data_set)
target = create_evaluator_target("evaluator-target-1", [data_set.id])
with temporary_cd():
OptimizationInputFactory._generate_evaluator_target(
target, 8000, None)
assert os.path.isfile("training-set.json")
off_data_set = PhysicalPropertyDataSet.from_json(
"training-set.json")
assert off_data_set.json() == data_set.to_evaluator().json()
assert os.path.isfile("options.json")
def test_study_with_children(requests_mock, monkeypatch):
# Overwrite the child factories so we don't need to provide
# sensible children and wait for them to be buit.
def mock_generate(model, **_):
os.makedirs(model.id, exist_ok=True)
monkeypatch.setattr(OptimizationInputFactory, "generate", mock_generate)
monkeypatch.setattr(BenchmarkInputFactory, "generate", mock_generate)
mock_get_data_set(requests_mock, create_data_set("data-set-1"))
optimization = create_optimization(
"project-1",
"study-1",
"optimization-1",
targets=[create_evaluator_target("evaluator-target", ["data-set-1"])],
)
benchmark = create_benchmark(
"project-1",
"study-1",
"benchmark-1",
["data-set-1"],
None,
create_force_field(),
)
study = create_study("project-1", "study-1")
study.optimizations = [optimization]
study.benchmarks = [benchmark]
mock_get_study(requests_mock, study)
with temporary_cd():
InputFactory.generate(study, "test-env", "12:34", "lilac-dask", 8000, 1, False)
assert os.path.isdir(study.id)
assert os.path.isdir(os.path.join(study.id, "optimizations"))
assert os.path.isdir(os.path.join(study.id, "benchmarks"))
assert os.path.isdir(os.path.join(study.id, "optimizations", optimization.id))
assert os.path.isdir(os.path.join(study.id, "benchmarks", benchmark.id))
def test_delete_with_dependent(self, db: Session, create_results: bool,
expected_error):
"""Test that a benchmark cannot be deleted until its results have
also been deleted.
"""
create_dependencies(db, self.dependencies())
model = self.create_model()
db.add(self.crud_class().create(db, model))
db.commit()
# Create the benchmark results
if create_results:
data_set = create_data_set("data-set-1")
data_set.entries[0].id = 1
db_result = BenchmarkResultCRUD.create(
db,
create_benchmark_result(model.project_id, model.study_id,
model.id, data_set),
)
db.add(db_result)
db.commit()
# Delete the model.
with expected_error:
self.crud_class().delete(db, model.project_id, model.study_id,
model.id)
if not create_results:
return
BenchmarkResultCRUD.delete(db, model.project_id, model.study_id,
model.id)
db.commit()
self.crud_class().delete(db, model.project_id, model.study_id,
model.id)
db.commit()
self.check_has_deleted(db)
def test_plot_scatter_results(
benchmarks_and_results,
file_type: Literal["png", "pdf"],
tmpdir,
):
benchmarks, results, _ = benchmarks_and_results
plot_scatter_results(
benchmarks,
results,
[create_data_set("data-set-1", 1)],
tmpdir,
file_type,
)
assert os.path.isfile(
os.path.join(tmpdir, "scatter-plots", f"density-2.{file_type}")
)
def test_plot(force_field, monkeypatch):
from nonbonded.library.plotting.seaborn import optimization as optimization_module
# Mock the required file inputs
data_set = create_data_set("data-set-1", 1)
data_set_collection = DataSetCollection(data_sets=[data_set])
optimization = create_optimization(
"project-1",
"study-1",
"optimization-1",
[create_evaluator_target("target-1", ["data-set-1"])],
)
optimization_result = create_optimization_result("project-1", "study-1",
"optimization-1",
["target-1"], [])
# Mock the already tested plotting methods.
monkeypatch.setattr(optimization_module, "plot_parameter_changes",
lambda *args: None)
monkeypatch.setattr(optimization_module, "plot_objective_per_iteration",
lambda *args: None)
monkeypatch.setattr(optimization_module, "plot_rmse_change",
lambda *args: None)
if "nonbonded.library.factories.plots.optimization" in sys.modules:
sys.modules.pop("nonbonded.library.factories.plots.optimization")
from nonbonded.library.factories.plots.optimization import OptimizationPlotFactory
with temporary_cd():
# Save the inputs in their expected locations.
data_set_collection.to_file("test-set-collection.json")
optimization.to_file("optimization.json")
os.makedirs("analysis")
optimization_result.to_file(
os.path.join("analysis", "optimization-results.json"))
OptimizationPlotFactory.plot([""], "png")
assert os.path.isdir("plots")
def test_generate_request_options_default(self, allow_reweighting):
training_set = create_data_set("data-set-1", 1)
target = create_evaluator_target("evaluator-target-1", ["data-set-1"])
target.allow_direct_simulation = True
target.allow_reweighting = allow_reweighting
request_options = OptimizationInputFactory._generate_request_options(
target, training_set.to_evaluator())
if allow_reweighting:
assert request_options.calculation_layers == [
"ReweightingLayer",
"SimulationLayer",
]
else:
assert request_options.calculation_layers == ["SimulationLayer"]
assert request_options.calculation_schemas == UNDEFINED
def _create_model(cls, db, create_dependencies=True):
project_id = "project-1"
study_id = "study-1"
benchmark_id = "benchmark-1"
data_sets = create_data_set("data-set-1")
for index, entry in enumerate(data_sets.entries):
entry.id = index + 1
if create_dependencies:
project, study, benchmark, data_set, _, _ = commit_benchmark(
db, False)
project_id = project.id
study_id = study.id
benchmark_id = benchmark.id
data_sets = data_set
benchmark_result = create_benchmark_result(
project_id,
study_id,
benchmark_id,
data_sets,
)
return (
benchmark_result,
{
"project_id": project_id,
"study_id": study_id,
"model_id": benchmark_id
},
)
def _create_model(cls, db, create_dependencies=True):
data_set = create_data_set("data-set-1")
data_set.entries[0].id = 1
return data_set, {"data_set_id": data_set.id}
"mock-env",
"01:00",
"lilac-dask",
8000,
1,
False,
None,
optimization_result,
)
@pytest.mark.parametrize(
"reference_data_sets, expected_raises",
[
(
[create_data_set("data-set-1"), create_qc_data_set("data-set-1")],
does_not_raise(),
),
(
[
create_data_set("data-set-1"),
create_data_set("data-set-1"),
create_qc_data_set("data-set-1"),
],
pytest.raises(AssertionError, match="multiple reference data sets of"),
),
(
[
create_data_set("data-set-1"),
create_qc_data_set("data-set-1"),
create_qc_data_set("data-set-1"),
def unsuccessful_result():
result = RequestResult()
result.unsuccessful_properties = create_data_set("data-set-1",
1).to_evaluator()
return result
def create_dependencies(db: Session, dependencies: List[str]):
"""Create any dependencies such as parent studies, projects, or data sets and
commit them to the database.
Parameters
----------
db
The current database session.
dependencies
The required dependencies.
"""
project = None
data_set_ids = []
qc_data_set_ids = []
if "data-set" in dependencies:
data_set_ids.append("data-set-1")
if "qc-data-set" in dependencies:
qc_data_set_ids.append("qc-data-set-1")
for data_set_id in data_set_ids:
data_set = create_data_set(data_set_id)
db_data_set = DataSetCRUD.create(db, data_set)
db.add(db_data_set)
for qc_data_set_id in qc_data_set_ids:
qc_data_set = create_qc_data_set(qc_data_set_id)
db_qc_data_set = QCDataSetCRUD.create(db, qc_data_set)
db.add(db_qc_data_set)
db.commit()
if ("project" in dependencies or "study" in dependencies
or "evaluator-target" in dependencies
or "recharge-target" in dependencies
or "benchmark" in dependencies):
project = create_project("project-1")
if ("study" in dependencies or "evaluator-target" in dependencies
or "recharge-target" in dependencies
or "benchmark" in dependencies):
project.studies = [create_study(project.id, "study-1")]
if "evaluator-target" in dependencies or "recharge-target" in dependencies:
targets = []
if "evaluator-target" in dependencies:
targets.append(
create_evaluator_target("evaluator-target-1", ["data-set-1"]))
if "recharge-target" in dependencies:
targets.append(
create_recharge_target("recharge-target-1", ["qc-data-set-1"]))
optimization = create_optimization(project.id, project.studies[0].id,
"optimization-1", targets)
project.studies[0].optimizations = [optimization]
if "benchmark" in dependencies:
benchmark = create_benchmark(
project.id,
project.studies[0].id,
"benchmark-1",
["data-set-1"],
None,
create_force_field(),
)
project.studies[0].benchmarks = [benchmark]
if project is not None:
db_project = ProjectCRUD.create(db, project)
db.add(db_project)
db.commit()
def create_model(cls, include_children=False, index=1):
data_set = create_data_set(f"data-set-{index}")
data_set.entries[0].id = index
return data_set
def benchmarks_and_results(
force_field: ForceField,
) -> Tuple[List[Benchmark], List[BenchmarkResult], List[DataSet]]:
benchmarks = []
benchmark_results = []
data_sets = [create_data_set("data-set-1", 1)]
for index in range(2):
benchmark = create_benchmark(
"project-1",
"study-1",
f"benchmark-{index + 1}",
["data-set-1"],
None,
force_field,
)
benchmark.name = f"Benchmark {index + 1}"
benchmarks.append(benchmark)
benchmark_result = create_benchmark_result(
"project-1", "study-1", "benchmark-1", data_sets
)
for statistic_entry in benchmark_result.data_set_result.statistic_entries:
statistic_entry.value /= index + 1
statistic_entry.lower_95_ci /= index + 1
statistic_entry.upper_95_ci /= index + 1
benchmark_results.append(benchmark_result)
# benchmarks = [
# Benchmark.from_rest(
# project_id="binary-mixture",
# study_id="expanded",
# sub_study_id="openff-1-0-0",
# ),
# Benchmark.from_rest(
# project_id="binary-mixture",
# study_id="expanded",
# sub_study_id="h-mix-rho-x-rho",
# ),
# Benchmark.from_rest(
# project_id="binary-mixture", study_id="expanded", sub_study_id="h-mix-rho-x"
# ),
# ]
# benchmark_results = [
# BenchmarkResult.from_rest(
# project_id="binary-mixture", study_id="expanded", model_id="openff-1-0-0"
# ),
# BenchmarkResult.from_rest(
# project_id="binary-mixture", study_id="expanded", model_id="h-mix-rho-x-rho"
# ),
# BenchmarkResult.from_rest(
# project_id="binary-mixture", study_id="expanded", model_id="h-mix-rho-x"
# ),
# ]
#
# data_set_ids = {
# test_set_id
# for benchmark in benchmarks
# for test_set_id in benchmark.test_set_ids
# }
# data_sets = [
# DataSet.from_rest(data_set_id=data_set_id) for data_set_id in data_set_ids
# ]
return benchmarks, benchmark_results, data_sets
