def test_run_calculations(request_error, results_error, expected_raises, monkeypatch): monkeypatch.setattr(EvaluatorServerConfig, "to_backend", lambda *_: does_not_raise()) monkeypatch.setattr(EvaluatorServerConfig, "to_server", lambda *_: does_not_raise()) empty_request = Request() empty_result = RequestResult() monkeypatch.setattr( EvaluatorClient, "request_estimate", lambda *args, **kwargs: (empty_request, request_error), ) monkeypatch.setattr(Request, "results", lambda *args, **kwargs: (empty_result, results_error)) server_config = EvaluatorServerConfig( backend_config=DaskLocalClusterConfig( resources_per_worker=ComputeResources())) with expected_raises as error_info: # noinspection PyTypeChecker _run_calculations(None, None, 1, None, server_config) error_value = None if error_info is None else error_info.value assert error_value == ( request_error if request_error is not None else results_error if results_error is not None else None)
def benchmark_model_perturbations(): return [ ({ "name": "updated" }, lambda db: [], does_not_raise()), ({ "description": "updated" }, lambda db: [], does_not_raise()), ( { "analysis_environments": [ChemicalEnvironment.Hydroxy] }, lambda db: [], does_not_raise(), ), # Test updating the test_sets. ({ "test_set_ids": ["data-set-1"] }, lambda db: [], does_not_raise()), ( { "test_set_ids": ["data-set-2"] }, lambda db: [], pytest.raises(DataSetNotFoundError), ), ]
def test_launch_required_services(optimization, server_config, expected_raises, monkeypatch): monkeypatch.setattr(EvaluatorServerConfig, "to_backend", lambda *_: does_not_raise()) monkeypatch.setattr(EvaluatorServerConfig, "to_server", lambda *_: does_not_raise()) with temporary_cd(): if server_config is not None: with open("server-config.json", "w") as file: file.write(server_config.json()) server_config = "server-config.json" with expected_raises: with _launch_required_services(optimization, server_config): pass
def project_model_perturbations(): updated_author = create_author() updated_author.email = "*****@*****.**" updated_study = create_study("project-1", "study-1") updated_study.name = "updated" return [ ({ "name": "updated" }, lambda db: [], does_not_raise()), ({ "description": "updated" }, lambda db: [], does_not_raise()), ( { "authors": [create_author(), updated_author] }, lambda db: [ db.query(models.Author.email).count() == 2, db.query(author_projects_table).count() == 2, ], does_not_raise(), ), # Delete a study. ( { "studies": [create_study("project-1", "study-1")] }, lambda db: [db.query(models.Study.id).count() == 1], does_not_raise(), ), # Update a study. ( { "studies": [updated_study, create_study("project-1", "study-2")] }, lambda db: [db.query(models.Study.id).count() == 2], does_not_raise(), ), # Add a study. ( { "studies": [ create_study("project-1", f"study-{index + 1}") for index in range(3) ] }, lambda db: [db.query(models.Study.id).count() == 3], does_not_raise(), ), ]
dict( project_id="project-1", study_id="study-1", id="optimization-1", target_results={ 0: { "target-1": RechargeTargetResult( objective_function=0.0, statistic_entries=[create_statistic()], ) } }, refit_force_field=create_force_field(), ), does_not_raise(), ), ( dict( project_id="project-1", study_id="study-1", id="optimization-1", target_results={}, refit_force_field=create_force_field(), ), pytest.raises(ValidationError), ), ( dict( project_id="project-1", study_id="study-1",
create_qc_data_set, create_study, ) from nonbonded.tests.utilities.mock import ( mock_get_data_set, mock_get_project, mock_get_qc_data_set, mock_get_study, ) @pytest.mark.usefixtures("change_api_url") @pytest.mark.parametrize( "data_set_ids, data_set_type, expected_data_set_ids, expected_raises", [ (["data-set-2"], DataSet, [2], does_not_raise()), (["data-set-3"], DataSet, [4], does_not_raise()), (["data-set-2", "data-set-3"], DataSet, [2, 4], does_not_raise()), ( ["data-set-4"], DataSet, [], pytest.raises(NoMockAddress, match="phys-prop/data-set-4"), ), (["data-set-1"], QCDataSet, [], does_not_raise()), (["data-set-2"], QCDataSet, [], does_not_raise()), (["data-set-1", "data-set-2"], QCDataSet, [], does_not_raise()), ( ["data-set-4"], QCDataSet, [],
assert expected_message in caplog.text else: assert caplog.text == "" expected_n_data_points = (1 if existing_result is None else len( existing_result.unsuccessful_properties)) assert len(data_set) == expected_n_data_points @pytest.mark.parametrize( "force_fields, expected_raises, expected_error_message", [ ( [("force-field.offxml", ForceField("openff-1.0.0.offxml"))], does_not_raise(), None, ), ([("force-field.json", TLeapForceFieldSource()) ], does_not_raise(), None), ( [ ("force-field.offxml", ForceField("openff-1.0.0.offxml")), ("force-field.json", TLeapForceFieldSource()), ], pytest.raises(RuntimeError), "Two valid force fields were found", ), ([], pytest.raises(RuntimeError), "No valid force field could be found."), ],
class TestBenchmarkCRUD(BaseCRUDTest): @classmethod def crud_class(cls): return BenchmarkCRUD @classmethod def dependencies(cls): return [ "project", "study", "evaluator-target", "data-set", ] @classmethod def create_model(cls, include_children=False, index=1): benchmark = create_benchmark( "project-1", "study-1", f"benchmark-{index}", ["data-set-1"], None, create_force_field(), ) return benchmark @classmethod def model_to_read_kwargs(cls, model): return { "project_id": model.project_id, "study_id": model.study_id, "sub_study_id": model.id, } @classmethod def model_to_read_all_kwargs(cls, model): return {"project_id": model.project_id, "study_id": model.study_id} @classmethod def not_found_error(cls): return BenchmarkNotFoundError @classmethod def already_exists_error(cls): return BenchmarkExistsError @classmethod def check_has_deleted(cls, db: Session): assert db.query(models.Benchmark.id).count() == 0 # assert db.query(models.ForceField.id).count() == 0 @pytest.mark.parametrize("perturbation, database_checks, expected_raise", benchmark_model_perturbations()) def test_update(self, db: Session, perturbation, database_checks, expected_raise): super(TestBenchmarkCRUD, self).test_update(db, perturbation, database_checks, expected_raise) @pytest.mark.parametrize( "dependencies, expected_error", [(["study", "evaluator-target"], StudyNotFoundError)], ) def test_missing_dependencies(self, db: Session, dependencies: List[str], expected_error): super(TestBenchmarkCRUD, self).test_missing_dependencies(db, dependencies, expected_error) @pytest.mark.parametrize( "optimization_id, create_results, expected_error", [ ("optimization-2", True, pytest.raises(OptimizationNotFoundError)), ("optimization-1", True, does_not_raise()), ("optimization-1", False, pytest.raises(UnableToCreateError)), ], ) def test_create_read_with_optimization(self, db: Session, optimization_id: str, create_results: bool, expected_error): """Test that a benchmark can be successfully created and then retrieved out again while targeting an optimization, or raises the correct error when no results have been created.. """ create_dependencies(db, self.dependencies()) model = self.create_model(True) model.force_field = None model.optimization_id = optimization_id # Create the optimization results if create_results: db_result = OptimizationResultCRUD.create( db, create_optimization_result( model.project_id, model.study_id, "optimization-1", ["evaluator-target-1"], [], ), ) db.add(db_result) db.commit() with expected_error: self.test_create_read(db, False, model) @pytest.mark.parametrize( "optimization_id, create_results, expected_error", [ ("optimization-2", True, pytest.raises(OptimizationNotFoundError)), ("optimization-1", True, does_not_raise()), ("optimization-1", False, pytest.raises(UnableToUpdateError)), ], ) def test_update_with_dependant(self, db: Session, optimization_id: str, create_results: bool, expected_error): """Test that a benchmark can be updated to target an optimization and then back to a force field. """ create_dependencies(db, self.dependencies()) model = self.create_model() db.add(self.crud_class().create(db, model)) db.commit() # Create the optimization results if create_results: db_result = OptimizationResultCRUD.create( db, create_optimization_result( model.project_id, model.study_id, "optimization-1", ["evaluator-target-1"], [], ), ) db.add(db_result) db.commit() # Update the model. model.force_field = None model.optimization_id = optimization_id with expected_error: update_and_compare_model( db, model, self.crud_class().update, functools.partial(self.crud_class().read, **self.model_to_read_kwargs(model)), self.crud_class().db_to_model, ) # Neither the refit force field nor the initial force field # should be deleted. assert db.query( models.ForceField.id).count() == 2 if create_results else 1 # Update the model back. model.force_field = create_force_field() model.optimization_id = None update_and_compare_model( db, model, self.crud_class().update, functools.partial(self.crud_class().read, **self.model_to_read_kwargs(model)), self.crud_class().db_to_model, ) assert db.query( models.ForceField.id).count() == 2 if create_results else 1 @pytest.mark.parametrize( "create_results, expected_error", [(False, does_not_raise()), (True, pytest.raises(UnableToDeleteError))], ) def test_delete_with_dependent(self, db: Session, create_results: bool, expected_error): """Test that a benchmark cannot be deleted until its results have also been deleted. """ create_dependencies(db, self.dependencies()) model = self.create_model() db.add(self.crud_class().create(db, model)) db.commit() # Create the benchmark results if create_results: data_set = create_data_set("data-set-1") data_set.entries[0].id = 1 db_result = BenchmarkResultCRUD.create( db, create_benchmark_result(model.project_id, model.study_id, model.id, data_set), ) db.add(db_result) db.commit() # Delete the model. with expected_error: self.crud_class().delete(db, model.project_id, model.study_id, model.id) if not create_results: return BenchmarkResultCRUD.delete(db, model.project_id, model.study_id, model.id) db.commit() self.crud_class().delete(db, model.project_id, model.study_id, model.id) db.commit() self.check_has_deleted(db)
def optimization_model_perturbations(): updated_engine_delete_prior = ForceBalance( priors={"vdW/Atom/epsilon": 0.1}) updated_engine_update_prior = ForceBalance(priors={ "vdW/Atom/epsilon": 0.2, "vdW/Atom/sigma": 2.0 }, ) updated_engine_add_prior = ForceBalance(priors={ "vdW/Atom/epsilon": 0.1, "vdW/Atom/sigma": 2.0, "vdW/Atom/r_min": 2.0 }, ) invalid_evaluator_target = create_evaluator_target("evaluator-target-1", ["data-set-999"]) invalid_recharge_target = create_recharge_target("recharge-target-1", ["qc-data-set-999"]) return [ ({ "name": "updated" }, lambda db: [], does_not_raise()), ({ "description": "updated" }, lambda db: [], does_not_raise()), ({ "max_iterations": 999 }, lambda db: [], does_not_raise()), ( { "analysis_environments": [ChemicalEnvironment.Hydroxy] }, lambda db: [], does_not_raise(), ), # Test updating the force field. ( { "force_field": create_force_field("updated") }, lambda db: [ "updated" in db.query(models.ForceField.inner_content).first()[ 0], db.query(models.ForceField.id).count() == 1, ], does_not_raise(), ), # Test updating the parameters to train. ( { "parameters_to_train": [ Parameter(handler_type="vdW", smirks="[#6:1]", attribute_name="epsilon"), ] }, lambda db: [db.query(models.Parameter.id).count() == 1], does_not_raise(), ), ( { "parameters_to_train": [ Parameter(handler_type="vdW", smirks="[#6:1]", attribute_name="epsilon"), Parameter(handler_type="vdW", smirks="[#6:1]", attribute_name="sigma"), Parameter(handler_type="vdW", smirks="[#1:1]", attribute_name="sigma"), ] }, lambda db: [db.query(models.Parameter.id).count() == 3], does_not_raise(), ), # Test updating an engine's priors ( { "engine": updated_engine_delete_prior }, lambda db: [db.query(models.ForceBalancePrior.id).count() == 1], does_not_raise(), ), ( { "engine": updated_engine_update_prior }, lambda db: [db.query(models.ForceBalancePrior.id).count() == 2], does_not_raise(), ), ( { "engine": updated_engine_add_prior }, lambda db: [db.query(models.ForceBalancePrior.id).count() == 3], does_not_raise(), ), # Test deleting a target ( { "targets": [ create_evaluator_target("evaluator-target-1", ["data-set-1"]) ] }, lambda db: [ db.query(models.EvaluatorTarget.id).count() == 1, db.query(models.RechargeTarget.id).count() == 0, ], does_not_raise(), ), ( { "targets": [ create_recharge_target("recharge-target-1", ["qc-data-set-1"]) ] }, lambda db: [ db.query(models.EvaluatorTarget.id).count() == 0, db.query(models.RechargeTarget.id).count() == 1, ], does_not_raise(), ), # Test adding a target ( { "targets": [ create_evaluator_target("evaluator-target-1", ["data-set-1"]), create_evaluator_target("evaluator-target-2", ["data-set-1"]), create_recharge_target("recharge-target-1", ["qc-data-set-1"]), ] }, lambda db: [ db.query(models.EvaluatorTarget.id).count() == 2, db.query(models.RechargeTarget.id).count() == 1, ], does_not_raise(), ), ( { "targets": [ create_evaluator_target("evaluator-target-1", ["data-set-1"]), create_recharge_target("recharge-target-1", ["qc-data-set-1"]), create_recharge_target("recharge-target-2", ["qc-data-set-1"]), ] }, lambda db: [ db.query(models.EvaluatorTarget.id).count() == 1, db.query(models.RechargeTarget.id).count() == 2, ], does_not_raise(), ), # Test invalidly updating a target's training set ( { "targets": [invalid_evaluator_target] }, lambda db: [], pytest.raises(DataSetNotFoundError), ), ( { "targets": [invalid_recharge_target] }, lambda db: [], pytest.raises(QCDataSetNotFoundError), ), ]
def study_model_perturbations(): updated_optimization = create_optimization( "project-1", "study-1", "optimization-1", [create_evaluator_target("evaluator-target-1", ["data-set-1"])], ) updated_optimization.max_iterations += 1 updated_benchmark = create_benchmark( "project-1", "study-1", "benchmark-1", ["data-set-1"], None, create_force_field(), ) updated_benchmark.name = "updated" return [ ({ "name": "updated" }, lambda db: [], does_not_raise()), ({ "description": "updated" }, lambda db: [], does_not_raise()), # Delete an optimization. ( { "optimizations": [] }, lambda db: [db.query(models.Optimization.id).count() == 0], does_not_raise(), ), # Update an optimization. ( { "optimizations": [updated_optimization] }, lambda db: [db.query(models.Optimization.id).count() == 1], does_not_raise(), ), # Add an optimization. ( { "optimizations": [ create_optimization( "project-1", "study-1", f"optimization-{index + 1}", [ create_evaluator_target("evaluator-target-1", ["data-set-1"]) ], ) for index in range(2) ] }, lambda db: [db.query(models.Optimization.id).count() == 2], does_not_raise(), ), # Delete a benchmark. ( { "benchmarks": [] }, lambda db: [db.query(models.Benchmark.id).count() == 0], does_not_raise(), ), # Update a benchmark. ( { "benchmarks": [updated_benchmark] }, lambda db: [db.query(models.Benchmark.id).count() == 1], does_not_raise(), ), # Add a benchmark. ( { "benchmarks": [ create_benchmark( "project-1", "study-1", f"benchmark-{index + 1}", ["data-set-1"], None, create_force_field(), ) for index in range(2) ] }, lambda db: [db.query(models.Benchmark.id).count() == 2], does_not_raise(), ), ]