def test_spec_validation_pass(qc_spec: QCOptions): """ Make sure we can correctly validate a specification. """ if qc_spec.program.lower() not in qcengine.list_available_programs(): pytest.skip(f"{qc_spec.program} missing skipping test") qc_spec.validate_specification()
def test_single_point_energy(program, basis, method, tmpdir): """ Make sure our qcengine wrapper works correctly. """ if program not in qcengine.list_available_programs(): pytest.skip(f"{program} missing skipping test.") with tmpdir.as_cwd(): mol = Ligand.from_file(file_name=get_data("water.pdb")) engine = QCEngine( program=program, basis=basis, method=method, memory=1, cores=1, driver="energy", ) result = engine.call_qcengine(molecule=mol) assert result.driver == "energy" assert result.model.basis == basis assert result.model.method == method assert result.provenance.creator.lower() == program # make sure the grid was set to ultrafine for psi4 if program == "psi4": assert result.keywords["dft_spherical_points"] == 590 assert result.keywords["dft_radial_points"] == 99
def test_spec_validation_pass(program, basis, method): """ Make sure we can correctly validate a specification. """ if program not in qcengine.list_available_programs(): pytest.skip(f"{program} missing skipping test") _ = GeometryOptimiser(program=program, basis=basis, method=method)
def test_spec_validation_fail(program, basis, method): """ Make sure than an invalid specification raises an error. """ if program not in qcengine.list_available_programs(): pytest.skip(f"{program} missing skipping test") with pytest.raises(SpecificationError): _ = GeometryOptimiser(program=program, basis=basis, method=method)
def is_program_new_enough(program, version_feature_introduced): """Returns True if `program` registered in QCEngine, locatable in environment, has parseable version, and that version in normalized form is equal to or later than `version_feature_introduced`. """ if program not in qcng.list_available_programs(): return False candidate_version = qcng.get_program(program).get_version() return parse_version(candidate_version) >= parse_version(version_feature_introduced)
def validate_program(self): """ Validate the choice of program against those supported by QCEngine and QUBEKit. """ programs = qcng.list_available_programs() programs.discard("dftd3") if self.program.lower() not in programs: raise SpecificationError( f"The program {self.program} is not available, available programs are {programs}" )
def test_optimiser_keywords(optimiser): """ For the given optimiser make sure the keywords are updated correctly. """ if "psi4" not in qcengine.list_available_programs(): pytest.skip("Psi4 missing skipping.") g = GeometryOptimiser( optimiser=optimiser, maxiter=1, convergence="GAU", program="psi4" ) keywords = g.build_optimiser_keywords() assert 1 in keywords.values() assert "GAU" in keywords.values()
def test_optmiser_fail_no_output(tmpdir): """ Make sure we raise an error correctly when there is no output from a failed optimisation. """ if "psi4" not in qcengine.list_available_programs(): pytest.skip("Psi4 missing skipping test.") with tmpdir.as_cwd(): mol = Ligand.from_file(file_name=get_data("water.pdb")) g = GeometryOptimiser( program="psi4", method="wb97x-dbj", basis="dzvp", maxiter=10 ) with pytest.raises(RuntimeError): g.optimise(molecule=mol, allow_fail=False)
def test_optimise(program, basis, method, tmpdir): """ Test running different optimisers with different programs. """ if program not in qcengine.list_available_programs(): pytest.skip(f"{program} missing skipping test.") with tmpdir.as_cwd(): mol = Ligand.from_file(get_data("water.pdb")) g = GeometryOptimiser( program=program, basis=basis, method=method, optimiser="geometric", convergence="GAU", ) result_mol, _ = g.optimise(molecule=mol, return_result=False) assert result_mol.coordinates.tolist() != mol.coordinates.tolist()
def test_optimise(qc_spec: QCOptions, tmpdir): """ Test running different optimisers with different programs. """ if qc_spec.program.lower() not in qcengine.list_available_programs(): pytest.skip(f"{qc_spec.program} missing skipping test.") with tmpdir.as_cwd(): mol = Ligand.from_file(get_data("water.pdb")) g = GeometryOptimiser( optimiser="geometric", convergence="GAU", ) result_mol, _ = g.optimise( molecule=mol, return_result=False, qc_spec=qc_spec, local_options=LocalResource(cores=1, memory=1), ) assert result_mol.coordinates.tolist() != mol.coordinates.tolist()
def test_quick_run_to_seminario(tmpdir): """ Do a quick run through each stage in run up to the modified Seminario stage. Run on water with a very small basis. """ if "psi4" not in qcengine.list_available_programs(): pytest.skip("Psi4 missing skipping test") with tmpdir.as_cwd(): water = Ligand.from_file(get_data("water.pdb")) water.home = os.getcwd() # make sure coverage picks up pool code water.threads = 1 water.parameter_engine = "openff" # parmetrise water = Execute.parametrise(molecule=water, verbose=False) # pre opt water.pre_opt_method = "mmff94" water_new_coords = Execute.pre_optimise(molecule=water) assert not np.allclose(water.coordinates, water_new_coords.coordinates) # qm optimise water_new_coords.bonds_engine = "psi4" water_new_coords.theory = "HF" water_new_coords.basis = "STO-3G" water_new_coords.threads = 1 water_qm = Execute.qm_optimise(molecule=water_new_coords) assert not np.allclose(water_qm.coordinates, water_new_coords.coordinates) # hessian water_hess = Execute.hessian(molecule=water_qm) # mod sem mod_sem = ModSeminario() final_water = mod_sem.run(molecule=water_hess) # make sure we have symmetry in parameters assert (final_water.BondForce[(0, 1)].length == final_water.BondForce[( 0, 2)].length) assert final_water.BondForce[(0, 1)].k == final_water.BondForce[(0, 2)].k # make sure they are different from the input assert final_water.BondForce[(0, 1)].k != pytest.approx( water.BondForce[(0, 1)].k)
def test_single_point_energy(qc_options: QCOptions, tmpdir, water): """ Make sure our qcengine wrapper works correctly. """ if qc_options.program.lower() not in qcengine.list_available_programs(): pytest.skip(f"{qc_options.program} missing skipping test.") with tmpdir.as_cwd(): result = call_qcengine( molecule=water, driver="energy", qc_spec=qc_options, local_options=LocalResource(cores=1, memory=1), ) assert result.driver == "energy" assert result.model.basis == qc_options.basis assert result.model.method == qc_options.method assert result.provenance.creator.lower() == qc_options.program # make sure the grid was set to ultrafine for psi4 if qc_options.program == "psi4": assert result.keywords["dft_spherical_points"] == 590 assert result.keywords["dft_radial_points"] == 99
_programs = { "fireworks": _plugin_import("fireworks"), "rdkit": _plugin_import("rdkit"), "psi4": _plugin_import("psi4"), "parsl": _plugin_import("parsl"), "dask": _plugin_import("dask"), "geometric": _plugin_import("geometric"), "torsiondrive": _plugin_import("torsiondrive"), "torchani": _plugin_import("torchani"), } if _programs["dask"]: _programs["dask.distributed"] = _plugin_import("dask.distributed") else: _programs["dask.distributed"] = False _programs["dftd3"] = "dftd3" in qcng.list_available_programs() def has_module(name): return _programs[name] def _build_pytest_skip(program): import_message = "Not detecting module {}. Install package if necessary to enable tests." return pytest.mark.skipif(has_module(program) is False, reason=import_message.format(program)) # Add a number of module testing options using_dask = _build_pytest_skip('dask.distributed') using_dftd3 = _build_pytest_skip('dftd3')
def test_check_program_avail(program): assert program in qcng.list_available_programs()
def __init__(self, client: Any, queue_client: Any, logger: Optional[logging.Logger] = None, max_tasks: int = 200, queue_tag: str = None, manager_name: str = "unlabeled", update_frequency: Union[int, float] = 2, verbose: bool = True, server_error_retries: Optional[int] = 1, stale_update_limit: Optional[int] = 10, cores_per_task: Optional[int] = None, memory_per_task: Optional[Union[int, float]] = None, scratch_directory: Optional[str] = None, retries: Optional[int] = 2): """ Parameters ---------- client : FractalClient A FractalClient connected to a server queue_client : QueueAdapter The DBAdapter class for queue abstraction logger : logging.Logger, Optional. Default: None A logger for the QueueManager max_tasks : int The maximum number of tasks to hold at any given time queue_tag : str Allows managers to pull from specific tags manager_name : str The cluster the manager belongs to update_frequency : int The frequency to check for new tasks in seconds verbose: bool, optional, Default: True Whether or not to have the manager be verbose (logger level debug and up) server_error_retries: int, optional, Default: 1 How many times finished jobs are attempted to be pushed to the server in in the event of a server communication error. After number of attempts, the failed jobs are dropped from this manager and considered "stale" Set to `None` to keep retrying stale_update_limit: int, optional, Default: 10 Number of stale update attempts to keep around If this limit is ever hit, the server initiates as shutdown as best it can since communication with the server has gone wrong too many times. Set to `None` for unlimited cores_per_task : int, optional, Default: None How many CPU cores per computation task to allocate for QCEngine None indicates "use however many you can detect" memory_per_task: int, optional, Default: None How much memory, in GiB, per computation task to allocate for QCEngine None indicates "use however much you can consume" scratch_directory : str, optional, Default: None Scratch directory location to do QCEngine compute None indicates "wherever the system default is"' retries : int, optional, Default: 2 Number of retries that QCEngine will attempt for RandomErrors detected when running its computations. After this many attempts (or on any other type of error), the error will be raised. """ # Setup logging if logger: self.logger = logger else: self.logger = logging.getLogger('QueueManager') self.name_data = {"cluster": manager_name, "hostname": socket.gethostname(), "uuid": str(uuid.uuid4())} self._name = self.name_data["cluster"] + "-" + self.name_data["hostname"] + "-" + self.name_data["uuid"] self.client = client self.cores_per_task = cores_per_task self.memory_per_task = memory_per_task self.scratch_directory = scratch_directory self.retries = retries self.queue_adapter = build_queue_adapter(queue_client, logger=self.logger, cores_per_task=self.cores_per_task, memory_per_task=self.memory_per_task, scratch_directory=self.scratch_directory, retries=self.retries, verbose=verbose) self.max_tasks = max_tasks self.queue_tag = queue_tag self.verbose = verbose self.statistics = QueueStatistics(max_concurrent_tasks=self.max_tasks, cores_per_task=cores_per_task, update_frequency=update_frequency ) self.scheduler = None self.update_frequency = update_frequency self.periodic = {} self.active = 0 self.exit_callbacks = [] # Server response/stale job handling self.server_error_retries = server_error_retries self.stale_update_limit = stale_update_limit self._stale_updates_tracked = 0 self._stale_payload_tracking = [] self.n_stale_jobs = 0 # QCEngine data self.available_programs = qcng.list_available_programs() self.available_procedures = qcng.list_available_procedures() self.logger.info("QueueManager:") self.logger.info(" Version: {}\n".format(get_information("version"))) if self.verbose: self.logger.info(" Name Information:") self.logger.info(" Cluster: {}".format(self.name_data["cluster"])) self.logger.info(" Hostname: {}".format(self.name_data["hostname"])) self.logger.info(" UUID: {}\n".format(self.name_data["uuid"])) self.logger.info(" Queue Adapter:") self.logger.info(" {}\n".format(self.queue_adapter)) if self.verbose: self.logger.info(" QCEngine:") self.logger.info(" Version: {}".format(qcng.__version__)) self.logger.info(" Task Cores: {}".format(self.cores_per_task)) self.logger.info(" Task Mem: {}".format(self.memory_per_task)) self.logger.info(" Scratch Dir: {}".format(self.scratch_directory)) self.logger.info(" Programs: {}".format(self.available_programs)) self.logger.info(" Procedures: {}\n".format(self.available_procedures)) # DGAS Note: Note super happy about how this if/else turned out. Looking for alternatives. if self.connected(): # Pull server info self.server_info = client.server_information() self.server_name = self.server_info["name"] self.server_version = self.server_info["version"] self.server_query_limit = self.server_info["query_limit"] if self.max_tasks > self.server_query_limit: self.max_tasks = self.server_query_limit self.logger.warning( "Max tasks was larger than server query limit of {}, reducing to match query limit.".format( self.server_query_limit)) self.heartbeat_frequency = self.server_info["heartbeat_frequency"] # Tell the server we are up and running payload = self._payload_template() payload["data"]["operation"] = "startup" self.client._automodel_request("queue_manager", "put", payload) if self.verbose: self.logger.info(" Connected:") self.logger.info(" Version: {}".format(self.server_version)) self.logger.info(" Address: {}".format(self.client.address)) self.logger.info(" Name: {}".format(self.server_name)) self.logger.info(" Queue tag: {}".format(self.queue_tag)) self.logger.info(" Username: {}\n".format(self.client.username)) else: self.logger.info(" QCFractal server information:") self.logger.info(" Not connected, some actions will not be available")
def __init__(self, client: Any, queue_client: Any, logger: Optional[logging.Logger] = None, max_tasks: int = 200, queue_tag: str = None, manager_name: str = "unlabled", update_frequency: Union[int, float] = 2, verbose: bool = True, cores_per_task: Optional[int] = None, memory_per_task: Optional[Union[int, float]] = None, scratch_directory: Optional[str] = None): """ Parameters ---------- client : FractalClient A FractalClient connected to a server queue_client : QueueAdapter The DBAdapter class for queue abstraction storage_socket : DBSocket A socket for the backend database logger : logging.Logger, Optional. Default: None A logger for the QueueManager max_tasks : int The maximum number of tasks to hold at any given time queue_tag : str Allows managers to pull from specific tags manager_name : str The cluster the manager belongs to update_frequency : int The frequency to check for new tasks in seconds cores_per_task : int, optional, Default: None How many CPU cores per computation task to allocate for QCEngine None indicates "use however many you can detect" memory_per_task: int, optional, Default: None How much memory, in GiB, per computation task to allocate for QCEngine None indicates "use however much you can consume" scratch_directory: str, optional, Default: None Scratch directory location to do QCEngine compute None indicates "wherever the system default is" """ # Setup logging if logger: self.logger = logger else: self.logger = logging.getLogger('QueueManager') self.name_data = { "cluster": manager_name, "hostname": socket.gethostname(), "uuid": str(uuid.uuid4()) } self._name = self.name_data["cluster"] + "-" + self.name_data[ "hostname"] + "-" + self.name_data["uuid"] self.client = client self.cores_per_task = cores_per_task self.memory_per_task = memory_per_task self.scratch_directory = scratch_directory self.queue_adapter = build_queue_adapter( queue_client, logger=self.logger, cores_per_task=self.cores_per_task, memory_per_task=self.memory_per_task, scratch_directory=self.scratch_directory) self.max_tasks = max_tasks self.queue_tag = queue_tag self.verbose = verbose self.scheduler = None self.update_frequency = update_frequency self.periodic = {} self.active = 0 self.exit_callbacks = [] # QCEngine data self.available_programs = qcng.list_available_programs() self.available_procedures = qcng.list_available_procedures() self.logger.info("QueueManager:") self.logger.info(" Version: {}\n".format( get_information("version"))) if self.verbose: self.logger.info(" Name Information:") self.logger.info(" Cluster: {}".format( self.name_data["cluster"])) self.logger.info(" Hostname: {}".format( self.name_data["hostname"])) self.logger.info(" UUID: {}\n".format( self.name_data["uuid"])) self.logger.info(" Queue Adapter:") self.logger.info(" {}\n".format(self.queue_adapter)) if self.verbose: self.logger.info(" QCEngine:") self.logger.info(" Version: {}".format( qcng.__version__)) self.logger.info(" Task Cores: {}".format( self.cores_per_task)) self.logger.info(" Task Mem: {}".format( self.memory_per_task)) self.logger.info(" Scratch Dir: {}".format( self.scratch_directory)) self.logger.info(" Programs: {}".format( self.available_programs)) self.logger.info(" Procedures: {}\n".format( self.available_procedures)) # DGAS Note: Note super happy about how this if/else turned out. Looking for alternatives. if self.connected(): # Pull server info self.server_info = client.server_information() self.server_name = self.server_info["name"] self.server_version = self.server_info["version"] self.server_query_limit = self.server_info["query_limit"] if self.max_tasks > self.server_query_limit: self.max_tasks = self.server_query_limit self.logger.warning( "Max tasks was larger than server query limit of {}, reducing to match query limit." .format(self.server_query_limit)) self.heartbeat_frequency = self.server_info["heartbeat_frequency"] # Tell the server we are up and running payload = self._payload_template() payload["data"]["operation"] = "startup" response = self.client._automodel_request("queue_manager", "put", payload) if self.verbose: self.logger.info(" Connected:") self.logger.info(" Version: {}".format( self.server_version)) self.logger.info(" Address: {}".format( self.client.address)) self.logger.info(" Name: {}".format( self.server_name)) self.logger.info(" Queue tag: {}".format( self.queue_tag)) self.logger.info(" Username: {}\n".format( self.client.username)) else: self.logger.info(" QCFractal server information:") self.logger.info( " Not connected, some actions will not be available")