def test_spec_validation_pass(qc_spec: QCOptions):
"""
Make sure we can correctly validate a specification.
"""
if qc_spec.program.lower() not in qcengine.list_available_programs():
pytest.skip(f"{qc_spec.program} missing skipping test")
qc_spec.validate_specification()
def test_single_point_energy(program, basis, method, tmpdir):
"""
Make sure our qcengine wrapper works correctly.
"""
if program not in qcengine.list_available_programs():
pytest.skip(f"{program} missing skipping test.")
with tmpdir.as_cwd():
mol = Ligand.from_file(file_name=get_data("water.pdb"))
engine = QCEngine(
program=program,
basis=basis,
method=method,
memory=1,
cores=1,
driver="energy",
)
result = engine.call_qcengine(molecule=mol)
assert result.driver == "energy"
assert result.model.basis == basis
assert result.model.method == method
assert result.provenance.creator.lower() == program
# make sure the grid was set to ultrafine for psi4
if program == "psi4":
assert result.keywords["dft_spherical_points"] == 590
assert result.keywords["dft_radial_points"] == 99
def test_spec_validation_pass(program, basis, method):
"""
Make sure we can correctly validate a specification.
"""
if program not in qcengine.list_available_programs():
pytest.skip(f"{program} missing skipping test")
_ = GeometryOptimiser(program=program, basis=basis, method=method)
def test_spec_validation_fail(program, basis, method):
"""
Make sure than an invalid specification raises an error.
"""
if program not in qcengine.list_available_programs():
pytest.skip(f"{program} missing skipping test")
with pytest.raises(SpecificationError):
_ = GeometryOptimiser(program=program, basis=basis, method=method)
def is_program_new_enough(program, version_feature_introduced):
"""Returns True if `program` registered in QCEngine, locatable in
environment, has parseable version, and that version in normalized
form is equal to or later than `version_feature_introduced`.
"""
if program not in qcng.list_available_programs():
return False
candidate_version = qcng.get_program(program).get_version()
return parse_version(candidate_version) >= parse_version(version_feature_introduced)
def validate_program(self):
"""
Validate the choice of program against those supported by QCEngine and QUBEKit.
"""
programs = qcng.list_available_programs()
programs.discard("dftd3")
if self.program.lower() not in programs:
raise SpecificationError(
f"The program {self.program} is not available, available programs are {programs}"
)
def test_optimiser_keywords(optimiser):
"""
For the given optimiser make sure the keywords are updated correctly.
"""
if "psi4" not in qcengine.list_available_programs():
pytest.skip("Psi4 missing skipping.")
g = GeometryOptimiser(
optimiser=optimiser, maxiter=1, convergence="GAU", program="psi4"
)
keywords = g.build_optimiser_keywords()
assert 1 in keywords.values()
assert "GAU" in keywords.values()
def test_optmiser_fail_no_output(tmpdir):
"""
Make sure we raise an error correctly when there is no output from a failed optimisation.
"""
if "psi4" not in qcengine.list_available_programs():
pytest.skip("Psi4 missing skipping test.")
with tmpdir.as_cwd():
mol = Ligand.from_file(file_name=get_data("water.pdb"))
g = GeometryOptimiser(
program="psi4", method="wb97x-dbj", basis="dzvp", maxiter=10
)
with pytest.raises(RuntimeError):
g.optimise(molecule=mol, allow_fail=False)
def test_optimise(program, basis, method, tmpdir):
"""
Test running different optimisers with different programs.
"""
if program not in qcengine.list_available_programs():
pytest.skip(f"{program} missing skipping test.")
with tmpdir.as_cwd():
mol = Ligand.from_file(get_data("water.pdb"))
g = GeometryOptimiser(
program=program,
basis=basis,
method=method,
optimiser="geometric",
convergence="GAU",
)
result_mol, _ = g.optimise(molecule=mol, return_result=False)
assert result_mol.coordinates.tolist() != mol.coordinates.tolist()
def test_optimise(qc_spec: QCOptions, tmpdir):
"""
Test running different optimisers with different programs.
"""
if qc_spec.program.lower() not in qcengine.list_available_programs():
pytest.skip(f"{qc_spec.program} missing skipping test.")
with tmpdir.as_cwd():
mol = Ligand.from_file(get_data("water.pdb"))
g = GeometryOptimiser(
optimiser="geometric",
convergence="GAU",
)
result_mol, _ = g.optimise(
molecule=mol,
return_result=False,
qc_spec=qc_spec,
local_options=LocalResource(cores=1, memory=1),
)
assert result_mol.coordinates.tolist() != mol.coordinates.tolist()
def test_quick_run_to_seminario(tmpdir):
"""
Do a quick run through each stage in run up to the modified Seminario stage.
Run on water with a very small basis.
"""
if "psi4" not in qcengine.list_available_programs():
pytest.skip("Psi4 missing skipping test")
with tmpdir.as_cwd():
water = Ligand.from_file(get_data("water.pdb"))
water.home = os.getcwd()
# make sure coverage picks up pool code
water.threads = 1
water.parameter_engine = "openff"
# parmetrise
water = Execute.parametrise(molecule=water, verbose=False)
# pre opt
water.pre_opt_method = "mmff94"
water_new_coords = Execute.pre_optimise(molecule=water)
assert not np.allclose(water.coordinates, water_new_coords.coordinates)
# qm optimise
water_new_coords.bonds_engine = "psi4"
water_new_coords.theory = "HF"
water_new_coords.basis = "STO-3G"
water_new_coords.threads = 1
water_qm = Execute.qm_optimise(molecule=water_new_coords)
assert not np.allclose(water_qm.coordinates,
water_new_coords.coordinates)
# hessian
water_hess = Execute.hessian(molecule=water_qm)
# mod sem
mod_sem = ModSeminario()
final_water = mod_sem.run(molecule=water_hess)
# make sure we have symmetry in parameters
assert (final_water.BondForce[(0, 1)].length == final_water.BondForce[(
0, 2)].length)
assert final_water.BondForce[(0, 1)].k == final_water.BondForce[(0,
2)].k
# make sure they are different from the input
assert final_water.BondForce[(0, 1)].k != pytest.approx(
water.BondForce[(0, 1)].k)
def test_single_point_energy(qc_options: QCOptions, tmpdir, water):
"""
Make sure our qcengine wrapper works correctly.
"""
if qc_options.program.lower() not in qcengine.list_available_programs():
pytest.skip(f"{qc_options.program} missing skipping test.")
with tmpdir.as_cwd():
result = call_qcengine(
molecule=water,
driver="energy",
qc_spec=qc_options,
local_options=LocalResource(cores=1, memory=1),
)
assert result.driver == "energy"
assert result.model.basis == qc_options.basis
assert result.model.method == qc_options.method
assert result.provenance.creator.lower() == qc_options.program
# make sure the grid was set to ultrafine for psi4
if qc_options.program == "psi4":
assert result.keywords["dft_spherical_points"] == 590
assert result.keywords["dft_radial_points"] == 99
_programs = {
"fireworks": _plugin_import("fireworks"),
"rdkit": _plugin_import("rdkit"),
"psi4": _plugin_import("psi4"),
"parsl": _plugin_import("parsl"),
"dask": _plugin_import("dask"),
"geometric": _plugin_import("geometric"),
"torsiondrive": _plugin_import("torsiondrive"),
"torchani": _plugin_import("torchani"),
}
if _programs["dask"]:
_programs["dask.distributed"] = _plugin_import("dask.distributed")
else:
_programs["dask.distributed"] = False
_programs["dftd3"] = "dftd3" in qcng.list_available_programs()
def has_module(name):
return _programs[name]
def _build_pytest_skip(program):
import_message = "Not detecting module {}. Install package if necessary to enable tests."
return pytest.mark.skipif(has_module(program) is False,
reason=import_message.format(program))
# Add a number of module testing options
using_dask = _build_pytest_skip('dask.distributed')
using_dftd3 = _build_pytest_skip('dftd3')
def test_check_program_avail(program):
assert program in qcng.list_available_programs()
def __init__(self,
client: Any,
queue_client: Any,
logger: Optional[logging.Logger] = None,
max_tasks: int = 200,
queue_tag: str = None,
manager_name: str = "unlabeled",
update_frequency: Union[int, float] = 2,
verbose: bool = True,
server_error_retries: Optional[int] = 1,
stale_update_limit: Optional[int] = 10,
cores_per_task: Optional[int] = None,
memory_per_task: Optional[Union[int, float]] = None,
scratch_directory: Optional[str] = None,
retries: Optional[int] = 2):
"""
Parameters
----------
client : FractalClient
A FractalClient connected to a server
queue_client : QueueAdapter
The DBAdapter class for queue abstraction
logger : logging.Logger, Optional. Default: None
A logger for the QueueManager
max_tasks : int
The maximum number of tasks to hold at any given time
queue_tag : str
Allows managers to pull from specific tags
manager_name : str
The cluster the manager belongs to
update_frequency : int
The frequency to check for new tasks in seconds
verbose: bool, optional, Default: True
Whether or not to have the manager be verbose (logger level debug and up)
server_error_retries: int, optional, Default: 1
How many times finished jobs are attempted to be pushed to the server in
in the event of a server communication error.
After number of attempts, the failed jobs are dropped from this manager and considered "stale"
Set to `None` to keep retrying
stale_update_limit: int, optional, Default: 10
Number of stale update attempts to keep around
If this limit is ever hit, the server initiates as shutdown as best it can
since communication with the server has gone wrong too many times.
Set to `None` for unlimited
cores_per_task : int, optional, Default: None
How many CPU cores per computation task to allocate for QCEngine
None indicates "use however many you can detect"
memory_per_task: int, optional, Default: None
How much memory, in GiB, per computation task to allocate for QCEngine
None indicates "use however much you can consume"
scratch_directory : str, optional, Default: None
Scratch directory location to do QCEngine compute
None indicates "wherever the system default is"'
retries : int, optional, Default: 2
Number of retries that QCEngine will attempt for RandomErrors detected when running
its computations. After this many attempts (or on any other type of error), the
error will be raised.
"""
# Setup logging
if logger:
self.logger = logger
else:
self.logger = logging.getLogger('QueueManager')
self.name_data = {"cluster": manager_name, "hostname": socket.gethostname(), "uuid": str(uuid.uuid4())}
self._name = self.name_data["cluster"] + "-" + self.name_data["hostname"] + "-" + self.name_data["uuid"]
self.client = client
self.cores_per_task = cores_per_task
self.memory_per_task = memory_per_task
self.scratch_directory = scratch_directory
self.retries = retries
self.queue_adapter = build_queue_adapter(queue_client,
logger=self.logger,
cores_per_task=self.cores_per_task,
memory_per_task=self.memory_per_task,
scratch_directory=self.scratch_directory,
retries=self.retries,
verbose=verbose)
self.max_tasks = max_tasks
self.queue_tag = queue_tag
self.verbose = verbose
self.statistics = QueueStatistics(max_concurrent_tasks=self.max_tasks,
cores_per_task=cores_per_task,
update_frequency=update_frequency
)
self.scheduler = None
self.update_frequency = update_frequency
self.periodic = {}
self.active = 0
self.exit_callbacks = []
# Server response/stale job handling
self.server_error_retries = server_error_retries
self.stale_update_limit = stale_update_limit
self._stale_updates_tracked = 0
self._stale_payload_tracking = []
self.n_stale_jobs = 0
# QCEngine data
self.available_programs = qcng.list_available_programs()
self.available_procedures = qcng.list_available_procedures()
self.logger.info("QueueManager:")
self.logger.info(" Version: {}\n".format(get_information("version")))
if self.verbose:
self.logger.info(" Name Information:")
self.logger.info(" Cluster: {}".format(self.name_data["cluster"]))
self.logger.info(" Hostname: {}".format(self.name_data["hostname"]))
self.logger.info(" UUID: {}\n".format(self.name_data["uuid"]))
self.logger.info(" Queue Adapter:")
self.logger.info(" {}\n".format(self.queue_adapter))
if self.verbose:
self.logger.info(" QCEngine:")
self.logger.info(" Version: {}".format(qcng.__version__))
self.logger.info(" Task Cores: {}".format(self.cores_per_task))
self.logger.info(" Task Mem: {}".format(self.memory_per_task))
self.logger.info(" Scratch Dir: {}".format(self.scratch_directory))
self.logger.info(" Programs: {}".format(self.available_programs))
self.logger.info(" Procedures: {}\n".format(self.available_procedures))
# DGAS Note: Note super happy about how this if/else turned out. Looking for alternatives.
if self.connected():
# Pull server info
self.server_info = client.server_information()
self.server_name = self.server_info["name"]
self.server_version = self.server_info["version"]
self.server_query_limit = self.server_info["query_limit"]
if self.max_tasks > self.server_query_limit:
self.max_tasks = self.server_query_limit
self.logger.warning(
"Max tasks was larger than server query limit of {}, reducing to match query limit.".format(
self.server_query_limit))
self.heartbeat_frequency = self.server_info["heartbeat_frequency"]
# Tell the server we are up and running
payload = self._payload_template()
payload["data"]["operation"] = "startup"
self.client._automodel_request("queue_manager", "put", payload)
if self.verbose:
self.logger.info(" Connected:")
self.logger.info(" Version: {}".format(self.server_version))
self.logger.info(" Address: {}".format(self.client.address))
self.logger.info(" Name: {}".format(self.server_name))
self.logger.info(" Queue tag: {}".format(self.queue_tag))
self.logger.info(" Username: {}\n".format(self.client.username))
else:
self.logger.info(" QCFractal server information:")
self.logger.info(" Not connected, some actions will not be available")
def __init__(self,
client: Any,
queue_client: Any,
logger: Optional[logging.Logger] = None,
max_tasks: int = 200,
queue_tag: str = None,
manager_name: str = "unlabled",
update_frequency: Union[int, float] = 2,
verbose: bool = True,
cores_per_task: Optional[int] = None,
memory_per_task: Optional[Union[int, float]] = None,
scratch_directory: Optional[str] = None):
"""
Parameters
----------
client : FractalClient
A FractalClient connected to a server
queue_client : QueueAdapter
The DBAdapter class for queue abstraction
storage_socket : DBSocket
A socket for the backend database
logger : logging.Logger, Optional. Default: None
A logger for the QueueManager
max_tasks : int
The maximum number of tasks to hold at any given time
queue_tag : str
Allows managers to pull from specific tags
manager_name : str
The cluster the manager belongs to
update_frequency : int
The frequency to check for new tasks in seconds
cores_per_task : int, optional, Default: None
How many CPU cores per computation task to allocate for QCEngine
None indicates "use however many you can detect"
memory_per_task: int, optional, Default: None
How much memory, in GiB, per computation task to allocate for QCEngine
None indicates "use however much you can consume"
scratch_directory: str, optional, Default: None
Scratch directory location to do QCEngine compute
None indicates "wherever the system default is"
"""
# Setup logging
if logger:
self.logger = logger
else:
self.logger = logging.getLogger('QueueManager')
self.name_data = {
"cluster": manager_name,
"hostname": socket.gethostname(),
"uuid": str(uuid.uuid4())
}
self._name = self.name_data["cluster"] + "-" + self.name_data[
"hostname"] + "-" + self.name_data["uuid"]
self.client = client
self.cores_per_task = cores_per_task
self.memory_per_task = memory_per_task
self.scratch_directory = scratch_directory
self.queue_adapter = build_queue_adapter(
queue_client,
logger=self.logger,
cores_per_task=self.cores_per_task,
memory_per_task=self.memory_per_task,
scratch_directory=self.scratch_directory)
self.max_tasks = max_tasks
self.queue_tag = queue_tag
self.verbose = verbose
self.scheduler = None
self.update_frequency = update_frequency
self.periodic = {}
self.active = 0
self.exit_callbacks = []
# QCEngine data
self.available_programs = qcng.list_available_programs()
self.available_procedures = qcng.list_available_procedures()
self.logger.info("QueueManager:")
self.logger.info(" Version: {}\n".format(
get_information("version")))
if self.verbose:
self.logger.info(" Name Information:")
self.logger.info(" Cluster: {}".format(
self.name_data["cluster"]))
self.logger.info(" Hostname: {}".format(
self.name_data["hostname"]))
self.logger.info(" UUID: {}\n".format(
self.name_data["uuid"]))
self.logger.info(" Queue Adapter:")
self.logger.info(" {}\n".format(self.queue_adapter))
if self.verbose:
self.logger.info(" QCEngine:")
self.logger.info(" Version: {}".format(
qcng.__version__))
self.logger.info(" Task Cores: {}".format(
self.cores_per_task))
self.logger.info(" Task Mem: {}".format(
self.memory_per_task))
self.logger.info(" Scratch Dir: {}".format(
self.scratch_directory))
self.logger.info(" Programs: {}".format(
self.available_programs))
self.logger.info(" Procedures: {}\n".format(
self.available_procedures))
# DGAS Note: Note super happy about how this if/else turned out. Looking for alternatives.
if self.connected():
# Pull server info
self.server_info = client.server_information()
self.server_name = self.server_info["name"]
self.server_version = self.server_info["version"]
self.server_query_limit = self.server_info["query_limit"]
if self.max_tasks > self.server_query_limit:
self.max_tasks = self.server_query_limit
self.logger.warning(
"Max tasks was larger than server query limit of {}, reducing to match query limit."
.format(self.server_query_limit))
self.heartbeat_frequency = self.server_info["heartbeat_frequency"]
# Tell the server we are up and running
payload = self._payload_template()
payload["data"]["operation"] = "startup"
response = self.client._automodel_request("queue_manager", "put",
payload)
if self.verbose:
self.logger.info(" Connected:")
self.logger.info(" Version: {}".format(
self.server_version))
self.logger.info(" Address: {}".format(
self.client.address))
self.logger.info(" Name: {}".format(
self.server_name))
self.logger.info(" Queue tag: {}".format(
self.queue_tag))
self.logger.info(" Username: {}\n".format(
self.client.username))
else:
self.logger.info(" QCFractal server information:")
self.logger.info(
" Not connected, some actions will not be available")
