def test_procedure_optimization(fractal_compute_server):
# Add a hydrogen molecule
hydrogen = ptl.Molecule.from_data([[1, 0, 0, -0.672], [1, 0, 0, 0.672]],
dtype="numpy",
units="bohr")
client = ptl.FractalClient(fractal_compute_server.get_address(""))
mol_ret = client.add_molecules([hydrogen])
kw = ptl.models.KeywordSet(
values={"scf_properties": ["quadrupole", "wiberg_lowdin_indices"]})
kw_id = client.add_keywords([kw])[0]
# Add compute
options = {
"keywords": None,
"qc_spec": {
"driver": "gradient",
"method": "HF",
"basis": "sto-3g",
"keywords": kw_id,
"program": "psi4"
},
}
# Ask the server to compute a new computation
r = client.add_procedure("optimization", "geometric", options, mol_ret)
assert len(r.ids) == 1
compute_key = r.ids[0]
# Manually handle the compute
fractal_compute_server.await_results()
assert len(fractal_compute_server.list_current_tasks()) == 0
# # Query result and check against out manual pul
results1 = client.query_procedures(procedure="optimization",
program="geometric")
results2 = client.query_procedures(id=compute_key)
for results in [results1, results2]:
assert len(results) == 1
assert isinstance(results[0].provenance.creator, str)
assert isinstance(str(results[0]), str) # Check that repr runs
assert pytest.approx(-1.117530188962681,
1e-5) == results[0].get_final_energy()
# Check pulls
traj = results[0].get_trajectory()
assert len(traj) == len(results[0].energies)
assert results[0].get_final_molecule().symbols == ["H", "H"]
# Check individual elements
for ind in range(len(results[0].trajectory)):
# Check keywords went through
assert traj[ind].provenance.creator.lower() == "psi4"
assert "SCF QUADRUPOLE XY" in traj[ind].extras["qcvars"]
assert "WIBERG_LOWDIN_INDICES" in traj[ind].extras["qcvars"]
# Make sure extra was popped
assert "_qcfractal_tags" not in traj[ind].extras
raw_energy = traj[ind].properties.return_energy
assert pytest.approx(raw_energy, 1.e-5) == results[0].energies[ind]
# Check result stdout
assert "RHF Reference" in traj[0].get_stdout()
# Check stdout
assert "internal coordinates" in results[0].get_stdout()
# Check that duplicates are caught
r = client.add_procedure("optimization", "geometric", options,
[mol_ret[0]])
assert len(r.ids) == 1
assert len(r.existing) == 1
tree.to_pickle(db=True, name=name)
print("{:12.1f} MB".format(os.path.getsize(name) / 1024**2))
if False:
# TODO: needs work still as ui can't be imported
# Load a list of datasets and build the index
# "datasets" is a file of datasets to load
datasets = offsb.ui.qcasb.load_dataset_input("datasets")
QCA = offsb.ui.qcasb.QCArchiveSpellBook(datasets=datasets).QCA
else:
# Just a quick shortcut to get something going
client = ptl.FractalClient()
ds = client.get_collection("TorsionDriveDataset",
"OpenFF Gen 2 Torsion Set 1 Roche")
QCA = QCA = qca.QCATree("QCA",
root_payload=client,
node_index=dict(),
db=dict())
drop = ["Intermediates", "Hessian"]
QCA.build_index(ds, drop=drop, keep_specs=["default"])
save(QCA)
entries = list(QCA.node_iter_depth_first(QCA.root(), select="Entry"))
print("There are {:d} entries total".format(len(entries)))
# The general query on the dataset.
def test_compute_queue_stack(fractal_compute_server):
# Build a client
client = ptl.FractalClient(fractal_compute_server)
# Add a hydrogen and helium molecule
hydrogen = ptl.Molecule.from_data([[1, 0, 0, -0.5], [1, 0, 0, 0.5]],
dtype="numpy",
units="bohr")
helium = ptl.Molecule.from_data([[2, 0, 0, 0.0]],
dtype="numpy",
units="bohr")
hydrogen_mol_id, helium_mol_id = client.add_molecules([hydrogen, helium])
kw = ptl.models.KeywordSet(**{"values": {"e_convergence": 1.e-8}})
kw_id = client.add_keywords([kw])[0]
# Add compute
compute = {
"meta": {
"procedure": "single",
"driver": "energy",
"method": "HF",
"basis": "sto-3g",
"keywords": kw_id,
"program": "psi4",
},
"data": [hydrogen_mol_id, helium.json_dict()],
}
# Ask the server to compute a new computation
r = client.add_compute("psi4", "HF", "sto-3g", "energy", kw_id,
[hydrogen_mol_id, helium])
assert len(r.ids) == 2
# Manually handle the compute
fractal_compute_server.await_results()
assert len(fractal_compute_server.list_current_tasks()) == 0
# Query result and check against out manual pul
results_query = {
"program": "psi4",
"molecule": [hydrogen_mol_id, helium_mol_id],
"method": compute["meta"]["method"],
"basis": compute["meta"]["basis"]
}
results = client.query_results(**results_query, status=None)
assert len(results) == 2
for r in results:
assert r.provenance.creator.lower() == "psi4"
if r.molecule == hydrogen_mol_id:
assert pytest.approx(-1.0660263371078127,
1e-5) == r.properties.scf_total_energy
elif r.molecule == helium_mol_id:
assert pytest.approx(-2.807913354492941,
1e-5) == r.properties.scf_total_energy
else:
raise KeyError("Returned unexpected Molecule ID.")
assert "RHF Reference" in results[0].get_stdout()
import qcfractal.interface as portal
# Build a interface to the server
p = portal.FractalClient("localhost:7777")
# Pull data from the server
db = portal.collections.Database("Water", p)
# Submit computations
r = db.query("SCF", "STO-3G", stoich="cp", scale="kcal")
print(db.df)
def test_security_auth_accept(sec_server):
client = ptl.FractalClient(sec_server, username="******", password=_users["write"]["pw"])
r = client.add_molecules([])
r = client.query_molecules(id=[])
def test_torsiondrive_dataset(fractal_compute_server):
client = ptl.FractalClient(fractal_compute_server)
ds = ptl.collections.TorsionDriveDataset("testing", client=client)
hooh1 = ptl.data.get_molecule("hooh.json")
hooh2 = hooh1.copy(
update={"geometry": hooh1.geometry + np.array([0, 0, 0.2])})
ds.add_entry("hooh1", [hooh1], [[0, 1, 2, 3]], [90],
attributes={"something": "hooh1"})
ds.add_entry("hooh2", [hooh2], [[0, 1, 2, 3]], [90],
attributes={"something": "hooh2"})
optimization_spec = {
"program": "geometric",
"keywords": {
"coordsys": "tric",
}
}
qc_spec = {
"driver": "gradient",
"method": "UFF",
"basis": "",
"keywords": None,
"program": "rdkit",
}
ds.add_specification("Spec1",
optimization_spec,
qc_spec,
description="This is a really cool spec")
ncompute = ds.compute("spec1")
assert ncompute == 2
assert ds.status(
"spec1")["Spec1"].sum() == 2 # Might have completed from previous run.
ds.save()
fractal_compute_server.await_services(max_iter=1)
# Check status
status_detail = ds.status("Spec1", detail=True)
assert status_detail.loc["hooh2", "Complete"] == 1
assert status_detail.loc["hooh2", "Total Points"] == 4
fractal_compute_server.await_services(max_iter=5)
ds = client.get_collection("torsiondrivedataset", "testing")
ds.query("spec1")
# Add another fake set, should instantly return
ds.add_specification("spec2",
optimization_spec,
qc_spec,
description="This is a really cool spec")
# Test subsets
ncompute = ds.compute("spec2", subset=set())
assert ncompute == 0
ncompute = ds.compute("spec2")
assert ncompute == 2
ds.query("spec2")
# We effectively computed the same thing twice with two duplicate specs
for row in ["hooh1", "hooh2"]:
for spec in ["Spec1", "spec2"]:
assert pytest.approx(ds.df.loc[row, spec].get_final_energies(90),
1.e-5) == 0.00015655375994799847
assert ds.status().loc["COMPLETE", "Spec1"] == 2
assert ds.status(collapse=False).loc["hooh1", "Spec1"] == "COMPLETE"
assert ds.counts("hooh1").loc["hooh1", "Spec1"] > 5
assert ds.counts("hooh1", specs="spec1",
count_gradients=True).loc["hooh1", "Spec1"] > 30
def test_compute_database(fractal_compute_server):
client = portal.FractalClient(fractal_compute_server.get_address(""))
db_name = "He_PES"
db = portal.collections.Database(db_name, client, db_type="ie")
# Adds options
option = portal.data.get_options("psi_default")
opt_ret = client.add_options([option])
opt_key = option["name"]
# Add two helium dimers to the DB at 4 and 8 bohr
He1 = portal.Molecule([[2, 0, 0, -2], [2, 0, 0, 2]],
dtype="numpy",
units="bohr",
frags=[1])
db.add_ie_rxn("He1",
He1,
attributes={"r": 4},
reaction_results={"default": {
"Benchmark": 0.0009608501557
}})
# Save the DB and re-acquire
r = db.save()
db = portal.collections.Database.from_server(client, db_name)
He2 = portal.Molecule([[2, 0, 0, -4], [2, 0, 0, 4]],
dtype="numpy",
units="bohr",
frags=[1])
db.add_ie_rxn(
"He2",
He2,
attributes={"r": 4},
reaction_results={"default": {
"Benchmark": -0.00001098794749
}})
# Save the DB and overwrite the result
r = db.save(overwrite=True)
# Open a new database
db = portal.collections.Database.from_server(client, db_name)
# Compute SCF/sto-3g
ret = db.compute("SCF", "STO-3G")
fractal_compute_server.objects["queue_nanny"].await_results()
# Query computed results
assert db.query("SCF", "STO-3G")
assert pytest.approx(0.6024530476071095, 1.e-5) == db.df.loc["He1",
"SCF/STO-3G"]
assert pytest.approx(-0.006895035942673289,
1.e-5) == db.df.loc["He2", "SCF/STO-3G"]
# Check results
assert db.query("Benchmark", "", reaction_results=True)
assert pytest.approx(0.00024477933196125805,
1.e-5) == db.statistics("MUE", "SCF/STO-3G")
def test_compute_dataset(fractal_compute_server):
client = portal.FractalClient(fractal_compute_server.get_address(""))
ds_name = "He_PES"
ds = portal.collections.Dataset(ds_name, client, ds_type="ie")
# Adds options
option = portal.data.get_options("psi_default")
opt_ret = client.add_options([option])
opt_key = option["name"]
# Add two helium dimers to the DB at 4 and 8 bohr
He1 = portal.Molecule([[2, 0, 0, -2], [2, 0, 0, 2]],
dtype="numpy",
units="bohr",
frags=[1])
ds.add_ie_rxn("He1",
He1,
attributes={"r": 4},
reaction_results={"default": {
"Benchmark": 0.0009608501557
}})
# Save the DB and re-acquire via classmethod
r = ds.save()
ds = portal.collections.Dataset.from_server(client, ds_name)
assert "Dataset(" in str(ds)
# Test collection lists
ret = client.list_collections()
assert ret == {"dataset": [ds_name]}
ret = client.list_collections("dataset")
assert ret == [ds_name]
He2 = portal.Molecule([[2, 0, 0, -4], [2, 0, 0, 4]],
dtype="numpy",
units="bohr",
frags=[1])
ds.add_ie_rxn(
"He2",
He2,
attributes={"r": 4},
reaction_results={"default": {
"Benchmark": -0.00001098794749
}})
# Save the DB and overwrite the result, reacquire via client
r = ds.save(overwrite=True)
ds = client.get_collection("dataset", ds_name)
# Compute SCF/sto-3g
ret = ds.compute("SCF", "STO-3G")
assert len(ret["submitted"]) == 3
fractal_compute_server.await_results()
# Query computed results
assert ds.query("SCF", "STO-3G")
assert pytest.approx(0.6024530476071095, 1.e-5) == ds.df.loc["He1",
"SCF/STO-3G"]
assert pytest.approx(-0.006895035942673289,
1.e-5) == ds.df.loc["He2", "SCF/STO-3G"]
# Check results
assert ds.query("Benchmark", "", reaction_results=True)
assert pytest.approx(0.00024477933196125805,
1.e-5) == ds.statistics("MUE", "SCF/STO-3G")
assert isinstance(ds.to_json(), dict)
def test_task_regenerate(fractal_compute_server):
client = ptl.FractalClient(fractal_compute_server)
# Add a single computation and a geometry optimization
# Both of these have invalid methods
mol = ptl.models.Molecule(**{"geometry": [1, 2, 3], "symbols": ["Ne"]})
geometric_options = {
"keywords": None,
"qc_spec": {
"driver": "gradient",
"method": "cookiemonster",
"basis": "",
"keywords": None,
"program": "rdkit"
},
}
ret1 = client.add_compute("rdkit", "cookiemonster", "", "energy", None,
[mol])
ret2 = client.add_procedure("optimization", "geometric", geometric_options,
[mol])
fractal_compute_server.await_results()
base_ids = [ret1.submitted[0], ret2.submitted[0]]
old_tasks = client.query_tasks(base_result=base_ids)
# Regenerate, but old one exists. Should be a no-op
upd = client.modify_tasks("regenerate", base_result=base_ids)
assert upd.n_updated == 0
new_tasks = client.query_tasks(base_result=base_ids)
for old_task, new_task in zip(old_tasks, new_tasks):
assert old_task.status == "ERROR"
assert old_task.id == new_task.id
assert old_task.base_result == new_task.base_result
assert old_task.modified_on == new_task.modified_on
assert old_task.created_on == new_task.created_on
# Manually delete the old task
db = fractal_compute_server.objects["storage_socket"]
db.del_tasks([x.id for x in old_tasks])
# Actually deleted?
del_task = client.query_tasks(base_result=base_ids)
assert len(del_task) == 0
# Now regenerate
upd = client.modify_tasks("regenerate", base_result=base_ids)
new_tasks = client.query_tasks(base_result=base_ids)
assert upd.n_updated == 2
for old_task, new_task in zip(old_tasks, new_tasks):
assert new_task.status == "WAITING"
assert old_task.id != new_task.id # Task ids must now be different
assert old_task.base_result == new_task.base_result
assert old_task.modified_on < new_task.modified_on # New task must be newer
assert old_task.created_on < new_task.created_on # New task must be newer
assert old_tasks[0].spec.args[0]["molecule"]["id"] == new_tasks[
0].spec.args[0]["molecule"]["id"]
assert (old_tasks[0].spec.args[0]["molecule"]["identifiers"]
["molecule_hash"] == new_tasks[0].spec.args[0]["molecule"]
["identifiers"]["molecule_hash"])
assert old_tasks[1].spec.args[0]["initial_molecule"]["id"] == new_tasks[
1].spec.args[0]["initial_molecule"]["id"]
assert (old_tasks[1].spec.args[0]["initial_molecule"]["identifiers"]
["molecule_hash"] == new_tasks[1].spec.args[0]["initial_molecule"]
["identifiers"]["molecule_hash"])
# The status of the result should be reset to incomplete
res = client.query_procedures(base_ids)
assert all(x.status == "INCOMPLETE" for x in res)
def generate_status_report( collection, name, logname):
"""
Iterates a dataset, checking for errors, and saves errors to the log file.
"""
client = ptl.FractalClient()
QCA = qca.QCATree( "QCA", root_payload=client, node_index=dict(), db=dict())
for s in sets:
ds = client.get_collection( collection, name)
QCA.build_index( ds, drop=["Gradient"], keep_specs=["default"],
start=0, limit=0)
default = [n for n in QCA.node_iter_depth_first(QCA.root())
if "QCS-default" in n.payload]
mindepth = QCA.node_depth(default[0])
tdfailures= 0
fid = open(logname, 'w')
for node in QCA.node_iter_dive(default):
i = QCA.node_depth(node) - mindepth+1
status=""
statbar= " "
errmsg = ""
hasstat = False
qcp = QCA.db[node.payload]['data']
try:
hasstat = "status" in QCA.db[node.payload]["data"]
except Exception:
pass
if hasstat:
status = qcp["status"][:]
if status != "COMPLETE" and node.name == "Optimization":
try:
statbar = "----"
client = qcp['client']
print("error", qcp['error'], qcp['error'] is None)
if not (qcp['error'] is None):
key=int(qcp['error'])
err = list(client.query_kvstore(key).values())[0]
errmsg += "####ERROR####\n"
errmsg += "Error type: " + err['error_type'] + "\n"
errmsg += "Error:\n" + err['error_message']
errmsg += "############\n"
elif status == "ERROR":
errmsg += "####FATAL####\n"
errmsg += "Job errored and no error on record. "
errmsg += "Input likely bogus; check your settings\n"
errmsg += "############\n"
if not (qcp['stdout'] is None):
key=int(qcp['stdout'])
msg = list(client.query_kvstore(key)).values())[0]
errmsg += "----ISSUE----\n"
errmsg += "Status was not complete; stdout is:\n"
errmsg += msg
errmsg += "-------------\n"
if not (qcp['stderr'] is None):
key=int(qcp['stderr'])
msg = list(client.query_kvstore(key)).values())[0]
errmsg += "####ISSUE####\n"
errmsg += "Status was not complete; stderr is:\n"
errmsg += msg
errmsg += "-------------\n"
except Exception as e:
fid.write("Internal issue:\n" + str(e) + '\n')
if status != "COMPLETE" and node.name == "TorsionDrive":
statbar = "XXXX"
if node.name == "TorsionDrive" and status != "COMPLETE":
tdfailures += 1
fid.write("{:2d} {} {} {}\n".format(tdi,statbar*i,
" ".join([str(node.index),str(node.name), str(node.payload)]),
status))
if errmsg != "":
fid.write("\n{}\n".format(errmsg))
def submit_qca_optimization_dataset(
dataset_name=None,
metadata=None,
compute_spec=None,
input_molecules=None,
server="from_file",
threads=None,
compute_tag=None,
priority="normal",
skip_compute=False,
):
"""
Create or update an optimization dataset.
Parameters
----------
dataset_name : str
The name of the dataset. This is needed if the dataset already exists and no
metadata is supplied. Useful when e.g. adding computes or molecules to an existing dataset.
metadata : str
A filename specifying the metadata needed to create a new dataset, in JSON format.
An example metadata has the following format:
{
"submitter": "trevorgokey",
"creation_date": "2020-09-18",
"collection_type": "OptimizationDataset",
"dataset_name": "OpenFF Sandbox CHO PhAlkEthOH v1.0",
"short_description": "A diverse set of CHO molecules",
"long_description_url": "https://github.com/openforcefield/qca-dataset-submission/tree/master/submissions/2020-09-18-OpenFF-Sandbox-CHO-PhAlkEthOH",
"long_description": "This dataset contains an expanded set of the AlkEthOH and
PhEthOH datasets, which were used in the original derivation of the smirnoff99Frosst
parameters.",
"elements": [
"C",
"H",
"O"
],
"change_log": [
{"author": "trevorgokey",
"date": "2020-09-18",
"version": "1.0",
"description": "A diverse set of CHO molecules. The molecules in this set were
generated to include all stereoisomers if chirality was ambiguous from the SMILES
input. Conformations were generated which had an RMSD of at least 4 Angstroms from
all other conformers"
}
]
}
compute_spec : str
A filename specifying the compute specifications for the dataset, in JSON format.
input_molecules : str
A filename specifying the molecules to load into the dataset as entries, in JSON format.
server : str
The server URI to connect to. The special value 'from_file' will read from the default
server connection config file for e.g. authentication
threads : int
The number of threads to use to when contacting the server.
compute_tag : str
The compute tag used to match computations with compute managers. For OpenFF calculations,
this should be "openff"
priorty : str
The priority of new calculations to submit. This must be either "low", "normal", or "high".
skip_compute : bool
Do not submit the tasks after the molecules and compute specifications have been added
Returns
-------
None
"""
ds_type = "OptimizationDataset"
ds_name = dataset_name
if server == "from_file":
# Connect to a server that needs authentication
client = ptl.FractalClient().from_file()
elif server is not None:
# Use a custom server, possibly a local, private server
client = ptl.FractalClient(server, verify=False)
else:
# Use the default public MOLSSI server
client = ptl.FractalClient()
try:
ds = client.get_collection(ds_type, ds_name)
logger.info("\nDataset loaded with the following metadata:")
logger.info(pformat(ds.data.metadata))
except KeyError:
assert metadata is not None
metadata = json.load(open(metadata))
metadata["collection_type"] = ds_type
if ds_name is not None:
metadata["dataset_name"] = ds_name
else:
ds_name = metadata["dataset_name"]
ds = getattr(ptl.collections, ds_type)(
ds_name,
client=client,
metadata=metadata,
description=metadata["short_description"],
tags=["openff"],
tagline=metadata["short_description"],
)
logger.info("\nDataset created with the following metadata:")
logger.info(pformat(metadata))
if compute_spec is not None:
specs = json.load(open(compute_spec))
add_compute_specs(ds, specs)
if input_molecules is not None:
pool = ProcessPoolExecutor(max_workers=threads)
new_mols = 0
new_calcs = 0
total_calcs = 0
logger.info("\nLoading {} into QCArchive...".format(input_molecules))
if input_molecules.endswith("lzma") or input_molecules.endswith("xz"):
input_ds = json.load(lzma.open(input_molecules, "rt"))
elif input_molecules.endswith("bz2"):
input_ds = json.load(bz2.open(input_molecules, "rt"))
else:
input_ds = json.load(open(input_molecules))
logger.info("Number of unique molecules: {}".format(len(input_ds)))
work = []
for j, index in enumerate(input_ds):
for i, mol in enumerate(input_ds[index], 1):
work_unit = pool.submit(submit, *(ds, index, mol, i))
work.append(work_unit)
ds.save()
ids = []
new_entries = 0
iterable = enumerate(as_completed(work))
if logger.getEffectiveLevel() >= logging.INFO:
iterable = tqdm.tqdm(iterable,
total=len(work),
ncols=80,
desc="Entries")
for j, unit in iterable:
unique_id, success = unit.result()
new_entries += int(success)
ids.append(unique_id)
new_mols += len(input_ds)
new_calcs += new_entries
total_calcs += len(ids)
logger.info("\nNumber of new entries: {}/{}".format(
new_entries, len(ids)))
stride = 20
# Only submit tasks that were explicitly given as parameters
if compute_spec is not None and not skip_compute:
new_tasks = 0
for qc_spec_name in specs:
out_str = (
"\nSubmitting calculations in batches of {} for specification {}"
)
logger.info(out_str.format(stride, qc_spec_name))
work = []
args = (qc_spec_name, )
kwargs = dict(priority=priority, tag=compute_tag)
for entry_list in chunk(ids, stride):
kwargs["subset"] = entry_list
work_unit = pool.submit(ds.compute, *args, **kwargs)
work.append(work_unit)
iterable = as_completed(work)
if logger.getEffectiveLevel() >= logging.INFO:
iterable = tqdm.tqdm(iterable,
total=len(work),
ncols=80,
desc="Tasks")
for unit in iterable:
submitted = unit.result()
new_tasks += submitted
logger.info("\nNumber of new tasks: {}".format(new_tasks))
pool.shutdown(wait=True)
def test_security_auth_decline_none(sec_server):
with pytest.raises(IOError) as excinfo:
client = ptl.FractalClient(sec_server)
assert "user not found" in str(excinfo.value).lower()
def test_security_auth_allow_read_block_add(sec_server_allow_read):
client = ptl.FractalClient(sec_server_allow_read)
with pytest.raises(IOError) as excinfo:
client.add_molecules([ptl.Molecule.from_data("He 0 0 0")])
assert "user not found" in str(excinfo.value).lower()
def test_security_auth_allow_read(sec_server_allow_read):
client = ptl.FractalClient(sec_server_allow_read)
def test_compute_reactiondataset_regression(fractal_compute_server):
"""
Tests an entire server and interaction energy dataset run
"""
client = ptl.FractalClient(fractal_compute_server)
ds_name = "He_PES"
ds = ptl.collections.ReactionDataset(ds_name, client, ds_type="ie")
# Add two helium dimers to the DB at 4 and 8 bohr
He1 = ptl.Molecule.from_data([[2, 0, 0, -2], [2, 0, 0, 2]],
dtype="numpy",
units="bohr",
frags=[1])
ds.add_ie_rxn("He1", He1, attributes={"r": 4})
# Save the DB and re-acquire via classmethod
r = ds.save()
ds = ptl.collections.ReactionDataset.from_server(client, ds_name)
ds.set_default_program("psi4")
assert "ReactionDataset(" in str(ds)
# Test collection lists
ret = client.list_collections(aslist=True)
assert ds_name in ret["ReactionDataset"]
ret = client.list_collections("reactiondataset", aslist=True)
assert ds_name in ret
He2 = ptl.Molecule.from_data([[2, 0, 0, -4], [2, 0, 0, 4]],
dtype="numpy",
units="bohr",
frags=[1])
ds.add_ie_rxn("He2", He2, attributes={"r": 4})
contrib = {
"name": "Benchmark",
"doi": None,
"theory_level": "very high",
"values": {
"He1": 0.0009608501557,
"He2": -0.00001098794749
},
"units": "hartree"
}
ds.add_contributed_values(contrib)
ds.set_default_benchmark("Benchmark")
# Save the DB and overwrite the result, reacquire via client
r = ds.save()
ds = client.get_collection("reactiondataset", ds_name)
with pytest.raises(KeyError):
ret = ds.compute("SCF", "STO-3G",
stoich="nocp") # Should be 'default' not 'nocp'
# Compute SCF/sto-3g
ret = ds.compute("SCF", "STO-3G")
assert len(ret.submitted) == 3
fractal_compute_server.await_results()
# Query computed results
assert ds.query("SCF", "STO-3G")
assert pytest.approx(0.6024530476, 1.e-5) == ds.df.loc["He1", "SCF/sto-3g"]
assert pytest.approx(-0.0068950359, 1.e-5) == ds.df.loc["He2",
"SCF/sto-3g"]
# Check results
assert pytest.approx(0.00024477933196125805,
1.e-5) == ds.statistics("MUE", "SCF/sto-3g")
assert pytest.approx([0.081193, 7.9533e-05],
1.e-1) == list(ds.statistics("URE", "SCF/sto-3g"))
assert pytest.approx(0.0406367,
1.e-5) == ds.statistics("MURE", "SCF/sto-3g")
assert pytest.approx(0.002447793, 1.e-5) == ds.statistics("MURE",
"SCF/sto-3g",
floor=10)
assert isinstance(ds.to_json(), dict)
assert ds.list_history(keywords=None).shape[0] == 1
ds.units = "eV"
assert pytest.approx(0.00010614635, 1.e-5) == ds.statistics("MURE",
"SCF/sto-3g",
floor=10)
def contributed_dataset_fixture(fractal_compute_server, tmp_path_factory,
request):
""" Fixture for testing rich contributed datasets with many properties and molecules of different sizes"""
client = ptl.FractalClient(fractal_compute_server)
try:
ds = client.get_collection("Dataset", "ds_contributed")
except KeyError:
# Build a dataset
ds = ptl.collections.Dataset("ds_contributed", client)
ds.add_entry("He1", ptl.Molecule.from_data("He -1 0 0\n--\nHe 0 0 1"))
ds.add_entry("He", ptl.Molecule.from_data("He -1.1 0 0"))
ds.units = "hartree"
ds.save()
energy = {
"name": "Fake Energy",
"theory_level": "pseudo-random values",
"values": {
"He1": 1234.5,
"He": 5.4321
},
"theory_level_details": {
"driver": "energy",
"program": "fake_program",
"basis": "fake_basis",
"method": "fake_method",
},
"units": "hartree"
}
gradient = {
"name": "Fake Gradient",
"theory_level": "pseudo-random values",
"values": {
"He1": [0.03, 0, 0.02, -0.02, 0, -0.03],
"He": [0.03, 0, 0.02]
},
"theory_level_details": {
"driver": "gradient",
"program": "fake_program",
"basis": "fake_basis",
"method": "fake_method",
},
"units": "hartree/bohr"
}
hessian = {
"name": "Fake Hessian",
"theory_level": "pseudo-random values",
"values": {
"He1": list(np.eye(6).ravel()),
"He": [1, 0.2, 0.1, 0.2, 1, 0.4, 0.1, 0.4, 1]
},
"theory_level_details": {
"driver": "hessian",
"program": "fake_program",
"basis": "fake_basis",
"method": "fake_method",
},
"units": "hartree/bohr**2"
}
dipole = {
"name": "Fake Dipole",
"theory_level": "pseudo-random values",
"values": {
"He": [1., 2., 3.],
"He1": [1., -2., 0.]
},
"theory_level_details": {
"driver": "dipole",
"program": "fake_program",
"basis": "fake_basis",
"method": "fake_method",
},
"units": "e * bohr"
}
energy_FF = {
"name": "Fake FF Energy",
"theory_level": "some force field",
"values": {
"He1": 0.5,
"He": 42.
},
"theory_level_details": {
"driver": "energy",
"program": "fake_program",
"basis": None,
"method": "fake_method"
},
"units": "hartree"
}
ds.add_contributed_values(energy)
ds.add_contributed_values(gradient)
ds.add_contributed_values(hessian)
ds.add_contributed_values(dipole)
ds.add_contributed_values(energy_FF)
with pytest.raises(KeyError):
ds.add_contributed_values(energy)
ds.save()
build_dataset_fixture_view(ds, fractal_compute_server)
ds = handle_dataset_fixture_params(client, "Dataset", ds,
fractal_compute_server, request)
yield client, ds
def test_missing_collection(fractal_compute_server):
client = ptl.FractalClient(fractal_compute_server)
with pytest.raises(KeyError):
client.get_collection("reactiondataset", "_waffles_")
def gradient_dataset_fixture(fractal_compute_server, tmp_path_factory,
request):
client = ptl.FractalClient(fractal_compute_server)
try:
ds = client.get_collection("Dataset", "ds_gradient")
except KeyError:
testing.check_has_module("psi4")
# Build a dataset
ds = ptl.collections.Dataset("ds_gradient",
client,
default_program="psi4",
default_driver="gradient",
default_units="hartree/bohr")
ds.add_entry("He1", ptl.Molecule.from_data("He -1 0 0\n--\nHe 0 0 1"))
ds.add_entry("He2",
ptl.Molecule.from_data("He -1.1 0 0\n--\nHe 0 0 1.1"))
ds.save()
contrib = {
"name": "Gradient",
"theory_level": "pseudo-random values",
"values": {
"He1": [0.03, 0, 0.02, -0.02, 0, -0.03],
"He2": [0.03, 0, 0.02, -0.02, 0, -0.03]
},
"theory_level_details": {
"driver": "gradient"
},
"units": "hartree/bohr"
}
contrib_no_details = {
"name": "no details",
"theory_level": "pseudo-random values",
"values": {
"He1": [0.03, 0, 0.02, -0.02, 0, -0.03],
"He2": [0.03, 0, 0.02, -0.02, 0, -0.03]
},
"units": "hartree/bohr"
}
contrib_all_details = {
"name": "all details",
"theory_level": "pseudo-random values",
"values": {
"He1": [0.03, 0, 0.02, -0.02, 0, -0.03],
"He2": [0.03, 0, 0.02, -0.02, 0, -0.03]
},
"theory_level_details": {
"driver": "gradient",
"program": "fake_program",
"basis": "fake_basis",
"method": "fake_method",
"keywords": "fake_keywords"
},
"units": "hartree/bohr"
}
ds.add_contributed_values(contrib)
ds.add_contributed_values(contrib_no_details)
ds.add_contributed_values(contrib_all_details)
ds.add_keywords("scf_default",
"psi4",
ptl.models.KeywordSet(values={}),
default=True)
ds.save()
ds.compute("HF", "sto-3g")
fractal_compute_server.await_results()
assert ds.get_records("HF", "sto-3g").iloc[0, 0].status == "COMPLETE"
assert ds.get_records("HF", "sto-3g").iloc[1, 0].status == "COMPLETE"
build_dataset_fixture_view(ds, fractal_compute_server)
ds = handle_dataset_fixture_params(client, "Dataset", ds,
fractal_compute_server, request)
yield client, ds
def test_server_information(test_server):
client = ptl.FractalClient(test_server)
server_info = client.server_information()
assert {"name", "heartbeat_frequency"} <= server_info.keys()
def test_compute_openffworkflow(fractal_compute_server):
"""
Tests the openffworkflow collection
"""
# Obtain a client and build a BioFragment
client = ptl.FractalClient(fractal_compute_server)
openff_workflow_options = {
# Blank Fragmenter options
"enumerate_states": {},
"enumerate_fragments": {},
"torsiondrive_input": {},
# TorsionDriveRecord, Geometric, and QC options
"torsiondrive_static_options": {
"keywords": {},
"optimization_spec": {
"program": "geometric",
"keywords": {
"coordsys": "tric",
}
},
"qc_spec": {
"driver": "gradient",
"method": "UFF",
"basis": "",
"keywords": None,
"program": "rdkit",
}
},
"optimization_static_options": {
"program": "geometric",
"keywords": {
"coordsys": "tric",
},
"qc_spec": {
"driver": "gradient",
"method": "UFF",
"basis": "",
"keywords": None,
"program": "rdkit",
}
}
}
wf = ptl.collections.OpenFFWorkflow("Workflow1",
client=client,
**openff_workflow_options)
# # Add a fragment and wait for the compute
hooh = ptl.data.get_molecule("hooh.json")
fragment_input = {
"label1": {
"type": "torsiondrive_input",
"initial_molecule": hooh.json_dict(),
"grid_spacing": [90],
"dihedrals": [[0, 1, 2, 3]],
},
}
wf.add_fragment("HOOH", fragment_input)
assert set(wf.list_fragments()) == {"HOOH"}
fractal_compute_server.await_services(max_iter=5)
final_energies = wf.list_final_energies()
assert final_energies.keys() == {"HOOH"}
assert final_energies["HOOH"].keys() == {"label1"}
final_molecules = wf.list_final_molecules()
assert final_molecules.keys() == {"HOOH"}
assert final_molecules["HOOH"].keys() == {"label1"}
optimization_input = {
"label2": {
"type": "optimization_input",
"initial_molecule": hooh.json_dict(),
"constraints": {
'set': [{
"type": 'dihedral',
"indices": [0, 1, 2, 3],
"value": 0
}]
}
}
}
wf.add_fragment("HOOH", optimization_input)
fractal_compute_server.await_services(max_iter=5)
final_energies = wf.list_final_energies()
assert final_energies["HOOH"].keys() == {"label1", "label2"}
assert pytest.approx(0.00259754, 1.e-4) == final_energies["HOOH"]["label2"]
final_molecules = wf.list_final_molecules()
assert final_molecules.keys() == {"HOOH"}
assert final_molecules["HOOH"].keys() == {"label1", "label2"}
# Add a second fragment
butane = ptl.data.get_molecule("butane.json")
butane_id = butane.identifiers.canonical_isomeric_explicit_hydrogen_mapped_smiles
fragment_input = {
"label1": {
"type": "torsiondrive_input",
"initial_molecule": butane.json_dict(),
"grid_spacing": [90],
"dihedrals": [[0, 2, 3, 1]],
},
}
wf.add_fragment(butane_id, fragment_input)
assert set(wf.list_fragments()) == {butane_id, "HOOH"}
final_energies = wf.list_final_energies()
assert final_energies.keys() == {butane_id, "HOOH"}
assert final_energies[butane_id].keys() == {"label1"}
assert final_energies[butane_id]["label1"] == {}
def test_procedure_optimization(fractal_compute_server):
# Add a hydrogen molecule
hydrogen = portal.Molecule([[1, 0, 0, -0.672], [1, 0, 0, 0.672]],
dtype="numpy",
units="bohr")
client = portal.FractalClient(fractal_compute_server.get_address(""))
mol_ret = client.add_molecules({"hydrogen": hydrogen.to_json()})
# Add compute
compute = {
"meta": {
"procedure": "optimization",
"program": "geometric",
"options": "none",
"qc_meta": {
"driver": "gradient",
"method": "HF",
"basis": "sto-3g",
"options": "none",
"program": "psi4"
},
},
"data": [mol_ret["hydrogen"]],
}
# Ask the server to compute a new computation
r = requests.post(fractal_compute_server.get_address("task_scheduler"),
json=compute)
assert r.status_code == 200
# Get the first submitted job, the second index will be a hash_index
submitted = r.json()["data"]["submitted"]
compute_key = submitted[0]
# Manually handle the compute
nanny = fractal_compute_server.objects["queue_nanny"]
nanny.await_results()
assert len(nanny.list_current_tasks()) == 0
# # Query result and check against out manual pul
results1 = client.get_procedures({"program": "geometric"})
results2 = client.get_procedures({"queue_id": compute_key})
for results in [results1, results2]:
assert len(results) == 1
assert isinstance(str(results[0]), str) # Check that repr runs
assert pytest.approx(-1.117530188962681,
1e-5) == results[0].final_energy()
# Check pulls
traj = results[0].get_trajectory(projection={"properties": True})
energies = results[0].energies()
assert len(traj) == len(energies)
assert results[0].final_molecule()["symbols"] == ["H", "H"]
# Check individual elements
for ind in range(len(results[0]._trajectory)):
raw_energy = traj[ind]["properties"]["return_energy"]
assert pytest.approx(raw_energy, 1.e-5) == energies[ind]
# Check that duplicates are caught
r = requests.post(fractal_compute_server.get_address("task_scheduler"),
json=compute)
assert r.status_code == 200
assert len(r.json()["data"]["completed"]) == 1
def test_service_gridoptimization_single_opt(fractal_compute_server):
client = ptl.FractalClient(fractal_compute_server)
# Add a HOOH
hooh = ptl.data.get_molecule("hooh.json")
initial_distance = hooh.measure([1, 2])
mol_ret = client.add_molecules([hooh])
# Options
service = GridOptimizationInput(**{
"keywords": {
"preoptimization":
True,
"scans": [{
"type": "distance",
"indices": [1, 2],
"steps": [-0.1, 0.0],
"step_type": "relative"
}, {
"type": "dihedral",
"indices": [0, 1, 2, 3],
"steps": [-90, 0],
"step_type": "absolute"
}]
},
"optimization_spec": {
"program": "geometric",
"keywords": {
"coordsys": "tric",
}
},
"qc_spec": {
"driver": "gradient",
"method": "UFF",
"basis": "",
"keywords": None,
"program": "rdkit",
},
"initial_molecule": mol_ret[0],
}) # yapf: disable
ret = client.add_service([service], tag="gridopt", priority="low")
fractal_compute_server.await_services()
assert len(fractal_compute_server.list_current_tasks()) == 0
result = client.query_procedures(id=ret.ids)[0]
assert result.status == "COMPLETE"
assert result.starting_grid == (1, 0)
assert pytest.approx(result.get_final_energies((0, 0)),
abs=1.e-4) == 0.0010044105443485617
assert pytest.approx(result.get_final_energies((1, 1)),
abs=1.e-4) == 0.0026440964897817623
assert result.starting_molecule != result.initial_molecule
# Check initial vs starting molecule
assert result.initial_molecule == mol_ret[0]
starting_mol = client.query_molecules(id=result.starting_molecule)[0]
assert pytest.approx(starting_mol.measure([1, 2])) != initial_distance
assert pytest.approx(starting_mol.measure([1, 2])) == 2.488686479260597
# Check tags on individual procedures
proc_id = result.grid_optimizations['[0, 0]']
# completed tasks should be deleted
task = client.query_tasks(base_result=proc_id)
assert not task
# Check final ResultRecords
final_result_records = result.get_final_results()
assert len(final_result_records) == 4
import qcfractal.interface as portal
# Build a interface to the server
p = portal.FractalClient("localhost:7777", verify=False)
# Pull data from the server
ds = portal.collections.ReactionDataset.from_server(p, "Water")
# Submit computations
r = ds.query("SCF", "sto-3g", stoich="cp", program="psi4")
# Print the Pandas DataFrame
print(ds.df)
# Tests to ensure the correct results are returned
# Safe to comment out
import pytest
assert pytest.approx(ds.df.loc["Water Dimer", "cp-SCF/sto-3g-Psi4"],
1.e-3) == -1.392710
assert pytest.approx(ds.df.loc["Water Dimer Stretch", "cp-SCF/sto-3g-Psi4"],
1.e-3) == 0.037144
assert pytest.approx(ds.df.loc["Helium Dimer", "cp-SCF/sto-3g-Psi4"],
1.e-3) == -0.003148
"""
This script skips the fragmentation step to find torsions and generate multiple conformations of molecules
"""
from fragmenter import workflow_api, chemi
from qcfractal import interface as ptl
import cmiles
from openeye import oechem
import json
client = ptl.FractalClient('https://localhost:7777/', verify=False)
fragmenter_wf = workflow_api.WorkFlow(client=client,
workflow_id='torsiondrive_input',
workflow_json='example_workflow.json')
# Load Roche molecules
roche_mols = chemi.file_to_oemols('OpenFF_references.sdf')
smiles = [
cmiles.utils.mol_to_smiles(mol, mapped=False, explicit_hydrogen=False)
for mol in roche_mols
]
# Put smiles in format for generating torsion input files
frags = {}
for sm in smiles:
identifiers = cmiles.get_molecule_ids(sm, toolkit='openeye', strict=False)
frags[sm] = {'identifiers': identifiers, 'provenance': {'routine': {}}}
# Generate torsiondrive input
torsiondrive_inputs = {}
for frag in frags:
