def setUp(self):
self.materials = MemoryStore()
self.materials.connect()
materials = loadfn(os.path.join(TEST_DIR, "test_materials.json"))
materials = jsanitize(materials, strict=True, allow_bson=True)
self.materials.update(materials)
self.propstore = MemoryStore()
self.propstore.connect()
def setUp(self):
"""Set up materials and robocrys stores."""
struct_docs = loadfn(test_mats, cls=None)
self.materials = MemoryStore("materials")
self.materials.connect()
self.materials.update(struct_docs)
self.robocrys = MemoryStore("robocrys")
def memory_data_jobstore():
from maggma.stores import MemoryStore
from jobflow import JobStore
store = JobStore(MemoryStore(), additional_stores={"data": MemoryStore()})
store.connect()
return store
def setUpClass(cls):
materials = MemoryStore("materials")
materials.connect()
docs = []
for n, mat_string in enumerate(["Si", "Sn", "TiO2", "VO2"]):
docs.append({"task_id": n, "structure": PymatgenTest.get_structure(mat_string).as_dict()})
materials.update(docs, key='task_id')
elasticity = MemoryStore("elasticity")
elasticity.connect()
elasticity.update(docs[0:1], key="task_id")
cls.materials = materials
cls.elasticity = elasticity
def test_get_item(self):
dt = MemoryStore()
at = MemoryStore()
qc = [{
'catalog': 'icsd',
'k': 100,
'exclude': [],
'filter': [('auid', '__eq__', 'aflow:0132ab6b9cddd429')],
'select': [],
'targets': ['data', 'auid']
}]
kw = ['ael_elastic_anisotropy']
afi = AflowIngester(data_target=dt,
auid_target=at,
keywords=kw,
query_configs=qc)
count = 0
for item in afi.get_items():
count += 1
self.assertEqual(count,
1,
msg="The query returned more than one material")
expected_attributes = [
'auid', 'aurl', 'compound', 'ael_elastic_anisotropy'
]
self.assertIsInstance(item, tuple)
self.assertEqual(len(item), 2)
entry, targets = item
self.assertIsInstance(entry, Entry)
self.assertIsInstance(targets, list)
self.assertCountEqual(list(entry.attributes.keys()),
expected_attributes)
self.assertCountEqual(targets, ['data', 'auid'])
self.assertEqual(entry.auid, 'aflow:0132ab6b9cddd429')
# Test empty query
empty_qc = [{
'catalog': 'lib1',
'k': 100,
'exclude': [],
'filter': [('auid', '__eq__', 'aflow:0132ab6b9cddd429')],
'select': [],
'targets': ['data', 'auid']
}]
afi = AflowIngester(data_target=dt,
auid_target=at,
keywords=kw,
query_configs=empty_qc)
with self.assertRaises(ValueError):
_ = afi.get_items().__next__()
def test_keys():
with mock_s3():
conn = boto3.client("s3")
conn.create_bucket(Bucket="bucket1")
index = MemoryStore("index", key=1)
with pytest.raises(AssertionError, match=r"Since we are.*"):
store = S3Store(index, "bucket1", s3_workers=4, key=1)
index = MemoryStore("index", key="key1")
with pytest.warns(UserWarning, match=r"The desired S3Store.*$"):
store = S3Store(index, "bucket1", s3_workers=4, key="key2")
store.connect()
store.update({"key1": "mp-1", "data": "1234"})
with pytest.raises(KeyError):
store.update({"key2": "mp-2", "data": "1234"})
assert store.key == store.index.key == "key1"
def setUpClass(cls):
# Set up test db, etc.
cls.test_materials = MemoryStore("mat_descriptors")
cls.test_materials.connect()
struct_docs = loadfn(test_structs, cls=None)
cls.test_materials.update(struct_docs)
def test_builder(self):
test_basic_descriptors = MemoryStore("test_basic_descriptors")
sd_builder = BasicDescriptorsBuilder(self.test_materials,
test_basic_descriptors)
sd_builder.connect()
meta_comp_descr = loadfn(test_meta_comp_descr, cls=None)
for t in sd_builder.get_items():
processed = sd_builder.process_item(t)
if processed:
sd_builder.update_targets([processed])
else:
import nose
nose.tools.set_trace()
for md in ['nsites', 'formula_pretty', 'nelements']:
self.assertEqual(processed['meta']['atomate'][md],
meta_comp_descr['meta'][t['task_id']][md])
for k, v in meta_comp_descr['comp_descr'][t['task_id']][0].items():
if k != 'name':
self.assertAlmostEqual(
processed['composition_descriptors'][0][k], v)
self.assertEqual(len([t for t in sd_builder.get_items()]), 0)
# Remove one data piece in diamond entry and test partial update.
test_basic_descriptors.collection.find_one_and_update(
{'task_id': 'mp-66'}, {'$unset': {
'site_descriptors': 1
}})
items = [e for e in list(sd_builder.get_items())]
self.assertEqual(len(items), 1)
def source2():
store = MemoryStore("source2", key="k", last_updated_field="lu")
store.connect()
store.ensure_index("k")
store.ensure_index("lu")
store.update([{"k": k, "c": "c", "d": "d"} for k in range(15)])
return store
def test_reporting(mongostore, reporting_store):
memorystore = MemoryStore("temp")
builder = CopyBuilder(mongostore, memorystore)
mongostore.update([{
mongostore.key: i,
mongostore.last_updated_field: datetime.utcnow()
} for i in range(10)])
runner = CliRunner()
with runner.isolated_filesystem():
dumpfn(builder, "test_builder.json")
dumpfn(reporting_store, "test_reporting_store.json")
result = runner.invoke(
run,
["-v", "test_builder.json", "-r", "test_reporting_store.json"])
assert result.exit_code == 0
report_docs = list(reporting_store.query())
assert len(report_docs) == 3
start_doc = next(d for d in report_docs
if d["event"] == "BUILD_STARTED")
assert "sources" in start_doc
assert "targets" in start_doc
end_doc = next(d for d in report_docs if d["event"] == "BUILD_ENDED")
assert "errors" in end_doc
assert "warnings" in end_doc
update_doc = next(d for d in report_docs if d["event"] == "UPDATE")
assert "items" in update_doc
def _get_correlation_values():
full_propstore = MemoryStore()
with open(os.path.join(CORR_TEST_DIR, "correlation_propnet_data.json"),
'r') as f:
data = json.load(f)
full_propstore.connect()
full_propstore.update(jsanitize(data, strict=True, allow_bson=True))
correlation_store = MemoryStore()
builder = CorrelationBuilder(full_propstore,
correlation_store,
props=PROPNET_PROPS,
funcs='all',
from_quantity_db=False)
runner = Runner([builder])
runner.run()
return builder
def s3store():
with mock_s3():
conn = boto3.client("s3")
conn.create_bucket(Bucket="bucket1")
index = MemoryStore("index")
store = S3Store(index, "bucket1")
store.connect()
store.update(
[
{
"task_id": "mp-1",
"data": "asd",
store.last_updated_field: datetime.utcnow(),
}
]
)
store.update(
[
{
"task_id": "mp-3",
"data": "sdf",
store.last_updated_field: datetime.utcnow(),
}
]
)
yield store
def setUpClass(self):
# Set up test db, etc.
self.test_materials = MemoryStore("mat_site_fingerprint")
self.test_materials.connect()
struct_docs = loadfn(test_structs, cls=None)
self.test_materials.update(struct_docs)
def create_mp_builder_test_docs():
"""
Create documents for propnet MP builder tests. Outputs JSON file to test directory.
"""
formulas = ["BaNiO3", "Si", "Fe2O3", "Cs"]
mgstore = MongograntStore("ro:knowhere.lbl.gov/mp_core", "materials")
builder = PropnetBuilder(mgstore,
MemoryStore(),
criteria={
"pretty_formula": {
"$in": formulas
},
"e_above_hull": 0
})
builder.connect()
materials = list(builder.get_items())
deprecated_item = mgstore.query_one({'deprecated': True, 'sbxn': 'core'})
# Create fake, non-core sandboxed material to represent proprietary info
sandboxed_item = materials[0].copy()
sandboxed_item.update({'sbxn': ['test_sbx'], 'task_id': 'mp-fakesbx'})
del sandboxed_item['_id']
materials.extend([deprecated_item, sandboxed_item])
dumpfn(materials, os.path.join(MP_TEST_DIR, "test_materials.json"))
def test_run_builder(mongostore):
memorystore = MemoryStore("temp")
builder = CopyBuilder(mongostore, memorystore)
mongostore.update([{
mongostore.key: i,
mongostore.last_updated_field: datetime.utcnow()
} for i in range(10)])
runner = CliRunner()
with runner.isolated_filesystem():
dumpfn(builder, "test_builder.json")
result = runner.invoke(run, ["-v", "test_builder.json"])
assert result.exit_code == 0
assert "CopyBuilder" in result.output
assert "SerialProcessor" in result.output
result = runner.invoke(run, ["-vvv", "--no_bars", "test_builder.json"])
assert result.exit_code == 0
assert "Get" not in result.output
assert "Update" not in result.output
result = runner.invoke(run, ["-v", "-n", "2", "test_builder.json"])
assert result.exit_code == 0
assert "CopyBuilder" in result.output
assert "MultiProcessor" in result.output
result = runner.invoke(
run, ["-vvv", "-n", "2", "--no_bars", "test_builder.json"])
assert result.exit_code == 0
assert "Get" not in result.output
assert "Update" not in result.output
def test_additional(memory_store):
from copy import deepcopy
import boto3
from maggma.stores import MemoryStore, S3Store
from moto import mock_s3
from jobflow import JobStore
with mock_s3():
conn = boto3.resource("s3", region_name="us-east-1")
conn.create_bucket(Bucket="bucket1")
index = MemoryStore("index", key="blob_uuid")
s3_store = S3Store(index, "bucket1", key="blob_uuid")
store = JobStore(
memory_store,
additional_stores={
"data": deepcopy(memory_store),
"data_s3": s3_store
},
)
with store as s:
assert s
assert s.name == "JobStore-mem://memory_db"
assert s._collection is not None
assert s.additional_stores["data_s3"].searchable_fields == [
"job_uuid",
"job_index",
]
def source(docs):
store = MemoryStore("source", key="k", last_updated_field="lu")
store.connect()
store.ensure_index("k")
store.ensure_index("lu")
store.update(docs)
return store
def setUp(self):
coords = list()
coords.append([0, 0, 0])
coords.append([0.75, 0.5, 0.75])
lattice = Lattice([[3.8401979337, 0.00, 0.00],
[1.9200989668, 3.3257101909, 0.00],
[0.00, -2.2171384943, 3.1355090603]])
self.structure = Structure(lattice, ["Si", "Si"], coords)
materials = MemoryStore("materials")
electronic_structure = MemoryStore("electronic_structure")
bandstructures = MemoryStore("bandstructure")
dos = MemoryStore("dos")
self.builder = ElectronicStructureImageBuilder(materials,
electronic_structure,
bandstructures, dos)
def source1():
store = MemoryStore("source1", key="k", last_updated_field="lu")
store.connect()
store.ensure_index("k")
store.ensure_index("lu")
store.update([{"k": k, "a": "a", "b": "b"} for k in range(10)])
return store
def setUp(self):
stores = [MemoryStore(str(i)) for i in range(7)]
builder1 = Bldr([stores[0], stores[1], stores[2]],
[stores[3], stores[4], stores[5]])
builder2 = Bldr([stores[0], stores[1], stores[3]],
[stores[3], stores[6]])
self.builders = [builder1, builder2]
def setUpClass(cls):
add_builtin_models_to_registry()
cls.materials = MemoryStore()
cls.materials.connect()
materials = loadfn(os.path.join(TEST_DATA_DIR, "test_materials.json"))
materials = jsanitize(materials, strict=True, allow_bson=True)
cls.materials.update(materials)
cls.propstore = None
def setUp(self):
self.memorystore = MemoryStore("test")
self.memorystore.connect()
self.aliasingstore = AliasingStore(self.memorystore, {
"a": "b",
"c.d": "e",
"f": "g.h"
})
def setUp(self):
self.index = MemoryStore("index'")
with patch("boto3.resource") as mock_resource:
mock_resource.return_value = MagicMock()
mock_resource("s3").list_buckets.return_value = [
"bucket1", "bucket2"
]
self.s3store = AmazonS3Store(self.index, "bucket1")
self.s3store.connect()
def memory_jobstore():
from maggma.stores import MemoryStore
from jobflow import JobStore
store = JobStore(MemoryStore())
store.connect()
return store
def test_no_bucket():
with mock_s3():
conn = boto3.client("s3")
conn.create_bucket(Bucket="bucket1")
index = MemoryStore("index")
store = S3Store(index, "bucket2")
with pytest.raises(RuntimeError, match=r".*Bucket not present.*"):
store.connect()
def alias_store():
memorystore = MemoryStore("test")
memorystore.connect()
alias_store = AliasingStore(memorystore, {
"a": "b",
"c.d": "e",
"f": "g.h"
})
return alias_store
def setUpClass(cls) -> None:
Registry.clear_all_registries()
add_builtin_symbols_to_registry()
cls.afa_web = AflowAdapter()
store_data = loadfn(os.path.join(TEST_DATA_DIR, 'aflow_store.json'))
store = MemoryStore()
store.connect()
store.update(store_data, key='auid')
cls.afa_store = AflowAdapter(store)
def setUp(self):
self.materials = JSONStore(boltztrap4dos_mat)
self.materials.connect()
self.bandstructure = JSONStore(boltztrap4dos_bs)
self.bandstructure.connect()
self.dos_ref = JSONStore(boltztrap4dos_dos)
self.dos_ref.connect()
self.dos = MemoryStore("dos")
self.dos.connect()
def setUp(self):
coords = list()
coords.append([0, 0, 0])
coords.append([0.75, 0.5, 0.75])
lattice = [
[3.8401979337, 0.00, 0.00],
[1.9200989668, 3.3257101909, 0.00],
[0.00, -2.2171384943, 3.1355090603],
]
structure = Structure(lattice, ["Si", "Si"], coords)
input_sets = {
"GGA Structure Optimization": MPRelaxSet(structure),
"GGA Static": MPStaticSet(structure),
"GGA NSCF Line": MPNonSCFSet(structure, mode="line"),
"GGA NSCF Uniform": MPNonSCFSet(structure, mode="uniform"),
}
tasks = []
t_id = 1
for task_type, input_set in input_sets.items():
doc = {
"true_task_type": task_type,
"last_updated": datetime.now(),
"task_id": t_id,
"state": "successful",
"orig_inputs": {
"incar": input_set.incar.as_dict(),
"kpoints": input_set.kpoints.as_dict(),
},
"output": {
"structure": structure.as_dict()
},
}
t_id += 1
tasks.append(doc)
self.test_tasks = MemoryStore("tasks")
self.task_types = MemoryStore("task_types")
self.test_tasks.connect()
self.task_types.connect()
self.test_tasks.update(tasks)
def setUpClass(cls):
add_builtin_models_to_registry()
cls.propnet_props = PROPNET_PROPS
cls.propstore = MemoryStore()
cls.propstore.connect()
materials_file = os.path.join(TEST_DATA_DIR,
"correlation_propnet_data.json")
if os.path.exists(materials_file):
with open(materials_file, 'r') as f:
materials = json.load(f)
materials = jsanitize(materials, strict=True, allow_bson=True)
cls.propstore.update(materials)
cls.quantity_store = MemoryStore()
cls.quantity_store.connect()
quantities_file = os.path.join(
TEST_DATA_DIR, "correlation_propnet_quantity_data.json")
if os.path.exists(quantities_file):
with open(quantities_file, 'r') as f:
quantities = json.load(f)
quantities = jsanitize(quantities, strict=True, allow_bson=True)
cls.quantity_store.update(quantities, key='internal_id')
cls.correlation = None
# vickers hardness (x-axis) vs. bulk modulus (y-axis)
cls.correlation_values_vickers_bulk = {
'linlsq': 0.49302857395714916,
'pearson': 0.7021599347421844,
'spearman': 0.7658801470036752,
'mic': 0.7069707677575771,
'theilsen': 0.4498696692135892,
'ransac': 0.412850153689996
}
cls.correlation_values_bulk_vickers = {
'linlsq': 0.49302857395714916,
'pearson': 0.7021599347421844,
'spearman': 0.7658801470036752,
'mic': 0.7069707677575771,
'theilsen': 0.46461142885951223,
'ransac': 0.3699261228882036
}
