def setUp(self):
self.materials = JSONStore(test_mats, lu_type='isoformat')
self.bonding = MemoryStore("bonding")
def validation_store(tasks_store):
validation_store = MemoryStore()
builder = TaskValidator(tasks=tasks_store,
task_validation=validation_store)
builder.run()
return validation_store
def test_serialization(tmpdir):
builder = MaterialsBuilder(MemoryStore(), MemoryStore(), MemoryStore())
dumpfn(builder.as_dict(), Path(tmpdir) / "test.json")
loadfn(Path(tmpdir) / "test.json")
def sandbox_store():
memstore = MemoryStore()
store = SandboxStore(memstore, sandbox="test")
store.connect()
return store
def owner_store():
store = MemoryStore("owners", key="name")
store.connect()
store.update([d.dict() for d in owners])
return store
def source():
store = MemoryStore("source", key="k", last_updated_field="lu")
store.connect()
store.ensure_index("k")
store.ensure_index("lu")
return store
def setUp(self):
# Set up test db, set up mpsft, etc.
self.test_tasks = JSONStore([test_tasks])
self.elasticity = MemoryStore("atomate_tasks")
self.test_tasks.connect()
self.elasticity.connect()
def setUp(self):
self.materials = JSONStore(test_mats, lu_type='isoformat')
self.materials.connect()
self.thermo = MemoryStore("thermo")
self.thermo.connect()
def test_process_item(self):
dt = MemoryStore()
at = MemoryStore()
qc = [{
'catalog': 'icsd',
'k': 100,
'exclude': ['compound'],
'filter': [('auid', '__eq__', 'aflow:0132ab6b9cddd429')],
'select': [],
'targets': ['data', 'auid']
}]
kw = ['auid', 'aurl', 'ael_elastic_anisotropy', 'files']
afi = AflowIngester(data_target=dt,
auid_target=at,
keywords=kw,
query_configs=qc)
item = afi.get_items().__next__()
processed = afi.process_item(item)
self.assertIsInstance(processed, tuple)
self.assertEqual(len(processed), 2)
data, auid = processed
expected_data_keys = kw + [
'AEL_elastic_tensor_json', 'CONTCAR_relax_vasp'
]
expected_auid_keys = ['auid', 'aurl', 'compound']
self.assertCountEqual(list(data.keys()), expected_data_keys)
self.assertTrue(all(v is not None for v in data.values()))
self.assertCountEqual(list(auid.keys()), expected_auid_keys)
self.assertIsNone(auid['compound'])
# Omit auid target
afi = AflowIngester(data_target=dt, keywords=kw, query_configs=qc)
item = afi.get_items().__next__()
processed = afi.process_item(item)
self.assertIsInstance(processed, tuple)
self.assertEqual(len(processed), 2)
_, auid = processed
self.assertEqual(auid, dict())
# Test for filtering null properties
qc = [{
'catalog': 'lib1',
'k': 100,
'exclude': ['compound'],
'filter': [('auid', '__eq__', 'aflow:7203c28b8396b9c9')],
'select': [],
'targets': ['data', 'auid']
}]
afi = AflowIngester(data_target=dt,
auid_target=at,
keywords=kw,
query_configs=qc,
filter_null_properties=True)
item = afi.get_items().__next__()
processed = afi.process_item(item)
self.assertIsInstance(processed, tuple)
self.assertEqual(len(processed), 2)
data, auid = processed
self.assertNotIn('ael_elastic_anisotropy', data)
self.assertNotIn('AEL_elastic_tensor_json', data)
self.assertNotIn('compound', auid)
self.logger.info("processing item: {}".format(item))
# time.sleep(random.randint(0,3))
return {item: "processed"}
def update_targets(self, items):
self.logger.info("Updating targets ...")
self.logger.info("Received {} processed items".format(len(items)))
self.logger.info("Updated items: {}".format(list(items)))
def finalize(self, cursor):
self.logger.info("Finalizing ...")
self.logger.info("DONE!")
if __name__ == '__main__':
N = 10
chunk_size = 3
stores = [MemoryStore(str(i)) for i in range(7)]
sources = [stores[0], stores[1], stores[3]]
targets = [stores[3], stores[6]]
mdb = MyDumbBuilder(N, sources, targets, chunk_size=chunk_size)
builders = [mdb]
runner = Runner(builders)
logstreamhandle(runner)
runner.run()
class TestThermo(unittest.TestCase):
def setUp(self):
self.materials = JSONStore(test_mats, lu_type='isoformat')
self.materials.connect()
self.thermo = MemoryStore("thermo")
self.thermo.connect()
def test_get_entries(self):
tbuilder = ThermoBuilder(self.materials, self.thermo)
self.assertEqual(len(tbuilder.get_entries("Sr")), 7)
self.assertEqual(len(tbuilder.get_entries("Hf")), 4)
self.assertEqual(len(tbuilder.get_entries("O")), 6)
self.assertEqual(len(tbuilder.get_entries("Hf-O-Sr")), 44)
self.assertEqual(len(tbuilder.get_entries("Sr-Hf")), 11)
def test_chemsys_permutations(self):
self.assertEqual(len(chemsys_permutations("Sr")), 1)
self.assertEqual(len(chemsys_permutations("Sr-Hf")), 3)
self.assertEqual(len(chemsys_permutations("Sr-Hf-O")), 7)
def test_process_items(self):
tbuilder = ThermoBuilder(self.materials, self.thermo)
# Ensure only one doc gets a 0 e_above_hull
entries = tbuilder.get_entries("Sr")
t_docs = tbuilder.process_item(("core", entries))
e_above_hulls = [t['thermo']['e_above_hull'] for t in t_docs]
self.assertEqual(len([e for e in e_above_hulls if e == 0.0]), 1)
entries = tbuilder.get_entries("Hf")
t_docs = tbuilder.process_item(("core", entries))
e_above_hulls = [t['thermo']['e_above_hull'] for t in t_docs]
self.assertEqual(len([e for e in e_above_hulls if e == 0.0]), 1)
entries = tbuilder.get_entries("O")
t_docs = tbuilder.process_item(("core", entries))
e_above_hulls = [t['thermo']['e_above_hull'] for t in t_docs]
self.assertEqual(len([e for e in e_above_hulls if e == 0.0]), 1)
# Ensure 4 docs iwth 0 e_above hull for convex hull for Sr-O
entries = tbuilder.get_entries("Sr-O")
t_docs = tbuilder.process_item(("core", entries))
e_above_hulls = [t['thermo']['e_above_hull'] for t in t_docs]
self.assertEqual(len([e for e in e_above_hulls if e == 0.0]), 4)
# Ensure 4 docs iwth 0 e_above hull for convex hull Hf-O
entries = tbuilder.get_entries("Hf-O")
t_docs = tbuilder.process_item(("core", entries))
e_above_hulls = [t['thermo']['e_above_hull'] for t in t_docs]
self.assertEqual(len([e for e in e_above_hulls if e == 0.0]), 3)
# Ensure 4 docs iwth 0 e_above hull for convex hull
entries = tbuilder.get_entries("Sr-Hf-O")
t_docs = tbuilder.process_item(("core", entries))
e_above_hulls = [t['thermo']['e_above_hull'] for t in t_docs]
self.assertEqual(len(e_above_hulls), 44)
self.assertEqual(len([e for e in e_above_hulls if e == 0.0]), 7)
def test_update_targets(self):
items = [[{
"task_id": 1,
"_sbxn": ["core"]
}] * 3, [{
"task_id": 2,
"_sbxn": ["core"]
}] * 4, [{
"task_id": 3,
"_sbxn": ["core"]
}] * 4]
tbuilder = ThermoBuilder(self.materials, self.thermo)
tbuilder.update_targets(items)
self.assertEqual(len(self.thermo.distinct("task_id")), 3)
self.assertEqual(tbuilder.completed_tasks, {1, 2, 3})
def test_get_items(self):
tbuilder = ThermoBuilder(self.materials, self.thermo)
self.assertEqual(len(list(tbuilder.get_items())), 1)
def setUp(self):
self.materials = JSONStore(test_mats)
self.materials.connect()
self.thermo = MemoryStore("thermo")
self.thermo.connect()
def setUp(self):
self.propstore = MemoryStore()
self.propstore.connect()
class MPBuilderTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
add_builtin_models_to_registry()
cls.materials = MemoryStore()
cls.materials.connect()
materials = loadfn(os.path.join(TEST_DATA_DIR, "test_materials.json"))
materials = jsanitize(materials, strict=True, allow_bson=True)
cls.materials.update(materials)
cls.propstore = None
def setUp(self):
self.propstore = MemoryStore()
self.propstore.connect()
def test_serial_runner(self):
builder = PropnetBuilder(self.materials, self.propstore)
runner = Runner([builder])
runner.run()
def test_multiproc_runner(self):
builder = PropnetBuilder(self.materials, self.propstore)
runner = Runner([builder], max_workers=2)
runner.run()
def test_process_item(self):
item = self.materials.query_one(criteria={"pretty_formula": "Cs"})
builder = PropnetBuilder(self.materials, self.propstore)
processed = builder.process(item)
self.assertIsNotNone(processed)
# Ensure vickers hardness gets populated
self.assertIn("vickers_hardness", processed)
if 'created_at' in item.keys():
date_value = item['created_at']
else:
date_value = ""
# Check that provenance values propagate correctly
current_quantity = processed['vickers_hardness']['quantities'][0]
at_deepest_level = False
while not at_deepest_level:
current_provenance = current_quantity['provenance']
if current_provenance['inputs'] is not None:
self.assertEqual(current_provenance['source']['source'],
"propnet")
self.assertEqual(current_provenance['source']['source_key'],
current_quantity['internal_id'])
self.assertNotIn(current_provenance['source']['date_created'],
("", None))
current_quantity = current_provenance['inputs'][0]
else:
self.assertEqual(current_provenance['source']['source'],
"Materials Project")
self.assertEqual(current_provenance['source']['source_key'],
item['task_id'])
self.assertEqual(current_provenance['source']['date_created'],
date_value)
at_deepest_level = True
def test_deprecated_filter(self):
# Check default (False)
builder = PropnetBuilder(self.materials, self.propstore)
self.assertFalse(builder.include_deprecated)
# Check False explicitly
builder = PropnetBuilder(self.materials,
self.propstore,
include_deprecated=False)
builder.connect()
is_deprecated = [item['deprecated'] for item in builder.get_items()]
self.assertTrue(all(not v for v in is_deprecated))
# Check True explicitly
builder = PropnetBuilder(self.materials,
self.propstore,
include_deprecated=True)
builder.connect()
is_deprecated = [item['deprecated'] for item in builder.get_items()]
self.assertTrue(any(is_deprecated))
def test_sandboxed_filter(self):
# Check default (False)
builder = PropnetBuilder(self.materials, self.propstore)
self.assertFalse(builder.include_sandboxed)
# Check False explicitly
builder = PropnetBuilder(self.materials,
self.propstore,
include_sandboxed=False)
builder.connect()
is_in_core = ['core' in item['sbxn'] for item in builder.get_items()]
self.assertTrue(all(v for v in is_in_core))
# Check True explicitly
builder = PropnetBuilder(self.materials,
self.propstore,
include_sandboxed=True)
builder.connect()
is_not_in_core = [
'core' not in item['sbxn'] for item in builder.get_items()
]
self.assertTrue(any(is_not_in_core))
for i in range(1000):
yield {"val": i, "task_id": i}
def process_item(self, item):
if item["val"] % 10 == 0:
self.logger.debug("Processing: {}".format(item["val"]))
proc_val = np.sqrt(np.square(float(item["val"])))
item["proc_val"] = proc_val
return item
def update_targets(self, items):
self.logger.info("Updating {} items".format(len(items)))
self.temp_storage_store.update(items)
if __name__ == "__main__":
root = logging.getLogger()
root.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
ch.setFormatter(formatter)
root.addHandler(ch)
mem = MemoryStore("processed")
bldr = DummyBuilder(mem)
mpi_proc = MPIProcessor([bldr])
mpi_proc.process(0)
def owner_store():
store = MemoryStore("owners", key="name")
store.connect()
store.update([jsonable_encoder(d) for d in owners])
return store
def setUp(self):
# Set up test db, set up mpsft, etc.
self.test_tasks = JSONStore(test_tasks)
self.task_types = MemoryStore("task_types")
self.test_tasks.connect()
self.task_types.connect()
def pet_store():
store = MemoryStore("pets", key="name")
store.connect()
store.update([jsonable_encoder(d) for d in pets])
return store
def target():
store = MemoryStore("target", key="k", last_updated_field="lu")
store.connect()
store.ensure_index("k")
store.ensure_index("lu")
return store
def search_helper(payload,
base: str = "/?",
debug=True) -> Tuple[Response, Any]:
"""
Helper function to directly query search endpoints
Args:
store: store f
base: base of the query, default to /query?
client: TestClient generated from FastAPI
payload: query in dictionary format
debug: True = print out the url, false don't print anything
Returns:
request.Response object that contains the response of the correspoding payload
"""
owner_store = MemoryStore("owners", key="name")
owner_store.connect()
owner_store.update([d.dict() for d in owners])
pets_store = MemoryStore("pets", key="name")
pets_store.connect()
pets_store.update([jsonable_encoder(d) for d in pets])
resources = {
"owners": [
ReadOnlyResource(
owner_store,
Owner,
query_operators=[
StringQueryOperator(model=Owner), # type: ignore
NumericQuery(model=Owner), # type: ignore
SparseFieldsQuery(model=Owner),
PaginationQuery(),
],
)
],
"pets": [
ReadOnlyResource(
pets_store,
Owner,
query_operators=[
StringQueryOperator(model=Pet),
NumericQuery(model=Pet),
SparseFieldsQuery(model=Pet),
PaginationQuery(),
],
)
],
}
api = API(resources=resources)
client = TestClient(api.app)
url = base + urlencode(payload)
if debug:
print(url)
res = client.get(url)
try:
data = res.json().get("data", [])
except Exception:
data = res.reason
return res, data
class BuilderTest(unittest.TestCase):
def setUp(self):
self.materials = MemoryStore()
self.materials.connect()
materials = loadfn(os.path.join(TEST_DIR, "test_materials.json"))
materials = jsanitize(materials, strict=True, allow_bson=True)
self.materials.update(materials)
self.propstore = MemoryStore()
self.propstore.connect()
def test_serial_runner(self):
builder = PropnetBuilder(self.materials, self.propstore)
runner = Runner([builder])
runner.run()
def test_multiproc_runner(self):
builder = PropnetBuilder(self.materials, self.propstore)
runner = Runner([builder])
runner.run()
def test_process_item(self):
item = self.materials.query_one(criteria={"pretty_formula": "Cs"})
builder = PropnetBuilder(self.materials, self.propstore)
processed = builder.process_item(item)
self.assertIsNotNone(processed)
# Ensure vickers hardness gets populated
self.assertIn("vickers_hardness", processed)
if 'created_at' in item.keys():
date_value = item['created_at']
else:
date_value = ""
# Check that provenance values propagate correctly
current_quantity = processed['vickers_hardness']['quantities'][0]
at_deepest_level = False
while not at_deepest_level:
current_provenance = current_quantity['provenance']
if current_provenance['inputs'] is not None:
self.assertEqual(current_provenance['source']['source'],
"propnet")
self.assertEqual(current_provenance['source']['source_key'],
current_quantity['internal_id'])
self.assertNotIn(current_provenance['source']['date_created'],
("", None))
current_quantity = current_provenance['inputs'][0]
else:
self.assertEqual(current_provenance['source']['source'],
"Materials Project")
self.assertEqual(current_provenance['source']['source_key'],
item['task_id'])
self.assertEqual(current_provenance['source']['date_created'],
date_value)
at_deepest_level = True
# @unittest.skipIf(not os.path.isfile("runner.json"), "No runner file")
# def test_runner_pipeline(self):
# from monty.serialization import loadfn
# runner = loadfn("runner.json")
# runner.builders[0].connect()
# items = list(runner.builders[0].get_items())
# processed = runner.builders[0].process_item(items[0])
# runner.run()
# Just here for reference, in case anyone wants to create a new set
# of test materials -jhm
@unittest.skipIf(True, "Skipping test materials creation")
def create_test_docs(self):
formulas = ["BaNiO3", "Si", "Fe2O3", "Cs"]
from maggma.advanced_stores import MongograntStore
from monty.serialization import dumpfn
mgstore = MongograntStore("ro:matgen2.lbl.gov/mp_prod", "materials")
builder = PropnetBuilder(
mgstore, self.propstore, criteria={"pretty_formula": {"$in": formulas},
"e_above_hull": 0})
builder.connect()
dumpfn(list(builder.get_items()), "test_materials.json")
class TestAliasingStore(unittest.TestCase):
def setUp(self):
self.memorystore = MemoryStore("test")
self.memorystore.connect()
self.aliasingstore = AliasingStore(self.memorystore, {
"a": "b",
"c.d": "e",
"f": "g.h"
})
def test_query(self):
d = [{"b": 1}, {"e": 2}, {"g": {"h": 3}}]
self.memorystore.collection.insert_many(d)
self.assertTrue("a" in list(
self.aliasingstore.query(criteria={"a": {
"$exists": 1
}}))[0])
self.assertTrue("c" in list(
self.aliasingstore.query(criteria={"c.d": {
"$exists": 1
}}))[0])
self.assertTrue("d" in list(
self.aliasingstore.query(criteria={"c.d": {
"$exists": 1
}}))[0].get("c", {}))
self.assertTrue("f" in list(
self.aliasingstore.query(criteria={"f": {
"$exists": 1
}}))[0])
def test_update(self):
self.aliasingstore.update([{
"task_id": "mp-3",
"a": 4
}, {
"task_id": "mp-4",
"c": {
"d": 5
}
}, {
"task_id": "mp-5",
"f": 6
}])
self.assertEqual(
list(self.aliasingstore.query(
criteria={"task_id": "mp-3"}))[0]["a"], 4)
self.assertEqual(
list(self.aliasingstore.query(
criteria={"task_id": "mp-4"}))[0]["c"]["d"], 5)
self.assertEqual(
list(self.aliasingstore.query(
criteria={"task_id": "mp-5"}))[0]["f"], 6)
self.assertEqual(
list(self.aliasingstore.store.query(
criteria={"task_id": "mp-3"}))[0]["b"], 4)
self.assertEqual(
list(self.aliasingstore.store.query(
criteria={"task_id": "mp-4"}))[0]["e"], 5)
self.assertEqual(
list(self.aliasingstore.store.query(
criteria={"task_id": "mp-5"}))[0]["g"]["h"], 6)
def test_substitute(self):
aliases = {"a": "b", "c.d": "e", "f": "g.h"}
d = {"b": 1}
substitute(d, aliases)
self.assertTrue("a" in d)
d = {"e": 1}
substitute(d, aliases)
self.assertTrue("c" in d)
self.assertTrue("d" in d.get("c", {}))
d = {"g": {"h": 4}}
substitute(d, aliases)
self.assertTrue("f" in d)
d = None
substitute(d, aliases)
self.assertTrue(d is None)
def test_bad_import(mocker):
mocker.patch("maggma.stores.aws.boto3", None)
with pytest.raises(RuntimeError):
index = MemoryStore("index'")
S3Store(index, "bucket1")
def setUp(self):
self.store = MemoryStore()
self.sandboxstore = SandboxStore(self.store, sandbox="test")
def validation_store():
return MemoryStore()
def setUp(self):
self.materials = JSONStore(test_mats)
self.bonding = MemoryStore("bonding")
def materials_store():
return MemoryStore()
def setUp(self):
self.materials = JSONStore(test_mats)
self.bond_valence = MemoryStore("bond_valence")
def test_get_all_basic_descriptors(self):
test_basic_descriptors = MemoryStore("test_basic_descriptors")
sd_builder = BasicDescriptorsBuilder(self.test_materials,
test_basic_descriptors)
C = self.test_materials.query_one(criteria={"task_id": "mp-66"})
NaCl = self.test_materials.query_one(criteria={"task_id": "mp-22862"})
Fe = self.test_materials.query_one(criteria={"task_id": "mp-13"})
# Diamond.
d = sd_builder.get_site_descriptors_from_struct(
Structure.from_dict(C["structure"]))
for di in d.values():
self.assertEqual(len(di), 2)
self.assertEqual(d['cn_VoronoiNN'][0]['CN_VoronoiNN'], 20)
self.assertAlmostEqual(d['cn_wt_VoronoiNN'][0]['CN_VoronoiNN'],
4.5381162)
self.assertEqual(d['cn_JMolNN'][0]['CN_JMolNN'], 4)
self.assertAlmostEqual(d['cn_wt_JMolNN'][0]['CN_JMolNN'], 4.9617398)
self.assertEqual(d['cn_MinimumDistanceNN'][0]['CN_MinimumDistanceNN'],
4)
self.assertAlmostEqual(
d['cn_wt_MinimumDistanceNN'][0]['CN_MinimumDistanceNN'], 4)
self.assertEqual(d['cn_MinimumOKeeffeNN'][0]['CN_MinimumOKeeffeNN'], 4)
self.assertAlmostEqual(
d['cn_wt_MinimumOKeeffeNN'][0]['CN_MinimumOKeeffeNN'], 4)
self.assertEqual(d['cn_MinimumVIRENN'][0]['CN_MinimumVIRENN'], 4)
self.assertAlmostEqual(d['cn_wt_MinimumVIRENN'][0]['CN_MinimumVIRENN'],
4)
self.assertEqual(
d['cn_BrunnerNN_reciprocal'][0]['CN_BrunnerNN_reciprocal'], 4)
self.assertAlmostEqual(
d['cn_wt_BrunnerNN_reciprocal'][0]['CN_BrunnerNN_reciprocal'], 4)
# Current value for the below quantity is 0.9618085
self.assertTrue(0.95 <= d['csf'][0]['tetrahedral CN_4'] <= 1)
ds = sd_builder.get_statistics(d)
self.assertTrue('csf' in list(ds.keys()))
for k, dsk in ds.items():
for di in dsk:
self.assertEqual(len(list(di.keys())), 3)
def get_index(li, optype):
try:
return next(i for i, d in enumerate(li) if d['name'] == optype)
except StopIteration:
raise RuntimeError('did not find optype {}'.format(optype))
# Current value for the three below quantities is 0.9618085
self.assertTrue(0.95 <= ds['csf'][get_index(
ds['csf'], 'tetrahedral CN_4')]['mean'] <= 1)
self.assertAlmostEqual(
ds['csf'][get_index(ds['csf'], 'tetrahedral CN_4')]['std'], 0)
self.assertAlmostEqual(
ds['csf'][get_index(ds['csf'], 'octahedral CN_6')]['mean'], 0)
# NaCl.
d = sd_builder.get_site_descriptors_from_struct(
Structure.from_dict(NaCl["structure"]))
self.assertAlmostEqual(d['csf'][0]['octahedral CN_6'], 1)
ds = sd_builder.get_statistics(d)
self.assertAlmostEqual(
ds['csf'][get_index(ds['csf'], 'octahedral CN_6')]['mean'], 1)
self.assertAlmostEqual(
ds['csf'][get_index(ds['csf'], 'octahedral CN_6')]['std'], 0)
# Iron.
d = sd_builder.get_site_descriptors_from_struct(
Structure.from_dict(Fe["structure"]))
self.assertAlmostEqual(d['csf'][0]['body-centered cubic CN_8'],
0.57918, 4)
ds = sd_builder.get_statistics(d)
self.assertAlmostEqual(
ds['csf'][get_index(ds['csf'],
'body-centered cubic CN_8')]['mean'], 0.57918,
4)
self.assertAlmostEqual(
ds['csf'][get_index(ds['csf'], 'body-centered cubic CN_8')]['std'],
0)
def memorystore():
store = MemoryStore()
store.connect()
return store
