def setUp(self): self.materials = JSONStore(test_mats, lu_type='isoformat') self.bonding = MemoryStore("bonding")
def validation_store(tasks_store): validation_store = MemoryStore() builder = TaskValidator(tasks=tasks_store, task_validation=validation_store) builder.run() return validation_store
def test_serialization(tmpdir): builder = MaterialsBuilder(MemoryStore(), MemoryStore(), MemoryStore()) dumpfn(builder.as_dict(), Path(tmpdir) / "test.json") loadfn(Path(tmpdir) / "test.json")
def sandbox_store(): memstore = MemoryStore() store = SandboxStore(memstore, sandbox="test") store.connect() return store
def owner_store(): store = MemoryStore("owners", key="name") store.connect() store.update([d.dict() for d in owners]) return store
def source(): store = MemoryStore("source", key="k", last_updated_field="lu") store.connect() store.ensure_index("k") store.ensure_index("lu") return store
def setUp(self): # Set up test db, set up mpsft, etc. self.test_tasks = JSONStore([test_tasks]) self.elasticity = MemoryStore("atomate_tasks") self.test_tasks.connect() self.elasticity.connect()
def setUp(self): self.materials = JSONStore(test_mats, lu_type='isoformat') self.materials.connect() self.thermo = MemoryStore("thermo") self.thermo.connect()
def test_process_item(self): dt = MemoryStore() at = MemoryStore() qc = [{ 'catalog': 'icsd', 'k': 100, 'exclude': ['compound'], 'filter': [('auid', '__eq__', 'aflow:0132ab6b9cddd429')], 'select': [], 'targets': ['data', 'auid'] }] kw = ['auid', 'aurl', 'ael_elastic_anisotropy', 'files'] afi = AflowIngester(data_target=dt, auid_target=at, keywords=kw, query_configs=qc) item = afi.get_items().__next__() processed = afi.process_item(item) self.assertIsInstance(processed, tuple) self.assertEqual(len(processed), 2) data, auid = processed expected_data_keys = kw + [ 'AEL_elastic_tensor_json', 'CONTCAR_relax_vasp' ] expected_auid_keys = ['auid', 'aurl', 'compound'] self.assertCountEqual(list(data.keys()), expected_data_keys) self.assertTrue(all(v is not None for v in data.values())) self.assertCountEqual(list(auid.keys()), expected_auid_keys) self.assertIsNone(auid['compound']) # Omit auid target afi = AflowIngester(data_target=dt, keywords=kw, query_configs=qc) item = afi.get_items().__next__() processed = afi.process_item(item) self.assertIsInstance(processed, tuple) self.assertEqual(len(processed), 2) _, auid = processed self.assertEqual(auid, dict()) # Test for filtering null properties qc = [{ 'catalog': 'lib1', 'k': 100, 'exclude': ['compound'], 'filter': [('auid', '__eq__', 'aflow:7203c28b8396b9c9')], 'select': [], 'targets': ['data', 'auid'] }] afi = AflowIngester(data_target=dt, auid_target=at, keywords=kw, query_configs=qc, filter_null_properties=True) item = afi.get_items().__next__() processed = afi.process_item(item) self.assertIsInstance(processed, tuple) self.assertEqual(len(processed), 2) data, auid = processed self.assertNotIn('ael_elastic_anisotropy', data) self.assertNotIn('AEL_elastic_tensor_json', data) self.assertNotIn('compound', auid)
self.logger.info("processing item: {}".format(item)) # time.sleep(random.randint(0,3)) return {item: "processed"} def update_targets(self, items): self.logger.info("Updating targets ...") self.logger.info("Received {} processed items".format(len(items))) self.logger.info("Updated items: {}".format(list(items))) def finalize(self, cursor): self.logger.info("Finalizing ...") self.logger.info("DONE!") if __name__ == '__main__': N = 10 chunk_size = 3 stores = [MemoryStore(str(i)) for i in range(7)] sources = [stores[0], stores[1], stores[3]] targets = [stores[3], stores[6]] mdb = MyDumbBuilder(N, sources, targets, chunk_size=chunk_size) builders = [mdb] runner = Runner(builders) logstreamhandle(runner) runner.run()
class TestThermo(unittest.TestCase): def setUp(self): self.materials = JSONStore(test_mats, lu_type='isoformat') self.materials.connect() self.thermo = MemoryStore("thermo") self.thermo.connect() def test_get_entries(self): tbuilder = ThermoBuilder(self.materials, self.thermo) self.assertEqual(len(tbuilder.get_entries("Sr")), 7) self.assertEqual(len(tbuilder.get_entries("Hf")), 4) self.assertEqual(len(tbuilder.get_entries("O")), 6) self.assertEqual(len(tbuilder.get_entries("Hf-O-Sr")), 44) self.assertEqual(len(tbuilder.get_entries("Sr-Hf")), 11) def test_chemsys_permutations(self): self.assertEqual(len(chemsys_permutations("Sr")), 1) self.assertEqual(len(chemsys_permutations("Sr-Hf")), 3) self.assertEqual(len(chemsys_permutations("Sr-Hf-O")), 7) def test_process_items(self): tbuilder = ThermoBuilder(self.materials, self.thermo) # Ensure only one doc gets a 0 e_above_hull entries = tbuilder.get_entries("Sr") t_docs = tbuilder.process_item(("core", entries)) e_above_hulls = [t['thermo']['e_above_hull'] for t in t_docs] self.assertEqual(len([e for e in e_above_hulls if e == 0.0]), 1) entries = tbuilder.get_entries("Hf") t_docs = tbuilder.process_item(("core", entries)) e_above_hulls = [t['thermo']['e_above_hull'] for t in t_docs] self.assertEqual(len([e for e in e_above_hulls if e == 0.0]), 1) entries = tbuilder.get_entries("O") t_docs = tbuilder.process_item(("core", entries)) e_above_hulls = [t['thermo']['e_above_hull'] for t in t_docs] self.assertEqual(len([e for e in e_above_hulls if e == 0.0]), 1) # Ensure 4 docs iwth 0 e_above hull for convex hull for Sr-O entries = tbuilder.get_entries("Sr-O") t_docs = tbuilder.process_item(("core", entries)) e_above_hulls = [t['thermo']['e_above_hull'] for t in t_docs] self.assertEqual(len([e for e in e_above_hulls if e == 0.0]), 4) # Ensure 4 docs iwth 0 e_above hull for convex hull Hf-O entries = tbuilder.get_entries("Hf-O") t_docs = tbuilder.process_item(("core", entries)) e_above_hulls = [t['thermo']['e_above_hull'] for t in t_docs] self.assertEqual(len([e for e in e_above_hulls if e == 0.0]), 3) # Ensure 4 docs iwth 0 e_above hull for convex hull entries = tbuilder.get_entries("Sr-Hf-O") t_docs = tbuilder.process_item(("core", entries)) e_above_hulls = [t['thermo']['e_above_hull'] for t in t_docs] self.assertEqual(len(e_above_hulls), 44) self.assertEqual(len([e for e in e_above_hulls if e == 0.0]), 7) def test_update_targets(self): items = [[{ "task_id": 1, "_sbxn": ["core"] }] * 3, [{ "task_id": 2, "_sbxn": ["core"] }] * 4, [{ "task_id": 3, "_sbxn": ["core"] }] * 4] tbuilder = ThermoBuilder(self.materials, self.thermo) tbuilder.update_targets(items) self.assertEqual(len(self.thermo.distinct("task_id")), 3) self.assertEqual(tbuilder.completed_tasks, {1, 2, 3}) def test_get_items(self): tbuilder = ThermoBuilder(self.materials, self.thermo) self.assertEqual(len(list(tbuilder.get_items())), 1)
def setUp(self): self.materials = JSONStore(test_mats) self.materials.connect() self.thermo = MemoryStore("thermo") self.thermo.connect()
def setUp(self): self.propstore = MemoryStore() self.propstore.connect()
class MPBuilderTest(unittest.TestCase): @classmethod def setUpClass(cls): add_builtin_models_to_registry() cls.materials = MemoryStore() cls.materials.connect() materials = loadfn(os.path.join(TEST_DATA_DIR, "test_materials.json")) materials = jsanitize(materials, strict=True, allow_bson=True) cls.materials.update(materials) cls.propstore = None def setUp(self): self.propstore = MemoryStore() self.propstore.connect() def test_serial_runner(self): builder = PropnetBuilder(self.materials, self.propstore) runner = Runner([builder]) runner.run() def test_multiproc_runner(self): builder = PropnetBuilder(self.materials, self.propstore) runner = Runner([builder], max_workers=2) runner.run() def test_process_item(self): item = self.materials.query_one(criteria={"pretty_formula": "Cs"}) builder = PropnetBuilder(self.materials, self.propstore) processed = builder.process(item) self.assertIsNotNone(processed) # Ensure vickers hardness gets populated self.assertIn("vickers_hardness", processed) if 'created_at' in item.keys(): date_value = item['created_at'] else: date_value = "" # Check that provenance values propagate correctly current_quantity = processed['vickers_hardness']['quantities'][0] at_deepest_level = False while not at_deepest_level: current_provenance = current_quantity['provenance'] if current_provenance['inputs'] is not None: self.assertEqual(current_provenance['source']['source'], "propnet") self.assertEqual(current_provenance['source']['source_key'], current_quantity['internal_id']) self.assertNotIn(current_provenance['source']['date_created'], ("", None)) current_quantity = current_provenance['inputs'][0] else: self.assertEqual(current_provenance['source']['source'], "Materials Project") self.assertEqual(current_provenance['source']['source_key'], item['task_id']) self.assertEqual(current_provenance['source']['date_created'], date_value) at_deepest_level = True def test_deprecated_filter(self): # Check default (False) builder = PropnetBuilder(self.materials, self.propstore) self.assertFalse(builder.include_deprecated) # Check False explicitly builder = PropnetBuilder(self.materials, self.propstore, include_deprecated=False) builder.connect() is_deprecated = [item['deprecated'] for item in builder.get_items()] self.assertTrue(all(not v for v in is_deprecated)) # Check True explicitly builder = PropnetBuilder(self.materials, self.propstore, include_deprecated=True) builder.connect() is_deprecated = [item['deprecated'] for item in builder.get_items()] self.assertTrue(any(is_deprecated)) def test_sandboxed_filter(self): # Check default (False) builder = PropnetBuilder(self.materials, self.propstore) self.assertFalse(builder.include_sandboxed) # Check False explicitly builder = PropnetBuilder(self.materials, self.propstore, include_sandboxed=False) builder.connect() is_in_core = ['core' in item['sbxn'] for item in builder.get_items()] self.assertTrue(all(v for v in is_in_core)) # Check True explicitly builder = PropnetBuilder(self.materials, self.propstore, include_sandboxed=True) builder.connect() is_not_in_core = [ 'core' not in item['sbxn'] for item in builder.get_items() ] self.assertTrue(any(is_not_in_core))
for i in range(1000): yield {"val": i, "task_id": i} def process_item(self, item): if item["val"] % 10 == 0: self.logger.debug("Processing: {}".format(item["val"])) proc_val = np.sqrt(np.square(float(item["val"]))) item["proc_val"] = proc_val return item def update_targets(self, items): self.logger.info("Updating {} items".format(len(items))) self.temp_storage_store.update(items) if __name__ == "__main__": root = logging.getLogger() root.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) formatter = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s - %(message)s") ch.setFormatter(formatter) root.addHandler(ch) mem = MemoryStore("processed") bldr = DummyBuilder(mem) mpi_proc = MPIProcessor([bldr]) mpi_proc.process(0)
def owner_store(): store = MemoryStore("owners", key="name") store.connect() store.update([jsonable_encoder(d) for d in owners]) return store
def setUp(self): # Set up test db, set up mpsft, etc. self.test_tasks = JSONStore(test_tasks) self.task_types = MemoryStore("task_types") self.test_tasks.connect() self.task_types.connect()
def pet_store(): store = MemoryStore("pets", key="name") store.connect() store.update([jsonable_encoder(d) for d in pets]) return store
def target(): store = MemoryStore("target", key="k", last_updated_field="lu") store.connect() store.ensure_index("k") store.ensure_index("lu") return store
def search_helper(payload, base: str = "/?", debug=True) -> Tuple[Response, Any]: """ Helper function to directly query search endpoints Args: store: store f base: base of the query, default to /query? client: TestClient generated from FastAPI payload: query in dictionary format debug: True = print out the url, false don't print anything Returns: request.Response object that contains the response of the correspoding payload """ owner_store = MemoryStore("owners", key="name") owner_store.connect() owner_store.update([d.dict() for d in owners]) pets_store = MemoryStore("pets", key="name") pets_store.connect() pets_store.update([jsonable_encoder(d) for d in pets]) resources = { "owners": [ ReadOnlyResource( owner_store, Owner, query_operators=[ StringQueryOperator(model=Owner), # type: ignore NumericQuery(model=Owner), # type: ignore SparseFieldsQuery(model=Owner), PaginationQuery(), ], ) ], "pets": [ ReadOnlyResource( pets_store, Owner, query_operators=[ StringQueryOperator(model=Pet), NumericQuery(model=Pet), SparseFieldsQuery(model=Pet), PaginationQuery(), ], ) ], } api = API(resources=resources) client = TestClient(api.app) url = base + urlencode(payload) if debug: print(url) res = client.get(url) try: data = res.json().get("data", []) except Exception: data = res.reason return res, data
class BuilderTest(unittest.TestCase): def setUp(self): self.materials = MemoryStore() self.materials.connect() materials = loadfn(os.path.join(TEST_DIR, "test_materials.json")) materials = jsanitize(materials, strict=True, allow_bson=True) self.materials.update(materials) self.propstore = MemoryStore() self.propstore.connect() def test_serial_runner(self): builder = PropnetBuilder(self.materials, self.propstore) runner = Runner([builder]) runner.run() def test_multiproc_runner(self): builder = PropnetBuilder(self.materials, self.propstore) runner = Runner([builder]) runner.run() def test_process_item(self): item = self.materials.query_one(criteria={"pretty_formula": "Cs"}) builder = PropnetBuilder(self.materials, self.propstore) processed = builder.process_item(item) self.assertIsNotNone(processed) # Ensure vickers hardness gets populated self.assertIn("vickers_hardness", processed) if 'created_at' in item.keys(): date_value = item['created_at'] else: date_value = "" # Check that provenance values propagate correctly current_quantity = processed['vickers_hardness']['quantities'][0] at_deepest_level = False while not at_deepest_level: current_provenance = current_quantity['provenance'] if current_provenance['inputs'] is not None: self.assertEqual(current_provenance['source']['source'], "propnet") self.assertEqual(current_provenance['source']['source_key'], current_quantity['internal_id']) self.assertNotIn(current_provenance['source']['date_created'], ("", None)) current_quantity = current_provenance['inputs'][0] else: self.assertEqual(current_provenance['source']['source'], "Materials Project") self.assertEqual(current_provenance['source']['source_key'], item['task_id']) self.assertEqual(current_provenance['source']['date_created'], date_value) at_deepest_level = True # @unittest.skipIf(not os.path.isfile("runner.json"), "No runner file") # def test_runner_pipeline(self): # from monty.serialization import loadfn # runner = loadfn("runner.json") # runner.builders[0].connect() # items = list(runner.builders[0].get_items()) # processed = runner.builders[0].process_item(items[0]) # runner.run() # Just here for reference, in case anyone wants to create a new set # of test materials -jhm @unittest.skipIf(True, "Skipping test materials creation") def create_test_docs(self): formulas = ["BaNiO3", "Si", "Fe2O3", "Cs"] from maggma.advanced_stores import MongograntStore from monty.serialization import dumpfn mgstore = MongograntStore("ro:matgen2.lbl.gov/mp_prod", "materials") builder = PropnetBuilder( mgstore, self.propstore, criteria={"pretty_formula": {"$in": formulas}, "e_above_hull": 0}) builder.connect() dumpfn(list(builder.get_items()), "test_materials.json")
class TestAliasingStore(unittest.TestCase): def setUp(self): self.memorystore = MemoryStore("test") self.memorystore.connect() self.aliasingstore = AliasingStore(self.memorystore, { "a": "b", "c.d": "e", "f": "g.h" }) def test_query(self): d = [{"b": 1}, {"e": 2}, {"g": {"h": 3}}] self.memorystore.collection.insert_many(d) self.assertTrue("a" in list( self.aliasingstore.query(criteria={"a": { "$exists": 1 }}))[0]) self.assertTrue("c" in list( self.aliasingstore.query(criteria={"c.d": { "$exists": 1 }}))[0]) self.assertTrue("d" in list( self.aliasingstore.query(criteria={"c.d": { "$exists": 1 }}))[0].get("c", {})) self.assertTrue("f" in list( self.aliasingstore.query(criteria={"f": { "$exists": 1 }}))[0]) def test_update(self): self.aliasingstore.update([{ "task_id": "mp-3", "a": 4 }, { "task_id": "mp-4", "c": { "d": 5 } }, { "task_id": "mp-5", "f": 6 }]) self.assertEqual( list(self.aliasingstore.query( criteria={"task_id": "mp-3"}))[0]["a"], 4) self.assertEqual( list(self.aliasingstore.query( criteria={"task_id": "mp-4"}))[0]["c"]["d"], 5) self.assertEqual( list(self.aliasingstore.query( criteria={"task_id": "mp-5"}))[0]["f"], 6) self.assertEqual( list(self.aliasingstore.store.query( criteria={"task_id": "mp-3"}))[0]["b"], 4) self.assertEqual( list(self.aliasingstore.store.query( criteria={"task_id": "mp-4"}))[0]["e"], 5) self.assertEqual( list(self.aliasingstore.store.query( criteria={"task_id": "mp-5"}))[0]["g"]["h"], 6) def test_substitute(self): aliases = {"a": "b", "c.d": "e", "f": "g.h"} d = {"b": 1} substitute(d, aliases) self.assertTrue("a" in d) d = {"e": 1} substitute(d, aliases) self.assertTrue("c" in d) self.assertTrue("d" in d.get("c", {})) d = {"g": {"h": 4}} substitute(d, aliases) self.assertTrue("f" in d) d = None substitute(d, aliases) self.assertTrue(d is None)
def test_bad_import(mocker): mocker.patch("maggma.stores.aws.boto3", None) with pytest.raises(RuntimeError): index = MemoryStore("index'") S3Store(index, "bucket1")
def setUp(self): self.store = MemoryStore() self.sandboxstore = SandboxStore(self.store, sandbox="test")
def validation_store(): return MemoryStore()
def setUp(self): self.materials = JSONStore(test_mats) self.bonding = MemoryStore("bonding")
def materials_store(): return MemoryStore()
def setUp(self): self.materials = JSONStore(test_mats) self.bond_valence = MemoryStore("bond_valence")
def test_get_all_basic_descriptors(self): test_basic_descriptors = MemoryStore("test_basic_descriptors") sd_builder = BasicDescriptorsBuilder(self.test_materials, test_basic_descriptors) C = self.test_materials.query_one(criteria={"task_id": "mp-66"}) NaCl = self.test_materials.query_one(criteria={"task_id": "mp-22862"}) Fe = self.test_materials.query_one(criteria={"task_id": "mp-13"}) # Diamond. d = sd_builder.get_site_descriptors_from_struct( Structure.from_dict(C["structure"])) for di in d.values(): self.assertEqual(len(di), 2) self.assertEqual(d['cn_VoronoiNN'][0]['CN_VoronoiNN'], 20) self.assertAlmostEqual(d['cn_wt_VoronoiNN'][0]['CN_VoronoiNN'], 4.5381162) self.assertEqual(d['cn_JMolNN'][0]['CN_JMolNN'], 4) self.assertAlmostEqual(d['cn_wt_JMolNN'][0]['CN_JMolNN'], 4.9617398) self.assertEqual(d['cn_MinimumDistanceNN'][0]['CN_MinimumDistanceNN'], 4) self.assertAlmostEqual( d['cn_wt_MinimumDistanceNN'][0]['CN_MinimumDistanceNN'], 4) self.assertEqual(d['cn_MinimumOKeeffeNN'][0]['CN_MinimumOKeeffeNN'], 4) self.assertAlmostEqual( d['cn_wt_MinimumOKeeffeNN'][0]['CN_MinimumOKeeffeNN'], 4) self.assertEqual(d['cn_MinimumVIRENN'][0]['CN_MinimumVIRENN'], 4) self.assertAlmostEqual(d['cn_wt_MinimumVIRENN'][0]['CN_MinimumVIRENN'], 4) self.assertEqual( d['cn_BrunnerNN_reciprocal'][0]['CN_BrunnerNN_reciprocal'], 4) self.assertAlmostEqual( d['cn_wt_BrunnerNN_reciprocal'][0]['CN_BrunnerNN_reciprocal'], 4) # Current value for the below quantity is 0.9618085 self.assertTrue(0.95 <= d['csf'][0]['tetrahedral CN_4'] <= 1) ds = sd_builder.get_statistics(d) self.assertTrue('csf' in list(ds.keys())) for k, dsk in ds.items(): for di in dsk: self.assertEqual(len(list(di.keys())), 3) def get_index(li, optype): try: return next(i for i, d in enumerate(li) if d['name'] == optype) except StopIteration: raise RuntimeError('did not find optype {}'.format(optype)) # Current value for the three below quantities is 0.9618085 self.assertTrue(0.95 <= ds['csf'][get_index( ds['csf'], 'tetrahedral CN_4')]['mean'] <= 1) self.assertAlmostEqual( ds['csf'][get_index(ds['csf'], 'tetrahedral CN_4')]['std'], 0) self.assertAlmostEqual( ds['csf'][get_index(ds['csf'], 'octahedral CN_6')]['mean'], 0) # NaCl. d = sd_builder.get_site_descriptors_from_struct( Structure.from_dict(NaCl["structure"])) self.assertAlmostEqual(d['csf'][0]['octahedral CN_6'], 1) ds = sd_builder.get_statistics(d) self.assertAlmostEqual( ds['csf'][get_index(ds['csf'], 'octahedral CN_6')]['mean'], 1) self.assertAlmostEqual( ds['csf'][get_index(ds['csf'], 'octahedral CN_6')]['std'], 0) # Iron. d = sd_builder.get_site_descriptors_from_struct( Structure.from_dict(Fe["structure"])) self.assertAlmostEqual(d['csf'][0]['body-centered cubic CN_8'], 0.57918, 4) ds = sd_builder.get_statistics(d) self.assertAlmostEqual( ds['csf'][get_index(ds['csf'], 'body-centered cubic CN_8')]['mean'], 0.57918, 4) self.assertAlmostEqual( ds['csf'][get_index(ds['csf'], 'body-centered cubic CN_8')]['std'], 0)
def memorystore(): store = MemoryStore() store.connect() return store