def test_load_by_oclass(self): """Test loading by oclass.""" c = city.City(name="Freiburg") p1 = city.Citizen(name="Peter") p2 = city.Citizen(name="Anna") p3 = city.Citizen(name="Julia") c.add(p1, p2, p3, rel=city.hasInhabitant) p1.add(p3, rel=city.hasChild) p2.add(p3, rel=city.hasChild) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add(c) session.commit() with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) cs = wrapper.get(c.uid) r = session.load_by_oclass(city.City) self.assertIs(next(r), cs) r = session.load_by_oclass(city.Citizen) self.assertEqual(set(r), {p1, p2, p3}) r = session.load_by_oclass(city.Person) self.assertEqual(set(r), {p1, p2, p3}) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) cs = wrapper.get(c.uid) r = session.load_by_oclass(city.Street) self.assertRaises(StopIteration, next, r)
def test_init(self): """Test of first level of children are loaded automatically.""" c = city.City(name="Freiburg") p1 = city.Citizen(name="Peter") p2 = city.Citizen(name="Anna") p3 = city.Citizen(name="Julia") c.add(p1, p2, p3, rel=city.hasInhabitant) p1.add(p3, rel=city.hasChild) p2.add(p3, rel=city.hasChild) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add(c) session.commit() with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) self.assertEqual(set(session._registry.keys()), {c.uid, wrapper.uid}) self.assertEqual(wrapper.get(c.uid).name, "Freiburg") self.assertEqual( session._registry.get(c.uid)._neighbors[city.hasInhabitant], { p1.uid: p1.oclasses, p2.uid: p2.oclasses, p3.uid: p3.oclasses }) self.assertEqual( session._registry.get(c.uid)._neighbors[city.isPartOf], {wrapper.uid: wrapper.oclasses})
def test_load_by_iri(self): """Test if it is possible to load objects by their IRIs.""" c = city.City(name="Freiburg") p1 = city.Citizen(name="Peter") p2 = city.Citizen(name="Anna") p3 = city.Citizen(name="Julia") c.add(p1, p2, p3, rel=city.hasInhabitant) p1.add(p3, rel=city.hasChild) p2.add(p3, rel=city.hasChild) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add(c) session.commit() with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) cs = wrapper.get(c.uid) r = session.load_from_iri(cs.iri) self.assertIs(next(r), cs) r = session.load_from_iri(p1.iri, p2.iri, p3.iri) self.assertEqual(set(r), {p1, p2, p3}) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) cs = wrapper.get(c.uid) r = session.load_from_iri(iri_from_uid(uuid.UUID(int=1))) self.assertEqual(set(r), {None})
def test_clear_database(self): """Test clearing the database.""" # db is empty (no error occurs) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) session._clear_database() with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.session.commit() session._clear_database() # db is not empty c = city.City(name="Freiburg") p1 = city.Citizen(name="Peter") p2 = city.Citizen(name="Anna") p3 = city.Citizen(name="Julia") c.add(p1, p2, p3, rel=city.hasInhabitant) p1.add(p3, rel=city.hasChild) p2.add(p3, rel=city.hasChild) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add(c) session.commit() session._clear_database() check_db_cleared(self, DB)
def test_vector(self): """Test capabilities to store vectors.""" c = city.City(name="Frankfurt", coordinates=[42, 24]) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add(c) wrapper.session.commit() with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) cw = wrapper.get(c.uid) np.testing.assert_array_equal(cw.coordinates, [42, 24])
def setUp(self): """Start the timer and fill the database.""" if not RUN_PERFORMANCE_TEST: return self.iterations = 1000 with SqliteSession(DB) as session: w = city.CityWrapper(session=session) self.fill_db(w, random_uid=False) session.commit() self.session = SqliteSession(DB) self.w = city.CityWrapper(session=self.session) gc.collect() self.start = time.time()
def test_sql_list_pattern(self): """Test transformation of value lists to SQLite patterns.""" p, v = SqliteSession._sql_list_pattern("pre", [42, "yo", 1.2, "hey"]) self.assertEqual(p, ":pre_0, :pre_1, :pre_2, :pre_3") self.assertEqual( v, {"pre_0": 42, "pre_1": "yo", "pre_2": 1.2, "pre_3": "hey"} )
def test_refresh(self): """Test refreshing CUDS objects.""" c = city.City(name="Freiburg") p1 = city.Citizen(name="Peter") p2 = city.Citizen(name="Anna") p3 = city.Citizen(name="Julia") c.add(p1, p2, p3, rel=city.hasInhabitant) p1.add(p3, rel=city.hasChild) p2.add(p3, rel=city.hasChild) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) cw = wrapper.add(c) p1w, p2w, p3w = cw.get(p1.uid, p2.uid, p3.uid) session.commit() self.assertEqual(cw.name, "Freiburg") self.assertEqual(p1w.name, "Peter") self.assertEqual(p2w.name, "Anna") self.assertEqual(p3w.name, "Julia") self.assertEqual(session._expired, {wrapper.uid}) update_db(DB, c, p1, p2, p3) session.refresh(cw, p1w, p2w, p3w) self.assertEqual(cw.name, "Paris") self.assertEqual(p1w.name, "Maria") self.assertEqual(set(cw.get()), {p1w}) self.assertEqual(p2w.get(), list()) self.assertFalse(hasattr(p3w, "name")) self.assertNotIn(p3w.uid, session._registry)
def test_expiring(self): """Test expring CUDS objects.""" c = city.City(name="Freiburg") p1 = city.Citizen(name="Peter") p2 = city.Citizen(name="Anna") p3 = city.Citizen(name="Julia") c.add(p1, p2, p3, rel=city.hasInhabitant) p1.add(p3, rel=city.hasChild) p2.add(p3, rel=city.hasChild) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) cw = wrapper.add(c) p1w, p2w, p3w = cw.get(p1.uid, p2.uid, p3.uid) session.commit() # p1w is no longer expired after the following assert self.assertEqual(p1w.name, "Peter") self.assertEqual(p2w.name, "Anna") update_db(DB, c, p1, p2, p3) self.assertEqual(p2w.name, "Anna") self.assertEqual(cw.name, "Paris") # expires outdated neighbor p2w self.assertEqual(p2w.name, "Jacob") self.assertEqual(p1w.name, "Peter") session.expire_all() self.assertEqual(p1w.name, "Maria") self.assertEqual(set(cw.get()), {p1w}) self.assertEqual(p2w.get(), list()) self.assertFalse(hasattr(p3w, "name")) self.assertNotIn(p3w.uid, session._registry)
def test_cuds_with_iri(self): """Try to assign IRIs as UIDs for CUDS objects.""" c = city.City( name="Freiburg", iri="http://example.org/namespace" "#Freiburg" ) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add(c) session.commit() with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) self.assertEqual( set(session._registry.keys()), {c.uid, wrapper.uid} ) self.assertEqual(wrapper.get(c.uid).name, "Freiburg") self.assertEqual(wrapper.get(c.uid).oclass, city.City)
def test_load_missing(self): """Test if missing objects are loaded automatically.""" c = city.City(name="Freiburg") p1 = city.Citizen(name="Peter") p2 = city.Citizen(name="Anna") p3 = city.Citizen(name="Julia") c.add(p1, p2, p3, rel=city.hasInhabitant) p1.add(p3, rel=city.hasChild) p2.add(p3, rel=city.hasChild) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add(c) session.commit() with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) self.assertEqual( set(session._registry.keys()), {c.uid, wrapper.uid} ) cw = wrapper.get(c.uid) p1w = cw.get(p1.uid) p2w = cw.get(p2.uid) p3w = p1w.get(p3.uid) self.assertEqual( set(session._registry.keys()), {c.uid, wrapper.uid, p1.uid, p2.uid, p3.uid}, ) self.assertEqual(p1w.name, "Peter") self.assertEqual(p2w.name, "Anna") self.assertEqual(p3w.name, "Julia") self.assertEqual( p3w._neighbors[city.isChildOf], {p1.uid: p1.oclasses, p2.uid: p2.oclasses}, ) self.assertEqual( p2w._neighbors[city.hasChild], {p3.uid: p3.oclasses} ) self.assertEqual( p2w._neighbors[city.INVERSE_OF_hasInhabitant], {c.uid: c.oclasses}, )
def test_multiple_users(self): """Test what happens if multiple users access the database.""" with SqliteSession(DB) as session1: wrapper1 = city.CityWrapper(session=session1) city1 = city.City(name="Freiburg") wrapper1.add(city1) session1.commit() with SqliteSession(DB) as session2: wrapper2 = city.CityWrapper(session=session2) wrapper2.add(city.City(name="Offenburg")) session2.commit() cw = wrapper1.add(city.City(name="Karlsruhe")) self.assertEqual(session1._expired, {city1.uid}) self.assertEqual(session1._buffers, [[{ cw.uid: cw }, { wrapper1.uid: wrapper1 }, dict()], [dict(), dict(), dict()]]) session1.commit()
def run_migration(self, schema_version): """Load the data from the given schema version + run the migration.""" filename = f"sqlite_schema_v{schema_version}.sql" with open(Path(__file__).parent / filename, encoding="utf-8") as f: with sqlite3.connect(DB) as con: con.executescript(f.read()) self.assertRaises(RuntimeError, check_supported_schema_version, SqliteSession(DB)) self.assertRaises(RuntimeError, city.CityWrapper, session=SqliteSession(DB)) self.assertEqual( detect_current_schema_version( SqliteSession(DB)._get_table_names("")), schema_version, ) with SqliteSession(DB) as session: m = SqlMigrate(session) m.run() self.assertEqual( detect_current_schema_version( SqliteSession(DB)._get_table_names("")), max(versions.values()), ) self.assertTrue(check_supported_schema_version(SqliteSession(DB)))
def test_insert(self): """Test inserting in the sqlite table.""" c = city.City(name="Freiburg") p1 = city.Citizen(name="Peter") p2 = city.Citizen(name="Georg") c.add(p1, p2, rel=city.hasInhabitant) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add(c) wrapper.session.commit() check_state(self, c, p1, p2)
def test_update(self): """Test updating the sqlite table.""" c = city.City(name="Paris") p1 = city.Citizen(name="Peter") c.add(p1, rel=city.hasInhabitant) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) cw = wrapper.add(c) session.commit() p2 = city.Citizen(name="Georg") cw.add(p2, rel=city.hasInhabitant) cw.name = "Freiburg" session.commit() check_state(self, c, p1, p2)
def install_from_terminal(): """Migrate sqlite databases from terminal.""" # Parse the user arguments parser = argparse.ArgumentParser( description="Migrate your sqlite database." ) parser.add_argument( "db_file", type=str, help="The path to the sqlite database file." ) args = parser.parse_args() if not os.path.exists(args.db_file): raise FileNotFoundError(args.db_file) with SqliteSession(args.db_file) as session: m = SqlMigrate(session) m.run()
def test_delete(self): """Test to delete cuds_objects from the sqlite table.""" c = city.City(name="Freiburg") p1 = city.Citizen(name="Peter") p2 = city.Citizen(name="Georg") p3 = city.Citizen(name="Hans") c.add(p1, p2, p3, rel=city.hasInhabitant) with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) cw = wrapper.add(c) session.commit() cw.remove(p3.uid) session._notify_read(wrapper) session.prune() session.commit() check_state(self, c, p1, p2)
def check_db_cleared(test_case, db_file): """Check whether the database has been cleared successfully.""" with sqlite3.connect(db_file) as conn: cursor = conn.cursor() cursor.execute(f"SELECT * FROM {CUDS_TABLE};") test_case.assertEqual(list(cursor), list()) cursor.execute(f"SELECT * FROM {ENTITIES_TABLE};") test_case.assertEqual(list(cursor), list()) cursor.execute(f"SELECT * FROM {TYPES_TABLE};") test_case.assertEqual(list(cursor), list()) cursor.execute(f"SELECT * FROM {NAMESPACES_TABLE};") test_case.assertEqual(list(cursor), list()) cursor.execute(f"SELECT * FROM {RELATIONSHIP_TABLE};") test_case.assertEqual(list(cursor), list()) # DATA TABLES with SqliteSession(DB) as s: table_names = s._get_table_names(DATA_TABLE_PREFIX) for table_name in table_names: cursor.execute(f"SELECT * FROM `{table_name}`;") test_case.assertEqual(list(cursor), list())
def run_test(self): """Test whether all data can be loaded correctly.""" with SqliteSession(DB) as session: w = city.CityWrapper(session=session) cities = w.get(rel=city.hasPart) c = cities[0] self.assertEqual(len(cities), 1) self.assertTrue(c.is_a(city.City)) self.assertEqual(c.name, "Freiburg") self.assertEqual(c.uid.hex, "affb72ee61754028bd7e39a92ba3bb77") self.assertEqual(c.get(rel=city.isPartOf), [w]) np.testing.assert_equal(c.coordinates, np.array([42, 12])) neighborhoods = c.get(oclass=city.Neighborhood) n = neighborhoods[0] self.assertEqual(len(neighborhoods), 1) self.assertEqual(n.uid.hex, "e30e0287f52b49f396b939a85fc9460d") self.assertEqual(n.name, "Zähringen") self.assertEqual(n.get(rel=city.isPartOf), [c]) np.testing.assert_equal(n.coordinates, np.array([0, 0])) streets = n.get() s = streets[0] self.assertEqual(len(streets), 1) self.assertEqual(s.uid.hex, "25cb6116e9d04ceb81cdd8cfcbead47b") self.assertEqual(s.name, "Le street") self.assertEqual(s.get(rel=city.isPartOf), [n]) np.testing.assert_equal(s.coordinates, np.array([1, 98])) citizen = c.get(rel=city.hasInhabitant) citizen = sorted(citizen, key=lambda p: p.name) self.assertEqual([p.name for p in citizen], ["Hans", "Michel", "Peter"]) for p in citizen: self.assertEqual(p.get(rel=city.hasInhabitant.inverse), [c]) self.assertEqual(p.get(), [])
def __init__(self, db_file): """Initialize the migration tool with an SqliteSession.""" super().__init__(SqliteSession(db_file))
def test_big_dataset_warning(self): """Tests the warnings emitted when committing big datasets.""" # Save the original warning settings to restore them at the end of # the test. original_warning_setting = osp.core.warnings.unreachable_cuds_objects original_large_dataset_size = ( osp.core.warnings.unreachable_cuds_objects_large_dataset_size ) def count_warnings_by_class( records: Iterable[logging.LogRecord], classes: Union[Type, Tuple[Type, ...]], ) -> int: """Given log records, count their "classes" if attached. For each record, checks if it has a `warning_class` attribute, and checks whether its value is a subclass of the classes provided. """ return sum( bool( issubclass(record.warning_class, classes) if hasattr(record, "warning_class") else False ) for record in records ) osp.core.warnings.unreachable_cuds_objects_large_dataset_size = 5 osp.core.warnings.unreachable_cuds_objects = True large_dataset_size = ( osp.core.warnings.unreachable_cuds_objects_large_dataset_size ) try: # No warning: small dataset, all CUDS reachable with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add( city.Citizen(name="citizen"), rel=city.hasInhabitant ) with self.assertLogs(logger=logger) as captured: logger.warning( "At least log entry is needed for " "`assertLogs`." ) session.commit() self.assertEqual( count_warnings_by_class( captured.records, (LargeDatasetWarning, UnreachableCUDSWarning), ), 0, ) if os.path.exists(DB): os.remove(DB) # Unreachable CUDS warning: small dataset, some CUDS not reachable with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add( city.Citizen(name="citizen"), rel=city.hasInhabitant ) city.Citizen(name="citizen") with self.assertLogs(logger=logger) as captured: logger.warning( "At least log entry is needed for " "`assertLogs`." ) session.commit() self.assertEqual( count_warnings_by_class( captured.records, LargeDatasetWarning ), 0, ) self.assertEqual( count_warnings_by_class( captured.records, UnreachableCUDSWarning ), 1, ) if os.path.exists(DB): os.remove(DB) # Large dataset warning: large dataset, all CUDS reachable with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add( *[ city.Citizen(name="citizen") for _ in range(0, large_dataset_size + 1) ], rel=city.hasInhabitant, ) with self.assertLogs(logger=logger) as captured: logger.warning( "At least log entry is needed for " "`assertLogs`." ) session.commit() self.assertEqual( count_warnings_by_class( captured.records, LargeDatasetWarning ), 1, ) self.assertEqual( count_warnings_by_class( captured.records, UnreachableCUDSWarning ), 0, ) if os.path.exists(DB): os.remove(DB) # Both warnings: large dataset, some CUDS not reachable with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add( *[ city.Citizen(name="citizen") for _ in range(0, large_dataset_size + 1) ], rel=city.hasInhabitant, ) city.Citizen(name="citizen") with self.assertLogs(logger=logger) as captured: logger.warning( "At least log entry is needed for " "`assertLogs`." ) session.commit() self.assertEqual( count_warnings_by_class( captured.records, LargeDatasetWarning ), 1, ) self.assertEqual( count_warnings_by_class( captured.records, UnreachableCUDSWarning ), 1, ) if os.path.exists(DB): os.remove(DB) # Edge case: large dataset warning after unreachable warning with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add( city.Citizen(name="citizen"), rel=city.hasInhabitant ) [ city.Citizen(name="citizen") for i in range(0, large_dataset_size) ] with self.assertLogs(logger=logger) as captured: logger.warning( "At least log entry is needed for " "`assertLogs`." ) session.commit() self.assertEqual( count_warnings_by_class( captured.records, LargeDatasetWarning ), 1, ) self.assertEqual( count_warnings_by_class( captured.records, UnreachableCUDSWarning ), 1, ) large_dataset_warning = set( record for record in captured.records if hasattr(record, "warning_class") if issubclass( record.warning_class, LargeDatasetWarning ) ).pop() self.assertIn( "the previous warning", large_dataset_warning.msg ) if os.path.exists(DB): os.remove(DB) # Test warning turned off for the both warnings case osp.core.warnings.unreachable_cuds_objects = False with SqliteSession(DB) as session: wrapper = city.CityWrapper(session=session) wrapper.add( *[ city.Citizen(name="citizen") for _ in range(0, large_dataset_size + 1) ], rel=city.hasInhabitant, ) city.Citizen(name="citizen") with self.assertLogs(logger=logger) as captured: logger.warning( "At least log entry is needed for " "`assertLogs`." ) session.commit() self.assertEqual( count_warnings_by_class( captured.records, (LargeDatasetWarning, UnreachableCUDSWarning), ), 0, ) if os.path.exists(DB): os.remove(DB) finally: osp.core.warnings.unreachable_cuds_objects = ( original_warning_setting ) osp.core.warnings.unreachable_cuds_objects_large_dataset_size = ( original_large_dataset_size )
class TestPerformance(unittest.TestCase): """Test the performance of the SQLite Wrapper.""" def setUp(self): """Start the timer and fill the database.""" if not RUN_PERFORMANCE_TEST: return self.iterations = 1000 with SqliteSession(DB) as session: w = city.CityWrapper(session=session) self.fill_db(w, random_uid=False) session.commit() self.session = SqliteSession(DB) self.w = city.CityWrapper(session=self.session) gc.collect() self.start = time.time() def tearDown(self): """Remove database file and print the performance of the test.""" if not RUN_PERFORMANCE_TEST: return self.stop = time.time() total = self.stop - self.start if total > 60: print("Total time: " + str(total / 60) + " minutes.") else: print("Total time: " + str(total) + " seconds.") self.session.close() self.w._session = None gc.collect() if os.path.exists(DB): os.remove(DB) def fill_db(self, c, random_uid=True): """Fill the database with data.""" for i in range(self.iterations): j = i * 9 uids = iter([None for i in range(9)]) if not random_uid: uids = iter(range(j * 9 + 1, (j + 1) * 9 + 1)) c.add(city.Citizen(uid=next(uids)), rel=city.hasInhabitant) c.add(city.Citizen(uid=next(uids)), rel=city.encloses) c.add(city.Citizen(uid=next(uids)), rel=city.hasPart) c.add( city.Neighborhood(name="", uid=next(uids)), rel=city.hasInhabitant, ) c.add(city.Neighborhood(name="", uid=next(uids)), rel=city.encloses) c.add(city.Neighborhood(name="", uid=next(uids)), rel=city.hasPart) c.add(city.Street(name="", uid=next(uids)), rel=city.hasInhabitant) c.add(city.Street(name="", uid=next(uids)), rel=city.encloses) c = c.add(city.Street(name="", uid=next(uids)), rel=city.hasPart) def test_fill_db_one_commit(self): """Tests filling the db with lots of data with one commit.""" if not RUN_PERFORMANCE_TEST: return print("Test filling database one commit") self.fill_db(self.w) def test_graph_walk(self): """Traverse the graph.""" if not RUN_PERFORMANCE_TEST: return print("Traverse db") for i in range(self.iterations): find_cuds_object( lambda x: True, self.w, rel=city.encloses, find_all=True, max_depth=10, ) self.session.commit()
), city.Neighborhood(name="Stühlinger"), city.Neighborhood(name="Herdern"), ) # Export from Core Session export_cuds(path="test.rdf", format="ttl") # Check output with open("test.rdf", encoding="utf-8") as f: print("Exported from Core Session") for line in f: print("\t", line.strip()) # Export from a Wrapper session with SqliteSession(path="test.db") as session: w = city.CityWrapper(session=session) w.add(c) export_cuds(session, path="test.rdf", format="ttl") # Check output with open("test.rdf", encoding="utf-8") as f: print("Exported from SqliteSession") for line in f: print("\t", line.strip()) # Usually, RDF data does not contain UUIDs as in the current output. # Replace UUIDs in the RDF file. # THIS IS JUST FOR DEMONSTRATION PURPOSES. # THIS ALLOWS US TO SHOW THAT OSP-CORE CAN IMPORT ANY RDF DATA, # AND NOT ONLY DATA THAT WAS PREVIOUSLY EXPORTED BY OSP CORE!
}, ) return self.entities[entity_iri] def get_col_spec_0(self, oclass): """Get the columns specification of CUDS tables in schema v0.""" attributes = list(oclass.attributes) columns = [x.argname for x in attributes] + ["uid"] datatypes = dict(uid="UID", **{x.argname: x.datatype for x in attributes}) return attributes, columns, datatypes def delete_old_tables_0(self): """Delete the old tables of v0.""" for table in self.tables: self.session._do_db_drop(table) def no_migration(self): """Do nothing.""" procedures = {0: {0: no_migration, 1: migrate_0_1}} if __name__ == "__main__": from osp.wrappers.sqlite import SqliteSession session = SqliteSession("test.db") m = SqlMigrate(session) m.run()