def diff_dbs(db1, db2, user=None): """ 1. new objects => mark for insert 2. deleted objects, no change locally after delete date => mark for deletion 3. deleted objects, change locally => mark for user confirm for deletion 4. updated objects => do a diff on differences, mark origin values as new data """ if user is None: user = User() missing_from_old = [] missing_from_new = [] diffs = [] with user.progress(_('Family Tree Differences'), _('Searching...'), 10) as step: for item in [ 'Person', 'Family', 'Source', 'Citation', 'Event', 'Media', 'Place', 'Repository', 'Note', 'Tag' ]: step() handles1 = sorted([ handle.decode('utf-8') for handle in db1._tables[item]["handles_func"]() ]) handles2 = sorted([ handle.decode('utf-8') for handle in db2._tables[item]["handles_func"]() ]) p1 = 0 p2 = 0 while p1 < len(handles1) and p2 < len(handles2): if handles1[p1] == handles2[p2]: # in both item1 = db1._tables[item]["handle_func"](handles1[p1]) item2 = db2._tables[item]["handle_func"](handles2[p2]) diff = diff_items(item, item1.to_struct(), item2.to_struct()) if diff: diffs += [(item, item1, item2)] # else same! p1 += 1 p2 += 1 elif handles1[p1] < handles2[p2]: # p1 is mssing in p2 item1 = db1._tables[item]["handle_func"](handles1[p1]) missing_from_new += [(item, item1)] p1 += 1 elif handles1[p1] > handles2[p2]: # p2 is mssing in p1 item2 = db2._tables[item]["handle_func"](handles2[p2]) missing_from_old += [(item, item2)] p2 += 1 while p1 < len(handles1): item1 = db1._tables[item]["handle_func"](handles1[p1]) missing_from_new += [(item, item1)] p1 += 1 while p2 < len(handles2): item2 = db2._tables[item]["handle_func"](handles2[p2]) missing_from_old += [(item, item2)] p2 += 1 return diffs, missing_from_old, missing_from_new
def run_action(self, action, handler): options, options_help = self.get_plugin_options(action.handle) args = {} for key, default_value in options.items(): args[key] = handler.get_argument(key) if action.ptype == "Report": clr = run_report(self.gramps_database, action.handle, of="/tmp/test.html", off="html", **args) # can check for results with clr elif action.ptype == "Import": filename = download( args["i"], "/tmp/%s-%s-%s.%s" % (str(profile.user.username), str(handle), timestamp(), args["iff"])) if filename is not None: import_file(self.gramps_database, filename, User()) # callback elif action.ptype == "Export": pmgr = BasePluginManager.get_instance() pdata = pmgr.get_plugin(action.handle) export_file(self.gramps_database, "export." + pdata.extension, User()) # callback handler.redirect("/action")
def diff_dbs(db1, db2, user=None): """ 1. new objects => mark for insert 2. deleted objects, no change locally after delete date => mark for deletion 3. deleted objects, change locally => mark for user confirm for deletion 4. updated objects => do a diff on differences, mark origin values as new data """ if user is None: user = User() missing_from_old = [] missing_from_new = [] diffs = [] with user.progress(_('Family Tree Differences'), _('Searching...'), 10) as step: for item in ['Person', 'Family', 'Source', 'Citation', 'Event', 'Media', 'Place', 'Repository', 'Note', 'Tag']: step() handles1 = sorted([handle for handle in db1.get_table_func(item,"handles_func")()]) handles2 = sorted([handle for handle in db2.get_table_func(item,"handles_func")()]) p1 = 0 p2 = 0 while p1 < len(handles1) and p2 < len(handles2): if handles1[p1] == handles2[p2]: # in both item1 = db1.get_table_func(item,"handle_func")(handles1[p1]) item2 = db2.get_table_func(item,"handle_func")(handles2[p2]) diff = diff_items(item, item1.to_struct(), item2.to_struct()) if diff: diffs += [(item, item1, item2)] # else same! p1 += 1 p2 += 1 elif handles1[p1] < handles2[p2]: # p1 is mssing in p2 item1 = db1.get_table_func(item,"handle_func")(handles1[p1]) missing_from_new += [(item, item1)] p1 += 1 elif handles1[p1] > handles2[p2]: # p2 is mssing in p1 item2 = db2.get_table_func(item,"handle_func")(handles2[p2]) missing_from_old += [(item, item2)] p2 += 1 while p1 < len(handles1): item1 = db1.get_table_func(item,"handle_func")(handles1[p1]) missing_from_new += [(item, item1)] p1 += 1 while p2 < len(handles2): item2 = db2.get_table_func(item,"handle_func")(handles2[p2]) missing_from_old += [(item, item2)] p2 += 1 return diffs, missing_from_old, missing_from_new
def diff_dbs(db1, db2, user=None): """ 1. new objects => mark for insert 2. deleted objects, no change locally after delete date => mark for deletion 3. deleted objects, change locally => mark for user confirm for deletion 4. updated objects => do a diff on differences, mark origin values as new data """ if user is None: user = User() missing_from_old = [] missing_from_new = [] diffs = [] with user.progress(_("Family Tree Differences"), _("Searching..."), 10) as step: for item in ["Person", "Family", "Source", "Citation", "Event", "Media", "Place", "Repository", "Note", "Tag"]: step() handles1 = sorted([handle2internal(handle) for handle in db1._tables[item]["handles_func"]()]) handles2 = sorted([handle2internal(handle) for handle in db2._tables[item]["handles_func"]()]) p1 = 0 p2 = 0 while p1 < len(handles1) and p2 < len(handles2): if handles1[p1] == handles2[p2]: # in both item1 = db1._tables[item]["handle_func"](handles1[p1]) item2 = db2._tables[item]["handle_func"](handles2[p2]) diff = diff_items(item, item1.to_struct(), item2.to_struct()) if diff: diffs += [(item, item1, item2)] # else same! p1 += 1 p2 += 1 elif handles1[p1] < handles2[p2]: # p1 is mssing in p2 item1 = db1._tables[item]["handle_func"](handles1[p1]) missing_from_new += [(item, item1)] p1 += 1 elif handles1[p1] > handles2[p2]: # p2 is mssing in p1 item2 = db2._tables[item]["handle_func"](handles2[p2]) missing_from_old += [(item, item2)] p2 += 1 while p1 < len(handles1): item1 = db1._tables[item]["handle_func"](handles1[p1]) missing_from_new += [(item, item1)] p1 += 1 while p2 < len(handles2): item2 = db2._tables[item]["handle_func"](handles2[p2]) missing_from_old += [(item, item2)] p2 += 1 return diffs, missing_from_old, missing_from_new
def load(self, directory, callback=None, mode=None, force_schema_upgrade=False, force_bsddb_upgrade=False, force_bsddb_downgrade=False, force_python_upgrade=False, update=True): DbGeneric.load(self, directory, callback, mode, force_schema_upgrade, force_bsddb_upgrade, force_bsddb_downgrade, force_python_upgrade) # Dictionary-specific load: from gramps.plugins.importer.importxml import importData from gramps.cli.user import User if self._directory: backups = sorted(glob.glob( os.path.join(self._directory, "backup-*.gramps")), reverse=True) if backups: filename = backups[0] if os.path.isfile(filename): importData(self, filename, User()) self.reindex_reference_map(lambda progress: None) self.rebuild_secondary(lambda progress: None) self.has_changed = False
def call(*args): """ Call Gramps to perform the action with out and err captured """ print("call:", args) gramps = Gramps(user=User(auto_accept=True, quiet=True)) out, err = gramps.run(*args) print("out:", out, "err:", err) return out, err
def __init__(self, name, include_private=True, include_living=True): """Initialize the database object for family tree `name`. This will raise if the database backend is not `sqlite`. The constructor does not open/lock the database yet. Parameters: - `include_private`: include records marked as private. Default True - `include_living`: include living people. Default True """ self.name = name self.include_private = include_private self.include_living = include_living self.dbstate = DbState() self.dbman = CLIDbManager(self.dbstate) self.user = User() self.smgr = CLIManager(self.dbstate, True, self.user) self.path = self.dbman.get_family_tree_path(name) if not self.path: raise ValueError( "Family tree {} not found. Known trees: {}".format( name, self.dbman.family_tree_list())) self.db_backend = self.get_dbid() if self.db_backend not in ALLOWED_DB_BACKENDS: raise ValueError( "Database backend '{}' of tree '{}' not supported.".format( self.db_backend, name))
def import_as_dict(filename, user=None): """ Import the filename into a DictionaryDb and return it. """ if user is None: user = User() db = DictionaryDb() db.load(None) db.set_feature("skip-import-additions", True) dbstate = DbState() climanager = CLIManager(dbstate, setloader=False, user=user) climanager.do_reg_plugins(dbstate, None) pmgr = BasePluginManager.get_instance() (name, ext) = os.path.splitext(os.path.basename(filename)) format = ext[1:].lower() import_list = pmgr.get_reg_importers() for pdata in import_list: if format == pdata.extension: mod = pmgr.load_plugin(pdata) if not mod: for item in pmgr.get_fail_list(): name, error_tuple, pdata = item # (filename, (exception-type, exception, traceback), pdata) etype, exception, traceback = error_tuple #print("ERROR:", name, exception) return False import_function = getattr(mod, pdata.import_function) results = import_function(db, filename, user) if results is None: return None return db return None
def import_from_filename(self, db, filename, user=None): """ Import the filename into the db. """ from .plug import BasePluginManager from .const import PLUGINS_DIR, USER_PLUGINS from gramps.cli.user import User pmgr = BasePluginManager.get_instance() if user is None: user = User() (name, ext) = os.path.splitext(os.path.basename(filename)) extension = ext[1:].lower() import_list = pmgr.get_reg_importers() if import_list == []: # This might happen if using gramps from outside, and # we haven't loaded plugins yet pmgr.reg_plugins(PLUGINS_DIR, self, None) pmgr.reg_plugins(USER_PLUGINS, self, None, load_on_reg=True) import_list = pmgr.get_reg_importers() for pdata in import_list: if extension == pdata.extension: mod = pmgr.load_plugin(pdata) if not mod: for item in pmgr.get_fail_list(): name, error_tuple, pdata = item etype, exception, traceback = error_tuple print("ERROR:", name, exception) return False import_function = getattr(mod, pdata.import_function) results = import_function(db, filename, user) return True return False
def diff_db_to_file(old_db, filename, user=None): if user is None: user = User() # First, get data as a DictionaryDb new_db = import_as_dict(filename, user, user) # Next get differences: diffs, m_old, m_new = diff_dbs(old_db, new_db, user) return diffs, m_old, m_new
def diff_db_to_file(old_db, filename, user=None): if user is None: user = User() # First, get data as a InMemoryDB new_db = import_as_dict(filename, user, user) if new_db is not None: # Next get differences: diffs, m_old, m_new = diff_dbs(old_db, new_db, user) return diffs, m_old, m_new
def __init__(self, user=None, dbstate=None): ## Setup: from gramps.cli.clidbman import CLIDbManager self.dbstate = dbstate or DbState() #we need a manager for the CLI session self.user = user or User(auto_accept=True, quiet=False) self.climanager = CLIManager(self.dbstate, setloader=True, user=self.user) self.clidbmanager = CLIDbManager(self.dbstate)
def setUp(self): self.database1 = make_database("sqlite") try: os.mkdir("/tmp/bsddb_exportsql_1") except: pass self.database1.load("/tmp/bsddb_exportsql_1") importXML(self.database1, gramps_path + "/example/gramps/example.gramps", User()) exportSQL(self.database1, "/tmp/exported1.sql", User(), None) self.database2 = make_database("sqlite") try: os.mkdir("/tmp/bsddb_exportsql_2") except: pass self.database2.load("/tmp/bsddb_exportsql_2")
def get_db(self, force_unlock: bool = False) -> DbState: """Open the database and return a dbstate instance. If `force_unlock` is `True`, will break an existing lock (use with care!). """ dbstate = DbState() user = User() smgr = CLIManager(dbstate, True, user) if force_unlock: self.break_lock() smgr.open_activate(self.path) return dbstate
class StructTest(unittest.TestCase): DB = import_as_dict( os.environ["GRAMPS_RESOURCES"] + "/example/gramps/data.gramps", User()) def __init__(self, *args, **kwargs): self.dbi = DBI(StructTest.DB, None) # no document here self.dbi.flat = True self.dbi.sdb = SimpleAccess(StructTest.DB) self.pcount = len(StructTest.DB._tables["Person"]["handles_func"]()) unittest.TestCase.__init__(self, *args, **kwargs) def runTest(self): # for python -i pass def test_struct1(self): with StructTest.DB._tables["Person"]["cursor_func"]() as cursor: for handle, person in cursor: p = StructTest.DB._tables["Person"]["class_func"](person) if p and len(p.parent_family_list) > 0: person_with_parents = p break to_struct = person_with_parents.to_struct() struct = Struct(to_struct, StructTest.DB) self.assertTrue( len(struct.parent_family_list) > 0, "Size not correct: %s is not > than %s" % (len(struct.parent_family_list), 0)) self.assertTrue( struct.parent_family_list[0].private == False, "Inproper value of private: %s != %s" % (struct.parent_family_list[0].private, False)) def test_struct2(self): with StructTest.DB._tables["Person"]["cursor_func"]() as cursor: for handle, person in cursor: p = StructTest.DB._tables["Person"]["class_func"](person) if p and len(p.event_ref_list) > 0: person_with_events = p break to_struct = person_with_events.to_struct() struct = Struct(to_struct, StructTest.DB) self.assertTrue( len(struct.event_ref_list) > 0, "Size not correct: %s is not > than %s" % (len(struct.event_ref_list), 0)) self.assertTrue( struct.event_ref_list[0] is not None, "not None: %s is not %s" % (struct.event_ref_list[0], None))
def get_db(self, lock: bool = False, force_unlock: bool = False) -> DbState: """Open the database and return a dbstate instance. If `lock` is `False`, will not write a lock file (use with care!). If `force_unlock` is `True`, will break an existing lock (use with care!). """ dbstate = DbState() user = User() smgr = WebDbSessionManager(dbstate, user) smgr.do_reg_plugins(dbstate, uistate=None) if force_unlock: self.break_lock() mode = DBMODE_W if lock else DBMODE_R smgr.open_activate(self.path, mode=mode) return dbstate
def get_db(self, readonly: bool = True, force_unlock: bool = False) -> DbState: """Open the database and return a dbstate instance. If `readonly` is `True` (default), write operations will fail (note, this is not enforced by Gramps but must be taken care of in Web API methods!). If `force_unlock` is `True`, will break an existing lock (use with care!). """ dbstate = DbState() user = User() smgr = WebDbSessionManager(dbstate, user) smgr.do_reg_plugins(dbstate, uistate=None) if force_unlock: self.break_lock() mode = DBMODE_R if readonly else DBMODE_W smgr.open_activate( self.path, mode=mode, username=self.username, password=self.password ) return dbstate
def load(self, directory, callback=None, mode=None, force_schema_upgrade=False, force_bsddb_upgrade=False, force_bsddb_downgrade=False, force_python_upgrade=False): super().load(directory, callback, mode, force_schema_upgrade, force_bsddb_upgrade, force_bsddb_downgrade, force_python_upgrade) # Dictionary-specific load: from gramps.plugins.importer.importxml import importData from gramps.cli.user import User if self._directory: filename = os.path.join(self._directory, "data.gramps") if os.path.isfile(filename): importData(self, filename, User()) self.reindex_reference_map(lambda progress: None) self.rebuild_secondary(lambda progress: None)
def setUp(self): self.database1 = dbstate.make_database("bsddb") try: os.mkdir("/tmp/bsddb_exportsql_1") except: pass self.database1.write_version("/tmp/bsddb_exportsql_1") self.database1.load("/tmp/bsddb_exportsql_1") importXML(self.database1, gramps_path + "/example/gramps/example.gramps", User()) exportSQL(self.database1, "/tmp/exported1.sql") self.database2 = dbstate.make_database("bsddb") try: os.mkdir("/tmp/bsddb_exportsql_2") except: pass self.database2.write_version("/tmp/bsddb_exportsql_2") self.database2.load("/tmp/bsddb_exportsql_2")
struct = obj.to_struct() serialized = obj.__class__.from_struct(struct) def test(self): self.assertEqual(obj.serialize(), serialized) name = "test_serialize_%s_%s" % (obj.__class__.__name__, obj.handle) setattr(DatabaseCheck, name, test) #### #def test2(self): # self.assertEqual(obj.serialize(), from_struct(struct).serialize()) #name = "test_create_%s_%s" % (obj.__class__.__name__, obj.handle) #setattr(DatabaseCheck, name, test2) db = import_as_dict("example/gramps/example.gramps", User()) for table in db._tables.keys(): for handle in db._tables[table]["handles_func"](): obj = db._tables[table]["handle_func"](handle) generate_case(obj) class StructTest(unittest.TestCase): def test(self): family = db.get_family_from_gramps_id("F0001") s = Struct(family.to_struct(), db) self.assertEqual(s["gramps_id"], "F0001") s["gramps_id"] = "TEST" self.assertEqual(s["gramps_id"], "TEST") self.assertEqual(s.father_handle.primary_name.first_name, "Allen Carl") s["father_handle.primary_name.first_name"] = "Edward"
def setUpClass(cls): """ Import example database. """ cls.db = import_as_dict(EXAMPLE, User())
class SelectTest(unittest.TestCase): DB = import_as_dict( os.environ["GRAMPS_RESOURCES"] + "/example/gramps/data.gramps", User()) def __init__(self, *args, **kwargs): self.dbi = DBI(SelectTest.DB, None) # no document here self.dbi.flat = True self.dbi.sdb = SimpleAccess(SelectTest.DB) self.pcount = len(SelectTest.DB._tables["Person"]["handles_func"]()) self.john_count = 0 with SelectTest.DB._tables["Person"]["cursor_func"]() as cursor: for handle, person in cursor: name = SelectTest.DB._tables["Person"]["class_func"]( person).get_primary_name() if name and "John" in name.first_name: self.john_count += 1 unittest.TestCase.__init__(self, *args, **kwargs) def runTest(self): # for python -i pass def do_query(self, test, string, count): self.dbi.parse(string) table = Table() self.dbi.process_table(table) self.assertTrue( len(table.data) == count, "Test #%d, Selected %d records from example.gramps; should have been %d: '%s'" % (test, len(table.data), count, string)) return table def test_select1(self): self.do_query(1, "select * from person;", self.pcount) def test_select2(self): self.do_query( 2, "select primary_name.first_name " "from person " "where 'John' in primary_name.first_name;", self.john_count) def test_select3(self): self.do_query( 3, "update person SET primary_name.first_name='XXX' " "where 'John' in primary_name.first_name;", self.john_count) def test_select4(self): self.do_query( 4, "select primary_name.first_name " "from person " "where primary_name.first_name == 'XXX';", self.john_count) def test_select5(self): self.do_query( 5, "UPDATE person SET private = (False or False) " "from person " "where primary_name.first_name == 'XXX';", self.john_count) def test_select6(self): self.do_query( 6, "select private, primary_name " "from person " "where primary_name.first_name == 'XXX' and private;", 0) def test_select7(self): self.do_query( 7, "SELECT private, primary_name " "FROM person " "where primary_name.first_name == 'XXX' and not private;", self.john_count) def test_select8(self): self.do_query( 8, "UPDATE person SET private = (False or True) " "from person " "where primary_name.first_name == 'XXX';", self.john_count) def test_select9(self): self.do_query( 9, "select private, primary_name " "from person " "where primary_name.first_name == 'XXX' and private;", self.john_count) def test_select10(self): self.do_query( 10, "select private, primary_name " "from person " "where primary_name.first_name == 'XXX' and not private;", 0) def test_select11(self): self.do_query(11, "SELECT * from person LIMIT 10, 20", 10) def test_select12(self): self.do_query(12, "SELECT * from person LIMIT 5", 5) def test_select13(self): self.do_query(13, "SELECT ROWNUM, random.random() from person LIMIT 5", 5) def test_select14(self): self.do_query( 14, "select * from person where not parent_family_list[0].private;", 38) def test_select15(self): self.do_query( 15.1, "UPDATE person SET private=True WHERE not parent_family_list[0].private;", 38) self.do_query(15.2, "SELECT * from person WHERE private;", 38) self.do_query(15.3, "UPDATE person SET private=False;", 60) self.do_query(15.4, "UPDATE person SET private=False WHERE private;", 0) self.do_query(15.5, "UPDATE person SET private=True;", 60) self.do_query(15.6, "UPDATE person SET private=True where not private;", 0) def test_select16(self): table = self.do_query(16.1, "SELECT gramps_id as id from person;", self.pcount) self.assertTrue( len(table.data) == 60, "Table should have selected 60 items") def test_select17(self): table = self.do_query( 17.1, "SELECT gramps_id as id from person where id == 'I0004';", 1) self.assertTrue( table.data[0][0] == "I0004", "First row, first col is %s, should be %s" % (table.data[0][0], "I0004")) table = self.do_query( 17.2, "SELECT gramps_id, father_handle.primary_name.first_name " "FROM family WHERE father_handle.primary_name.first_name;", 23) table.data = sorted(table.data) self.assertTrue( table.data[0][0] == "F0000", "First row, first col is %s, should be %s" % (table.data[0][0], "F0000")) self.assertTrue( table.data[0][1] == "Martin", "First row, second col is %s, should be %s" % (table.data[0][1], "Martin")) self.assertTrue(len(table.data) == 23, "Should have selected 23 rows") table = self.do_query( 17.3, "UPDATE family SET father_handle.primary_name.first_name='Father' WHERE gramps_id == 'F0005';", 1) self.assertTrue( table.data[0][0] == "F0005", "First row, first col is %s, should be %s" % (table.data[0][0], "F0005")) table = self.do_query( 17.4, "SELECT gramps_id, father_handle.primary_name.first_name, father_handle.gramps_id " "FROM family WHERE gramps_id == 'F0005';", 1) self.assertTrue( table.data[0][0] == "F0005", "1 First row, first col is %s, should be %s" % (table.data[0][0], "F0005")) self.assertTrue( table.data[0][1] == "Father", "1 First row, second col is %s, should be %s" % (table.data[0][1], "Father")) self.assertTrue( table.data[0][2] == "I0012", "1 First row, third col is %s, should be %s" % (table.data[0][2], "I0012")) table = self.do_query( 17.5, "SELECT gramps_id, primary_name.first_name " "FROM person WHERE gramps_id == 'I0012';", 1) self.assertTrue( table.data[0][0] == "I0012", "First row, first col is %s, should be %s" % (table.data[0][0], "I0012")) self.assertTrue( table.data[0][1] == "Father", "First row, second col is %s, should be %s" % (table.data[0][1], "Father")) def test_select18(self): table = self.do_query( 18.1, "SELECT gramps_id, father_handle.GIVEN from family where gramps_id == 'F0005';", 1) self.assertTrue( table.data[0][1] == "Father", "First row, second col is %s, should be %s" % (table.data[0][1], "Father"))
struct = obj.to_struct() serialized = obj.__class__.from_struct(struct) def test(self): self.assertEqual(obj.serialize(), serialized) name = "test_serialize_%s_%s" % (obj.__class__.__name__, obj.handle) setattr(DatabaseCheck, name, test) #### #def test2(self): # self.assertEqual(obj.serialize(), from_struct(struct).serialize()) #name = "test_create_%s_%s" % (obj.__class__.__name__, obj.handle) #setattr(DatabaseCheck, name, test2) db = import_as_dict(EXAMPLE, User()) for table in db.get_table_func(): for handle in db.get_table_func(table, "handles_func")(): obj = db.get_table_func(table, "handle_func")(handle) generate_case(obj) class StructTest(unittest.TestCase): def test(self): family = db.get_family_from_gramps_id("F0001") s = Struct(family.to_struct(), db) self.assertEqual(s["gramps_id"], "F0001") s["gramps_id"] = "TEST" self.assertEqual(s["gramps_id"], "TEST") self.assertEqual(s.father_handle.primary_name.first_name, "Allen Carl") s["father_handle.primary_name.first_name"] = "Edward"
def test_export_sql(self): importSQL(self.database2, "/tmp/exported1.sql", User())
def setUpClass(cls): """ Import example database. """ cls.db = import_as_dict("example/gramps/example.gramps", User())