def __init__(self, dbstate, user, options_class, name, callback=None):
uistate = user.uistate
tool.Tool.__init__(self, dbstate, options_class, name)
if self.db.readonly:
return
self.db.disable_signals()
if uistate:
self.callback = uistate.pulse_progressbar
uistate.set_busy_cursor(True)
uistate.progress.show()
uistate.push_message(dbstate, _("Rebuilding secondary indexes..."))
UpdateCallback.__init__(self, self.callback)
self.set_total(12)
self.db.rebuild_secondary(self.update)
self.reset()
uistate.set_busy_cursor(False)
uistate.progress.hide()
OkDialog(_("Secondary indexes rebuilt"),
_('All secondary indexes have been rebuilt.'),
parent=uistate.window)
else:
print("Rebuilding Secondary Indexes...")
self.db.rebuild_secondary(self.update_empty)
print("All secondary indexes have been rebuilt.")
self.db.enable_signals()
def __init__(self, dbstate, user, options_class, name, callback=None):
uistate = user.uistate
tool.Tool.__init__(self, dbstate, options_class, name)
if self.db.readonly:
return
self.db.disable_signals()
if uistate:
self.callback = uistate.pulse_progressbar
uistate.set_busy_cursor(True)
uistate.progress.show()
uistate.push_message(dbstate, _("Rebuilding reference maps..."))
else:
self.callback = None
print(_("Rebuilding reference maps..."))
UpdateCallback.__init__(self, self.callback)
self.set_total(6)
self.db.reindex_reference_map(self.update)
self.reset()
if uistate:
uistate.set_busy_cursor(False)
uistate.progress.hide()
OkDialog(_("Reference maps rebuilt"),
_('All reference maps have been rebuilt.'),
parent=uistate.window)
else:
print(_("All reference maps have been rebuilt."))
self.db.enable_signals()
def __init__(self, dbstate, user, options_class, name, callback=None):
uistate = user.uistate
tool.Tool.__init__(self, dbstate, options_class, name)
if self.db.readonly:
return
self.db.disable_signals()
if uistate:
self.callback = uistate.pulse_progressbar
uistate.set_busy_cursor(True)
uistate.progress.show()
uistate.push_message(dbstate, _("Rebuilding gender statistics for name gender guessing..."))
else:
self.callback = None
print("Rebuilding gender statistics for name gender guessing...")
UpdateCallback.__init__(self, self.callback)
self.set_total(self.db.get_number_of_people())
self.rebuild_genderstats()
self.reset()
if uistate:
uistate.set_busy_cursor(False)
uistate.progress.hide()
OkDialog(_("Gender statistics rebuilt"),
_('Gender statistics for name gender guessing have been rebuilt.'),
parent=uistate.window)
else:
print("Gender statistics for name gender guessing have been rebuilt.")
self.db.enable_signals()
def rebuild_secondary(self, callback=None):
"""
Rebuild secondary indices
"""
if self.readonly:
return
total = 0
for tbl in ('people', 'families', 'events', 'places', 'sources',
'citations', 'media', 'repositories', 'notes', 'tags'):
total += self.method("get_number_of_%s", tbl)()
UpdateCallback.__init__(self, callback)
self.set_total(total)
# First, expand blob to individual fields:
self._txn_begin()
for obj_type in ('Person', 'Family', 'Event', 'Place', 'Repository',
'Source', 'Citation', 'Media', 'Note', 'Tag'):
for handle in self.method('get_%s_handles', obj_type)():
obj = self.method('get_%s_from_handle', obj_type)(handle)
self._update_secondary_values(obj)
self.update()
self._txn_commit()
# Next, rebuild stats:
gstats = self.get_gender_stats()
self.genderStats = GenderStats(gstats)
def __init__(self, dbstate, user, options_class, name, callback=None):
uistate = user.uistate
tool.Tool.__init__(self, dbstate, options_class, name)
if self.db.readonly:
return
self.db.disable_signals()
if uistate:
self.callback = uistate.pulse_progressbar
uistate.set_busy_cursor(True)
uistate.progress.show()
uistate.push_message(dbstate, _("Rebuilding secondary indexes..."))
UpdateCallback.__init__(self, self.callback)
self.set_total(12)
self.db.rebuild_secondary(self.update)
self.reset()
uistate.set_busy_cursor(False)
uistate.progress.hide()
OkDialog(_("Secondary indexes rebuilt"),
_('All secondary indexes have been rebuilt.'),
parent=uistate.window)
else:
print("Rebuilding Secondary Indexes...")
self.db.rebuild_secondary(self.update_empty)
print("All secondary indexes have been rebuilt.")
self.db.enable_signals()
self.db.request_rebuild()
def __init__(self, dbstate, user, options_class, name, callback=None):
uistate = user.uistate
tool.Tool.__init__(self, dbstate, options_class, name)
if self.db.readonly:
return
self.db.disable_signals()
if uistate:
self.callback = uistate.pulse_progressbar
uistate.set_busy_cursor(True)
uistate.progress.show()
uistate.push_message(dbstate, _("Rebuilding reference maps..."))
else:
self.callback = None
print(_("Rebuilding reference maps..."))
UpdateCallback.__init__(self, self.callback)
self.set_total(6)
self.db.reindex_reference_map(self.update)
self.reset()
if uistate:
uistate.set_busy_cursor(False)
uistate.progress.hide()
OkDialog(_("Reference maps rebuilt"), # parent-OK
_('All reference maps have been rebuilt.'),
parent=uistate.window)
else:
print(_("All reference maps have been rebuilt."))
self.db.enable_signals()
def __init__(self, dbstate, user, options_class, name, callback=None):
uistate = user.uistate
self.title = _('Unused Objects')
tool.Tool.__init__(self, dbstate, options_class, name)
if self.db.readonly:
return
ManagedWindow.__init__(self, uistate,[], self.__class__)
UpdateCallback.__init__(self, self.uistate.pulse_progressbar)
self.dbstate = dbstate
self.uistate = uistate
self.tables = {
'events' : {'get_func': self.db.get_event_from_handle,
'remove' : self.db.remove_event,
'get_text': self.get_event_text,
'editor' : 'EditEvent',
'icon' : 'gramps-event',
'name_ix' : 4},
'sources' : {'get_func': self.db.get_source_from_handle,
'remove' : self.db.remove_source,
'get_text': None,
'editor' : 'EditSource',
'icon' : 'gramps-source',
'name_ix' : 2},
'places' : {'get_func': self.db.get_place_from_handle,
'remove' : self.db.remove_place,
'get_text': None,
'editor' : 'EditPlace',
'icon' : 'gramps-place',
'name_ix' : 2},
'media' : {'get_func': self.db.get_object_from_handle,
'remove' : self.db.remove_object,
'get_text': None,
'editor' : 'EditMedia',
'icon' : 'gramps-media',
'name_ix' : 4},
'repos' : {'get_func': self.db.get_repository_from_handle,
'remove' : self.db.remove_repository,
'get_text': None,
'editor' : 'EditRepository',
'icon' : 'gramps-repository',
'name_ix' : 3},
'notes' : {'get_func': self.db.get_note_from_handle,
'remove' : self.db.remove_note,
'get_text': self.get_note_text,
'editor' : 'EditNote',
'icon' : 'gramps-notes',
'name_ix' : 2},
}
self.init_gui()
def __init__(self, dbstate, user, options_class, name, callback=None):
uistate = user.uistate
self.title = _('Unused Objects')
tool.Tool.__init__(self, dbstate, options_class, name)
if self.db.readonly:
return
ManagedWindow.__init__(self, uistate,[], self.__class__)
UpdateCallback.__init__(self, self.uistate.pulse_progressbar)
self.dbstate = dbstate
self.uistate = uistate
self.tables = {
'events' : {'get_func': self.db.get_event_from_handle,
'remove' : self.db.remove_event,
'get_text': self.get_event_text,
'editor' : 'EditEvent',
'stock' : 'gramps-event',
'name_ix' : 4},
'sources' : {'get_func': self.db.get_source_from_handle,
'remove' : self.db.remove_source,
'get_text': None,
'editor' : 'EditSource',
'stock' : 'gramps-source',
'name_ix' : 2},
'places' : {'get_func': self.db.get_place_from_handle,
'remove' : self.db.remove_place,
'get_text': None,
'editor' : 'EditPlace',
'stock' : 'gramps-place',
'name_ix' : 2},
'media' : {'get_func': self.db.get_object_from_handle,
'remove' : self.db.remove_object,
'get_text': None,
'editor' : 'EditMedia',
'stock' : 'gramps-media',
'name_ix' : 4},
'repos' : {'get_func': self.db.get_repository_from_handle,
'remove' : self.db.remove_repository,
'get_text': None,
'editor' : 'EditRepository',
'stock' : 'gramps-repository',
'name_ix' : 3},
'notes' : {'get_func': self.db.get_note_from_handle,
'remove' : self.db.remove_note,
'get_text': self.get_note_text,
'editor' : 'EditNote',
'stock' : 'gramps-notes',
'name_ix' : 2},
}
self.init_gui()
def __init__(self, dbstate, user, options_class, name, callback=None):
self.uistate = user.uistate
self.dbstate = dbstate.db
if self.uistate:
tool.BatchTool.__init__(self, dbstate, user, options_class, name)
if self.fail:
return # user denied to modify Gramps IDs
ManagedWindow.__init__(self, self.uistate, [], self.__class__)
if not self.uistate:
UpdateCallback.__init__(self, user.callback)
self.object_status = True
self.change_status = False
self.start_zero = True
self.step_cnt, self.step_list = 0, ['1', '2', '5', '10']
self.keep_status = True
self.obj_values = {} # enable access to all internal values
self.active_entries, self.format_entries = {}, {}
self.change_entries = {}
self.start_entries, self.step_entries = {}, {}
self.keep_entries = {}
self.prim_methods, self.obj_methods = {}, {}
for prim_obj, prim_objs in self.xobjects:
class_type = prim_obj.title()
iter_handles = "self.dbstate.iter_%s_handles" % prim_obj
get_number_obj = "self.dbstate.get_number_of_%s" % prim_objs
prefix_fmt = "self.dbstate.%s_prefix" % prim_obj
get_from_id = "self.dbstate.get_%s_from_gramps_id" % prim_obj
get_from_handle = "self.dbstate.get_%s_from_handle" % prim_obj
next_from_id = "self.dbstate.find_next_%s_gramps_id" % prim_obj
commit = "self.dbstate.commit_%s" % prim_obj
self.prim_methods[prim_obj] = (eval(prefix_fmt),
eval(get_number_obj)(),
eval(next_from_id)())
self.obj_methods[prim_obj] = (eval(class_type), eval(iter_handles),
eval(commit), eval(get_from_id),
eval(get_from_handle),
eval(next_from_id))
object_fmt, quant_id, next_id = self.prim_methods[prim_obj]
obj_value = ReorderEntry(object_fmt, quant_id, next_id)
self.obj_values[prim_obj] = obj_value
if self.uistate:
self._display()
else:
self._execute()
def __init__(self, dbstate, user, options_class, name, callback=None):
self.uistate = user.uistate
self.db = dbstate.db
if self.uistate:
tool.BatchTool.__init__(self, dbstate, user, options_class, name)
if self.fail:
return # user denied to modify Gramps IDs
ManagedWindow.__init__(self, self.uistate, [], self.__class__)
if not self.uistate:
UpdateCallback.__init__(self, user.callback)
self.object_status = True
self.change_status = False
self.start_zero = True
self.step_cnt, self.step_list = 0, ['1', '2', '5', '10']
self.keep_status = True
self.obj_values = {} # enable access to all internal values
self.active_entries, self.format_entries = {}, {}
self.change_entries = {}
self.start_entries, self.step_entries = {}, {}
self.keep_entries = {}
self.prim_methods, self.obj_methods = {}, {}
for prim_obj, prim_objs in self.xobjects:
iter_handles = "iter_%s_handles" % prim_obj
get_number_obj = "get_number_of_%s" % prim_objs
prefix_fmt = "%s_prefix" % prim_obj
get_from_id = "get_%s_from_gramps_id" % prim_obj
get_from_handle = "get_%s_from_handle" % prim_obj
next_from_id = "find_next_%s_gramps_id" % prim_obj
commit = "commit_%s" % prim_obj
self.prim_methods[prim_obj] = (getattr(self.db, prefix_fmt),
getattr(self.db, get_number_obj)(),
getattr(self.db, next_from_id)())
self.obj_methods[prim_obj] = (getattr(self.db, iter_handles),
getattr(self.db, commit),
getattr(self.db, get_from_id),
getattr(self.db, get_from_handle),
getattr(self.db, next_from_id))
object_fmt, quant_id, next_id = self.prim_methods[prim_obj]
obj_value = ReorderEntry(object_fmt, quant_id, next_id, prim_obj)
self.obj_values[prim_obj] = obj_value
if self.uistate:
self._display()
else:
self._execute()
def reindex_reference_map(self, callback):
"""
Reindex all primary records in the database.
"""
self._txn_begin()
self.dbapi.execute("DELETE FROM reference")
total = 0
for tbl in ('people', 'families', 'events', 'places', 'sources',
'citations', 'media', 'repositories', 'notes', 'tags'):
total += self.method("get_number_of_%s", tbl)()
UpdateCallback.__init__(self, callback)
self.set_total(total)
primary_table = (
(self.get_person_cursor, Person),
(self.get_family_cursor, Family),
(self.get_event_cursor, Event),
(self.get_place_cursor, Place),
(self.get_source_cursor, Source),
(self.get_citation_cursor, Citation),
(self.get_media_cursor, Media),
(self.get_repository_cursor, Repository),
(self.get_note_cursor, Note),
(self.get_tag_cursor, Tag),
)
# Now we use the functions and classes defined above
# to loop through each of the primary object tables.
for cursor_func, class_func in primary_table:
logging.info("Rebuilding %s reference map", class_func.__name__)
with cursor_func() as cursor:
for found_handle, val in cursor:
obj = class_func.create(val)
references = set(obj.get_referenced_handles_recursively())
# handle addition of new references
for (ref_class_name, ref_handle) in references:
self.dbapi.execute(
"INSERT INTO reference "
"(obj_handle, obj_class, ref_handle, ref_class) "
"VALUES (?, ?, ?, ?)",
[obj.handle,
obj.__class__.__name__,
ref_handle,
ref_class_name])
self.update()
self._txn_commit()
def __init__(self, dbstate, user, options_class, name, callback=None):
uistate = user.uistate
tool.Tool.__init__(self, dbstate, options_class, name)
if self.db.readonly:
return
self.db.disable_signals()
if uistate:
self.callback = uistate.pulse_progressbar
uistate.set_busy_cursor(True)
uistate.progress.show()
uistate.push_message(
dbstate,
_("Rebuilding gender statistics for name gender guessing..."))
else:
self.callback = None
print("Rebuilding gender statistics for name gender guessing...")
UpdateCallback.__init__(self, self.callback)
self.set_total(self.db.get_number_of_people())
self.rebuild_genderstats()
self.reset()
if uistate:
uistate.set_busy_cursor(False)
uistate.progress.hide()
OkDialog(
_("Gender statistics rebuilt"),
_('Gender statistics for name gender guessing have been rebuilt.'
),
parent=uistate.window)
else:
print(
"Gender statistics for name gender guessing have been rebuilt."
)
self.db.enable_signals()
def __init__(self, db, callback):
UpdateCallback.__init__(self, callback)
self.db = db
self.prepared = False
def __init__(self, db, callback):
UpdateCallback.__init__(self, callback)
self.db = db
self.prepared = False
def load(self,
dirname,
callback=None,
mode=DBMODE_W,
force_schema_upgrade=False,
update=True,
username=None,
password=None):
"""
Here we create a sqlite db, and copy the bsddb into it.
The new db is initially in a new directory, when we finish the copy
we replace the contents of the original directory with the new db.
We alway raise an exception to complete this, as the new db still
needs to be upgraded some more. When we raise the exception, the new
db is closed.
"""
if not update:
raise DbException("Not Available")
if not force_schema_upgrade: # make sure user wants to upgrade
raise DbSupportedError(_("BSDDB"))
UpdateCallback.__init__(self, callback)
# Here we open the dbapi db (a new one) for writing
new_path = find_next_db_dir()
os.mkdir(new_path)
# store dbid in new dir
dbid = 'sqlite'
backend_path = os.path.join(new_path, DBBACKEND)
with open(backend_path, "w", encoding='utf8') as backend_file:
backend_file.write(dbid)
super().load(new_path,
callback=None,
mode='w',
force_schema_upgrade=False,
username=username,
password=password)
# now read in the bsddb and copy to dpapi
schema_vers = None
total = 0
tables = (('person', 'person'), ('family', 'family'),
('event', 'event'), ('place', 'place'),
('repo', 'repository'), ('source', 'source'),
('citation', 'citation'), ('media', 'media'),
('note', 'note'), ('tag', 'tag'), ('meta_data', 'metadata'))
# open each dbmap, and get its length for the total
file_name = os.path.join(dirname, 'name_group.db')
if os.path.isfile(file_name):
name_group_dbmap = DB()
name_group_dbmap.set_flags(DB_DUP)
name_group_dbmap.open(file_name, 'name_group', DB_HASH, DB_RDONLY)
total += len(name_group_dbmap)
else:
name_group_dbmap = None
table_list = []
for old_t, new_t in (tables):
file_name = os.path.join(dirname, old_t + '.db')
if not os.path.isfile(file_name):
continue
dbmap = DB()
dbmap.open(file_name, old_t, DB_HASH, DB_RDONLY)
total += len(dbmap)
table_list.append((old_t, new_t, dbmap))
self.set_total(total)
# copy data from each dbmap to sqlite table
for old_t, new_t, dbmap in table_list:
self._txn_begin()
if new_t == 'metadata':
sql = ("REPLACE INTO metadata (setting, value) VALUES "
"(?, ?)")
else:
sql = ("INSERT INTO %s (handle, blob_data) VALUES "
"(?, ?)" % new_t)
for key in dbmap.keys():
self.update()
data = pickle.loads(dbmap[key], encoding='utf-8')
if new_t == 'metadata':
if key == b'version':
# found a schema version in metadata
schema_vers = data
elif key == b'researcher':
if len(data[0]) == 7: # Pre-3.3 format
# Upgrade researcher data to include a locality
# field in the address.
addr = tuple([data[0][0], ''] + list(data[0][1:]))
new_data = (addr, data[1], data[2], data[3])
else:
new_data = data
data = Researcher().unserialize(new_data)
elif key == b'name_formats':
# upgrade formats if they were saved in the old way
for format_ix in range(len(data)):
fmat = data[format_ix]
if len(fmat) == 3:
fmat = fmat + (True, )
data[format_ix] = fmat
elif key == b'gender_stats':
# data is a dict, containing entries (see GenderStats)
self.dbapi.execute("DELETE FROM gender_stats")
g_sql = ("INSERT INTO gender_stats "
"(given_name, female, male, unknown) "
"VALUES (?, ?, ?, ?)")
for name in data:
female, male, unknown = data[name]
self.dbapi.execute(g_sql,
[name, female, male, unknown])
continue # don't need this in metadata anymore
elif key == b'default':
# convert to string and change key
if isinstance(data, bytes):
data = data.decode('utf-8')
key = b'default-person-handle'
elif key == b'mediapath':
# change key
key = b'media-path'
elif key in [
b'surname_list', # created by db now
b'pevent_names', # obsolete
b'fevent_names'
]: # obsolete
continue
elif (b'_names' in key or b'refs' in key
or b'_roles' in key or b'rels' in key
or b'_types' in key):
# These are list, but need to be set
data = set(data)
self.dbapi.execute(sql,
[key.decode('utf-8'),
pickle.dumps(data)])
# get schema version from file if not in metadata
if new_t == 'metadata' and schema_vers is None:
versionpath = os.path.join(dirname, str(SCHVERSFN))
if os.path.isfile(versionpath):
with open(versionpath, "r") as version_file:
schema_vers = int(version_file.read().strip())
else:
schema_vers = 0
# and put schema version into metadata
self.dbapi.execute(sql, ["version", schema_vers])
self._txn_commit()
dbmap.close()
if new_t == 'metadata' and schema_vers < _MINVERSION:
raise DbVersionError(schema_vers, _MINVERSION, _DBVERSION)
if name_group_dbmap:
self._txn_begin()
for key in name_group_dbmap.keys():
self.update()
# name_group data (grouping) is NOT pickled
data = name_group_dbmap[key]
name = key.decode('utf-8')
grouping = data.decode('utf-8')
self.dbapi.execute(
"INSERT INTO name_group (name, grouping) VALUES (?, ?)",
[name, grouping])
self._txn_commit()
name_group_dbmap.close()
# done with new sqlite db, close it. Cannot use normal close as it
# overwrites the metadata.
self._close()
try:
clear_lock_file(self.get_save_path())
except IOError:
pass
self.db_is_open = False
self._directory = None
# copy tree name to new dir
old_db_name = os.path.join(dirname, NAME_FILE)
db_name = os.path.join(new_path, NAME_FILE)
with open(old_db_name, "r", encoding='utf8') as _file:
name = _file.read().strip()
with open(db_name, "w", encoding='utf8') as _file:
_file.write(name)
# remove files from old dir
for filename in os.listdir(dirname):
file_path = os.path.join(dirname, filename)
try:
os.unlink(file_path)
except Exception as e:
LOG.error('Failed to delete %s. Reason: %s' % (file_path, e))
# copy new db files to old dir
for filename in os.listdir(new_path):
old_file_path = os.path.join(new_path, filename)
file_path = os.path.join(dirname, filename)
try:
os.replace(old_file_path, file_path)
except Exception as e:
LOG.error('Failed to move %s. Reason: %s' % (old_file_path, e))
os.rmdir(new_path)
# done preparing new db, but we still need to finish schema upgrades
raise DbUpgradeRequiredError(schema_vers, 'xx')