def instance_from_struct(cls, struct):
"""
Given a struct with metadata, create a Gramps object.
self is class when called as a classmethod.
"""
from gramps.gen.lib import (Person, Family, Event, Source, Place,
Citation, Repository, Media, Note, Tag,
Date)
if isinstance(struct, dict):
if "_class" in struct.keys():
if struct["_class"] == "Person":
return Person.create(Person.from_struct(struct))
elif struct["_class"] == "Family":
return Family.create(Family.from_struct(struct))
elif struct["_class"] == "Event":
return Event.create(Event.from_struct(struct))
elif struct["_class"] == "Source":
return Source.create(Source.from_struct(struct))
elif struct["_class"] == "Place":
return Place.create(Place.from_struct(struct))
elif struct["_class"] == "Citation":
return Citation.create(Citation.from_struct(struct))
elif struct["_class"] == "Repository":
return Repository.create(Repository.from_struct(struct))
elif struct["_class"] == "Media":
return Media.create(Media.from_struct(struct))
elif struct["_class"] == "Note":
return Note.create(Note.from_struct(struct))
elif struct["_class"] == "Tag":
return Tag.create(Tag.from_struct(struct))
elif struct["_class"] == "Date":
return Date().unserialize(
Date.from_struct(struct, full=True))
raise AttributeError("invalid struct: %s" % struct)
def from_struct(struct):
"""
Given a struct with metadata, create a Gramps object.
"""
from gramps.gen.lib import (Person, Family, Event, Source, Place, Citation,
Repository, MediaObject, Note, Tag)
if isinstance(struct, dict):
if "_class" in struct.keys():
if struct["_class"] == "Person":
return Person.create(Person.from_struct(struct))
elif struct["_class"] == "Family":
return Family.create(Family.from_struct(struct))
elif struct["_class"] == "Event":
return Event.create(Event.from_struct(struct))
elif struct["_class"] == "Source":
return Source.create(Source.from_struct(struct))
elif struct["_class"] == "Place":
return Place.create(Place.from_struct(struct))
elif struct["_class"] == "Citation":
return Citation.create(Citation.from_struct(struct))
elif struct["_class"] == "Repository":
return Repository.create(Repository.from_struct(struct))
elif struct["_class"] == "MediaObject":
return MediaObject.create(MediaObject.from_struct(struct))
elif struct["_class"] == "Note":
return Note.create(Note.from_struct(struct))
elif struct["_class"] == "Tag":
return Tag.create(Tag.from_struct(struct))
raise AttributeError("invalid struct: %s" % struct)
def instance_from_struct(cls, struct):
"""
Given a struct with metadata, create a Gramps object.
self is class when called as a classmethod.
"""
from gramps.gen.lib import (Person, Family, Event, Source, Place, Citation,
Repository, Media, Note, Tag, Date)
if isinstance(struct, dict):
if "_class" in struct.keys():
if struct["_class"] == "Person":
return Person.create(Person.from_struct(struct))
elif struct["_class"] == "Family":
return Family.create(Family.from_struct(struct))
elif struct["_class"] == "Event":
return Event.create(Event.from_struct(struct))
elif struct["_class"] == "Source":
return Source.create(Source.from_struct(struct))
elif struct["_class"] == "Place":
return Place.create(Place.from_struct(struct))
elif struct["_class"] == "Citation":
return Citation.create(Citation.from_struct(struct))
elif struct["_class"] == "Repository":
return Repository.create(Repository.from_struct(struct))
elif struct["_class"] == "Media":
return Media.create(Media.from_struct(struct))
elif struct["_class"] == "Note":
return Note.create(Note.from_struct(struct))
elif struct["_class"] == "Tag":
return Tag.create(Tag.from_struct(struct))
elif struct["_class"] == "Date":
return Date().unserialize(Date.from_struct(struct, full=True))
raise AttributeError("invalid struct: %s" % struct)
def from_struct(struct):
"""
Given a struct with metadata, create a Gramps object.
"""
from gramps.gen.lib import (Person, Family, Event, Source, Place, Citation,
Repository, MediaObject, Note, Tag)
if isinstance(struct, dict):
if "_class" in struct.keys():
if struct["_class"] == "Person":
return Person.create(Person.from_struct(struct))
elif struct["_class"] == "Family":
return Family.create(Family.from_struct(struct))
elif struct["_class"] == "Event":
return Event.create(Event.from_struct(struct))
elif struct["_class"] == "Source":
return Source.create(Source.from_struct(struct))
elif struct["_class"] == "Place":
return Place.create(Place.from_struct(struct))
elif struct["_class"] == "Citation":
return Citation.create(Citation.from_struct(struct))
elif struct["_class"] == "Repository":
return Repository.create(Repository.from_struct(struct))
elif struct["_class"] == "MediaObject":
return MediaObject.create(MediaObject.from_struct(struct))
elif struct["_class"] == "Note":
return Note.create(Note.from_struct(struct))
elif struct["_class"] == "Tag":
return Tag.create(Tag.from_struct(struct))
raise AttributeError("invalid struct: %s" % struct)
def __find_tag(self, name):
tag = self.dbstate.db.get_tag_from_name(name)
if tag is None:
tag = Tag()
tag.set_name(name)
self.dbstate.db.add_tag(tag, self.trans)
tag = self.dbstate.db.get_tag_from_name(name)
self.dbstate.db.commit_tag(tag, self.trans)
return tag
def cb_add_clicked(self, button, top):
"""
Create a new tag.
"""
tag = Tag()
tag.set_priority(self.db.get_number_of_tags())
EditTag(self.db, self.uistate, self.track, tag)
if tag.get_handle():
self.namemodel.add((tag.get_priority(), tag.get_handle(),
tag.get_name(), tag.get_color()))
def cb_add_clicked(self, button, top):
"""
Create a new tag.
"""
tag = Tag()
tag.set_priority(self.db.get_number_of_tags())
edit_dialog = EditTag(self.db, top, tag)
edit_dialog.run()
if tag.get_handle():
self.namemodel.add((tag.get_priority(), tag.get_handle(),
tag.get_name(), tag.get_color()))
def importData(db, filename, user):
"""Function called by Gramps to import data on persons in CSV format."""
db.disable_signals()
try:
with DbTxn(_("JSON import"), db, batch=True) as trans:
with OpenFileOrStdin(filename, encoding="utf-8") as fp:
line = fp.readline()
while line:
data = json.loads(line)
if data["_class"] == "Person":
obj = Person.create(Person.from_struct(data))
db.add_person(obj, trans)
elif data["_class"] == "Family":
obj = Family.create(Family.from_struct(data))
db.add_family(obj, trans)
elif data["_class"] == "Event":
obj = Event.create(Event.from_struct(data))
db.add_event(obj, trans)
elif data["_class"] == "Media":
obj = Media.create(Media.from_struct(data))
db.add_media(obj, trans)
elif data["_class"] == "Repository":
obj = Repository.create(Repository.from_struct(data))
db.add_repository(obj, trans)
elif data["_class"] == "Tag":
obj = Tag.create(Tag.from_struct(data))
db.add_tag(obj, trans)
elif data["_class"] == "Source":
obj = Source.create(Source.from_struct(data))
db.add_source(obj, trans)
elif data["_class"] == "Citation":
obj = Citation.create(Citation.from_struct(data))
db.add_citation(obj, trans)
elif data["_class"] == "Note":
obj = Note.create(Note.from_struct(data))
db.add_note(obj, trans)
elif data["_class"] == "Place":
obj = Place.create(Place.from_struct(data))
db.add_place(obj, trans)
else:
LOG.warn("ignored: " + data)
line = fp.readline()
except EnvironmentError as err:
user.notify_error(_("%s could not be opened\n") % filename, str(err))
db.enable_signals()
db.request_rebuild()
def buildTag(self, tname, thandle=None):
if thandle == None:
thandle = id.create_id() # 26-merkkinen tunniste
tag = Tag()
tag.set_name(tname)
tag.set_change_time(self.chgtime)
# tag.set_color("#EF2929")
tag.set_handle(thandle)
# print ("Tag ") ; print(tag.to_struct())
return ([tag, thandle])
def find_tag(self,tagdata):
tagname = tagdata["name"]
if tagname in self.tags: return self.tags[tagname]
tag = Tag()
tag.set_name(tagname)
tag.set_color(tagdata["color"])
tag.set_priority(tagdata["priority"])
taghandle = self.db.add_tag(tag,self.trans)
self.tags[tagname] = taghandle
return taghandle
def applyTagClicked(self, button):
progress = None
rows = self.treeSelection.count_selected_rows()
tag_name = str(self.tagcombo.get_active_text())
# start the db transaction
with DbTxn("Tag not related", self.db) as transaction:
tag = self.db.get_tag_from_name(tag_name)
if not tag:
# create the tag if it doesn't already exist
tag = Tag()
tag.set_name(tag_name)
tag.set_priority(self.db.get_number_of_tags())
tag_handle = self.db.add_tag(tag, transaction)
else:
tag_handle = tag.get_handle()
# if more than 1 person is selected, use a progress indicator
if rows > 1:
progress = ProgressMeter(self.title, _('Starting'))
progress.set_pass(
# translators: leave all/any {...} untranslated
#TRANS: no singular form needed, as rows is always > 1
ngettext("Setting tag for {number_of} person",
"Setting tag for {number_of} people",
rows).format(number_of=rows),
rows)
# iterate through all of the selected rows
(model, paths) = self.treeSelection.get_selected_rows()
for path in paths:
if progress:
progress.step()
# for the current row, get the GID and the person from the database
iter = self.model.get_iter(path)
personGid = self.model.get_value(iter, 1)
person = self.db.get_person_from_gramps_id(personGid)
# add the tag to the person
person.add_tag(tag_handle)
# save this change
self.db.commit_person(person, transaction)
# refresh the tags column
self.treeView.set_model(None)
for path in paths:
iter = self.model.get_iter(path)
personGid = self.model.get_value(iter, 1)
person = self.db.get_person_from_gramps_id(personGid)
self.model.set_value(iter, 3, self.get_tag_list(person))
self.treeView.set_model(self.model)
self.treeView.expand_all()
if progress:
progress.close()
def create_tag(self, tag_name, tag_color):
"""
Create a tag if it doesn't already exist.
"""
tag = self.db.get_tag_from_name(tag_name)
if tag is None:
tag = Tag()
tag.set_name(tag_name)
if tag_color is not None:
tag.set_color(tag_color)
tag.set_priority(self.db.get_number_of_tags())
tag_handle = self.db.add_tag(tag, self.trans)
else:
tag_handle = tag.get_handle()
return tag_handle
def cb_add_clicked(self, button, top):
"""
Create a new tag.
"""
tag = Tag()
tag.set_priority(self.db.get_number_of_tags())
EditTag(self.db, self.uistate, self.track, tag)
if tag.get_handle():
self.namemodel.add((tag.get_priority(),
tag.get_handle(),
tag.get_name(),
tag.get_color()))
def get_tag_from_name(self, name):
"""
Find a Tag in the database from the passed Tag name.
If no such Tag exists, None is returned.
"""
self.dbapi.execute("SELECT blob_data FROM tag WHERE name = ?", [name])
row = self.dbapi.fetchone()
if row:
return Tag.create(pickle.loads(row[0]))
return None
def get_tag_from_name(self, name):
"""
Find a Tag in the database from the passed Tag name.
If no such Tag exists, None is returned.
"""
self.dbapi.execute("SELECT blob_data FROM tag WHERE name = ?", [name])
row = self.dbapi.fetchone()
if row:
return Tag.create(pickle.loads(row[0]))
return None
def cb_add_clicked(self, button, top):
"""
Create a new tag.
"""
tag = Tag()
tag.set_priority(self.db.get_number_of_tags())
edit_dialog = EditTag(self.db, top, tag)
edit_dialog.run()
if tag.get_handle():
self.namemodel.add((tag.get_priority(),
tag.get_handle(),
tag.get_name(),
tag.get_color()))
def create_tag(self, tag_name, tag_color):
"""
Create a tag if it doesn't already exist.
"""
tag = self.db.get_tag_from_name(tag_name)
if tag is None:
tag = Tag()
tag.set_name(tag_name)
if tag_color is not None:
tag.set_color(tag_color)
tag.set_priority(self.db.get_number_of_tags())
tag_handle = self.db.add_tag(tag, self.trans)
else:
tag_handle = tag.get_handle()
return tag_handle
def checkTagExistence(self, otext):
with DbTxn(_("Read Tag"), self.db):
tag = self.db.get_tag_from_name(otext)
if tag != None:
LOG.debug('Tag found by name, no duplicates: ' + otext + ' ' + tag.get_name())
else:
tag = Tag()
tag.set_name(otext)
tag.set_color("#EF2929")
with DbTxn(_("Add Tag"), self.db) as trans:
thandle = self.db.add_tag(tag, trans)
LOG.debug('Tag added: ' + tag.get_name() + ' ' + thandle)
return tag
def cb_new_tag(self, action):
"""
Create a new tag and tag the selected objects.
"""
tag = Tag()
tag.set_priority(self.db.get_number_of_tags())
EditTag(self.db, self.uistate, [], tag)
if tag.get_handle():
self.tag_selected_rows(tag.get_handle())
def __checkTagExistence(self, otext, color):
tag = self.dbstate.db.get_tag_from_name(otext)
if tag != None:
#print('Tag found by name, no duplicates: ' + otext + ' ' + tag.get_name())
pass
else:
tag = Tag()
tag.set_name(otext)
tag.set_color(color)
with DbTxn(_("Add Tag"), self.dbstate.db) as trans:
thandle = self.dbstate.db.add_tag(tag, trans)
tag = self.dbstate.db.get_tag_from_name(otext)
self.dbstate.db.commit_tag(tag, trans)
print('Tag added: ' + tag.get_name() + ' ' + thandle)
return tag
def on_ok_clicked(self, obj):
clean = self.clear_button.get_active()
copy = self.copy_button.get_active()
tag = self.tag_button.get_active()
if not (copy or clean or tag):
return
self.db.disable_signals()
if tag:
tag_name = _('Legacy place')
# start the db transaction
with DbTxn("Tag for place titles", self.db) as self.tag_trans:
mark = self.db.get_tag_from_name(tag_name)
if not mark:
# create the tag if it doesn't already exist
mark = Tag()
mark.set_name(tag_name)
mark.set_priority(self.db.get_number_of_tags())
tag_handle = self.db.add_tag(mark, self.tag_trans)
else:
tag_handle = mark.get_handle()
with DbTxn(_("Modify Place titles"), self.db, batch=True) as trans:
for row in self.model:
if row[1] == True:
place = self.db.get_place_from_handle(row[0])
if copy:
self.create_note(place, row[2], trans)
if clean:
place.set_title("")
if tag:
place.add_tag(tag_handle)
self.db.commit_place(place, trans)
self.db.enable_signals()
self.db.request_rebuild()
self.close()
def on_ok_clicked(self, obj):
clean = self.clear_button.get_active()
copy = self.copy_button.get_active()
tag = self.tag_button.get_active()
if not (copy or clean or tag):
return
self.db.disable_signals()
if tag:
tag_name = _('Legacy place')
# start the db transaction
with DbTxn("Tag for place titles", self.db) as self.tag_trans:
mark = self.db.get_tag_from_name(tag_name)
if not mark:
# create the tag if it doesn't already exist
mark = Tag()
mark.set_name(tag_name)
mark.set_priority(self.db.get_number_of_tags())
tag_handle = self.db.add_tag(mark, self.tag_trans)
else:
tag_handle = mark.get_handle()
with DbTxn(_("Modify Place titles"), self.db, batch=True) as trans:
for row in self.model:
if row[1] == True:
place = self.db.get_place_from_handle(row[0])
if copy:
self.create_note(place, row[2], trans)
if clean:
place.set_title("")
if tag:
place.add_tag(tag_handle)
self.db.commit_place(place, trans)
self.db.enable_signals()
self.db.request_rebuild()
self.close()
def importSources(db, filename, user):
def findNextRidno(ridstrt):
with DbTxn(_("Find next ridno"), db):
db.set_repository_id_prefix(ridstrt + '%03d')
next_ridno = db.find_next_repository_gramps_id()
LOG.debug('Next ridno = ' + next_ridno)
db.set_repository_id_prefix('R%04d')
return next_ridno
def findNextSidno(ridno):
with DbTxn(_("Find next sidno"), db):
db.set_source_id_prefix(ridno + '%03d')
next_sidno = db.find_next_source_gramps_id()
LOG.debug('Next sidno = ' + next_sidno)
db.set_source_id_prefix('S%04d')
return next_sidno
fdir = os.path.dirname(filename)
'''
fh = logging.FileHandler(fdir + '\\sourceimport.log')
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
LOG.addHandler(fh)
'''
LOG.info(" fdir = " + fdir)
cout = fdir + "\\result0.csv"
LOG.debug('ini file handling')
config = configman.register_manager("importsources")
'''
config.register("options.repositoryidrng", "1000")
config.register("options.repositoryincr", "1")
config.register("options.sourceidrng", "1000")
config.register("options.sourceidincr", "1")
'''
config.register("options.refstring", "r")
config.load()
config.save()
'''
repository_idrange = int(config.get('options.repositoryidrng'))
repository_incr = int(config.get('options.repositoryincr'))
source_idrange = int(config.get('options.sourceidrng'))
source_idincr = int(config.get('options.sourceidincr'))
'''
refstr = config.get('options.refstring')
'''
repository_idno = 0
source_idno = 0
'''
t_count = 0
r_count = 0
s_count = 0
c_count = 0
u_count = 0
ridno = None
sidno = None
tags = {} # Dictionary recordtype: tag
chgtime = int(time.time())
LOG.info(" chgtime = " + str(chgtime))
try:
with open(cout, 'w', newline='\n', encoding="utf-8") as csv_out:
r_writer = csv.writer(csv_out, delimiter=';')
with open(filename, 'r', encoding="utf-8") as t_in:
rhandle = None
t_dialect = csv.Sniffer().sniff(t_in.read(1024))
t_in.seek(0)
t_reader = csv.reader(t_in, t_dialect)
LOG.info('CSV input file delimiter is ' + t_dialect.delimiter)
for row in t_reader:
rectype = row[
0] # Record type = Gramps object id prefix character
LOG.debug('Row type: -' + row[0] + '-')
if rectype == '#':
LOG.debug('Comment row: ' + row[0])
c_count += 1
r_writer.writerow([
row[0], row[1], row[2], row[3], row[4], row[5],
row[6], row[7]
])
else:
idno = row[
2] # Possibly previously assigned Gramps object id
handle = row[3].strip(
'"'
) # Possibly previously assigned Gramps object handle
otext = row[4].strip()
LOG.debug('Handle = ' + handle)
if rectype == 'T':
LOG.debug('Tag row: ' + row[0])
thandle = ''
t_count += 1
recobj = row[
1] # Tag related to repositories or sources
tag = None
with DbTxn(_("Read Tag"), db):
tag = db.get_tag_from_name(otext)
if tag != None:
LOG.info('Tag found by name, no duplicates: ' +
otext + ' ' + tag.get_name())
thandle = tag.get_handle()
else:
tag = Tag()
tag.set_name(otext)
tag.set_change_time(chgtime)
tag.set_color("#EF2929")
with DbTxn(_("Add Tag"), db) as trans:
thandle = db.add_tag(tag, trans)
LOG.info('Tag added: ' + tag.get_name() +
' ' + thandle)
tags[recobj] = tag
try:
r_writer.writerow([
rectype, recobj, '', '"' + thandle + '"',
otext, '', '', '', ''
])
except IOError:
LOG.error('Error writing T-csv ' +
IOError.strerror)
elif rectype == 'R':
LOG.debug('Repository row: ' + row[0])
rhandle = ''
# source_idno = 0
r_count += 1
repotype = row[1] # repository type number
if idno == '':
# repository type based numbering should be applied but not supplied by Gramps
ridno = findNextRidno(rectype + 'r' + repotype)
# repository_idno = repository_idno + repository_incr
# ridno = rectype + refstr + str(int(repotype) * repository_idrange + repository_idno)
else:
ridno = idno
LOG.debug('Ridno = ' + str(ridno))
repository = Repository()
if handle != '':
with DbTxn(_("Read Repository"), db) as trans:
repository = db.get_repository_from_handle(
handle)
if repository == None:
LOG.error(
'Repository NOT found by handle: '
+ handle + ' ' + otext)
raise GrampsImportError(
'Repository NOT found by handle: ',
handle + '/' + otext)
repositoryType = RepositoryType()
repositoryType.set(int(repotype))
repository.set_type(repositoryType)
repository.set_gramps_id(ridno)
repository.set_name(otext)
repository.set_change_time(chgtime)
if tags.get(rectype) != None:
repository.add_tag(tags[rectype].get_handle())
if handle == '':
with DbTxn(_("Add Repository"), db) as trans:
rhandle = db.add_repository(
repository, trans)
else:
with DbTxn(_("Update Repository"),
db) as trans:
db.commit_repository(repository, trans)
rhandle = handle
try:
r_writer.writerow([
rectype, repotype, ridno,
'"' + rhandle + '"', otext, '', '', '', ''
])
except IOError:
LOG.error('Error writing R-csv ' +
IOError.strerror)
elif rectype == 'S':
LOG.debug('Source row: ' + row[0])
shandle = ''
sidno = ''
s_count += 1
attribs = (row[5], row[6], row[7])
if idno == '':
LOG.debug('Ridno for sidno = ' + str(ridno))
sidno = findNextSidno(ridno)
# source_idno = source_idno + source_idincr
# sidno = rectype + refstr + str((int(repotype) * repository_idrange + repository_idno) * source_idrange + source_idno)
else:
sidno = idno
LOG.debug('Sidno = ' + str(sidno))
source = Source()
if handle != '':
with DbTxn(_("Read Source"), db) as trans:
source = db.get_source_from_handle(handle)
if source == None:
LOG.error(
'Source NOT found by handle: ' +
handle + ' ' + otext)
raise GrampsImportError(
'Source NOT found by handle: ',
handle + '/' + otext)
source.set_gramps_id(sidno)
source.set_title(otext)
source.set_author(attribs[0])
source.set_publication_info(attribs[1])
source.set_abbreviation(attribs[2])
if tags.get(rectype) != None:
source.add_tag(tags[rectype].get_handle())
repoRef = RepoRef()
repoRef.set_reference_handle(rhandle)
source.add_repo_reference(repoRef)
source.set_change_time(chgtime)
if handle == '':
with DbTxn(_("Add Source"), db) as trans:
shandle = db.add_source(source, trans)
else:
with DbTxn(_("Update Source"), db) as trans:
db.commit_source(source, trans)
shandle = handle
try:
r_writer.writerow([
rectype, '', sidno, '"' + shandle + '"',
otext, attribs[0], attribs[1], attribs[2],
''
])
except IOError:
LOG.error('Error writing S-csv ' +
IOError.strerror)
else:
u_count += 1
LOG.debug('Unknown rectype: ' + rectype)
raise GrampsImportError('Unknown record type ' +
rectype)
except:
exc = sys.exc_info()[0]
LOG.error('*** Something went really wrong! ', exc)
return ImportInfo({_('Results'): _('Something went really wrong ')})
results = {
_('Results'):
_('Input file handled.'),
_(' Tags '):
str(t_count),
_(' Repositories '):
str(r_count),
_(' Comments '):
str(c_count),
_(' Unknown types '):
str(u_count),
_(' Total '):
str(t_count + r_count + s_count + c_count + u_count)
}
LOG.info('Input file handled.')
LOG.info(' Tags ' + str(t_count))
LOG.info(' Repositories ' + str(r_count))
LOG.info(' Sources ' + str(s_count))
LOG.info(' Comments ' + str(c_count))
LOG.info(' Unknown types ' + str(u_count))
LOG.info(' Total ' +
str(t_count + r_count + s_count + c_count + u_count))
db.enable_signals()
db.request_rebuild()
return ImportInfo(results)
def make_tag(self, tag):
data = self.dji.get_tag(tag)
return Tag.create(data)
def exportData(database, filename,
error_dialog=None, option_box=None, callback=None):
if not callable(callback):
callback = lambda percent: None # dummy
with OpenFileOrStdout(filename, encoding="utf-8") as fp:
total = (len(database.note_map) +
len(database.person_map) +
len(database.event_map) +
len(database.family_map) +
len(database.repository_map) +
len(database.place_map) +
len(database.media_map) +
len(database.citation_map) +
len(database.source_map) +
len(database.tag_map))
count = 0.0
# ---------------------------------
# Notes
# ---------------------------------
for handle in database.note_map.keys():
serial = database.note_map[handle]
write_line(fp, Note.create(serial))
count += 1
callback(100 * count/total)
# ---------------------------------
# Event
# ---------------------------------
for handle in database.event_map.keys():
serial = database.event_map[handle]
write_line(fp, Event.create(serial))
count += 1
callback(100 * count/total)
# ---------------------------------
# Person
# ---------------------------------
for handle in database.person_map.keys():
serial = database.person_map[handle]
write_line(fp, Person.create(serial))
count += 1
callback(100 * count/total)
# ---------------------------------
# Family
# ---------------------------------
for handle in database.family_map.keys():
serial = database.family_map[handle]
write_line(fp, Family.create(serial))
count += 1
callback(100 * count/total)
# ---------------------------------
# Repository
# ---------------------------------
for handle in database.repository_map.keys():
serial = database.repository_map[handle]
write_line(fp, Repository.create(serial))
count += 1
callback(100 * count/total)
# ---------------------------------
# Place
# ---------------------------------
for handle in database.place_map.keys():
serial = database.place_map[handle]
write_line(fp, Place.create(serial))
count += 1
callback(100 * count/total)
# ---------------------------------
# Source
# ---------------------------------
for handle in database.source_map.keys():
serial = database.source_map[handle]
write_line(fp, Source.create(serial))
count += 1
callback(100 * count/total)
# ---------------------------------
# Citation
# ---------------------------------
for handle in database.citation_map.keys():
serial = database.citation_map[handle]
write_line(fp, Citation.create(serial))
count += 1
callback(100 * count/total)
# ---------------------------------
# Media
# ---------------------------------
for handle in database.media_map.keys():
serial = database.media_map[handle]
write_line(fp, Media.create(serial))
count += 1
callback(100 * count/total)
# ---------------------------------
# Tag
# ---------------------------------
for handle in database.tag_map.keys():
serial = database.tag_map[handle]
write_line(fp, Tag.create(serial))
count += 1
callback(100 * count/total)
return True
def __init__(self, dbase, user, default_tag_format=None):
self.db = dbase
self.user = user
self.trans = None
self.lineno = 0
self.index = 0
self.fam_count = 0
self.indi_count = 0
self.pref = {} # person ref, internal to this sheet
self.fref = {} # family ref, internal to this sheet
column2label = {
"surname": ("Lastname", "Surname", _("Surname"), "lastname",
"last_name", "surname", _("surname")),
"firstname": ("Firstname", "Given name", _("Given name"), "Given",
_("Given"), "firstname", "first_name", "given_name",
"given name", _("given name"), "given", _("given")),
"callname":
("Callname", "Call name", _("Call name"), "Call", _("Call"),
"callname", "call_name", "call name", "call", _("call")),
"title": ("Title", _("Person|Title"), "title", _("Person|title")),
"prefix": ("Prefix", _("Prefix"), "prefix", _("prefix")),
"suffix": ("Suffix", _("Suffix"), "suffix", _("suffix")),
"gender": ("Gender", _("Gender"), "gender", _("gender")),
"source": ("Source", _("Source"), "source", _("source")),
"note": ("Note", _("Note"), "note", _("note")),
"birthplace":
("Birthplace", "Birth place", _("Birth place"), "birthplace",
"birth_place", "birth place", _("birth place")),
"birthdate": ("Birthdate", "Birth date", _("Birth date"),
"birthdate", "birth_date", "birth date",
_("birth date")),
"birthsource": ("Birthsource", "Birth source", _("Birth source"),
"birthsource", "birth_source", "birth source",
_("birth source")),
"baptismplace":
("Baptismplace", "Baptism place", _("Baptism place"),
"baptismplace", "baptism place", _("baptism place")),
"baptismdate": ("Baptismdate", "Baptism date", _("Baptism date"),
"baptismdate", "baptism date", _("baptism date")),
"baptismsource": ("Baptismsource", "Baptism source",
_("Baptism source"), "baptismsource",
"baptism source", _("baptism source")),
"burialplace": ("Burialplace", "Burial place", _("Burial place"),
"burialplace", "burial place", _("burial place")),
"burialdate": ("Burialdate", "Burial date", _("Burial date"),
"burialdate", "burial date", _("burial date")),
"burialsource":
("Burialsource", "Burial source", _("Burial source"),
"burialsource", "burial source", _("burial source")),
"deathplace": ("Deathplace", "Death place", _("Death place"),
"deathplace", "death_place", "death place",
_("death place")),
"deathdate": ("Deathdate", "Death date", _("Death date"),
"deathdate", "death_date", "death date",
_("death date")),
"deathsource": ("Deathsource", "Death source", _("Death source"),
"deathsource", "death_source", "death source",
_("death source")),
"deathcause": ("Deathcause", "Death cause", _("Death cause"),
"deathcause", "death_cause", "death cause",
_("death cause")),
"grampsid": ("Grampsid", "ID", "Gramps id", _("Gramps ID"),
"grampsid", "id", "gramps_id", "gramps id",
_("Gramps id")),
"person": ("Person", _("Person"), "person", _("person")),
# ----------------------------------
"child": ("Child", _("Child"), "child", _("child")),
"family": ("Family", _("Family"), "family", _("family")),
# ----------------------------------
"wife": ("Mother", _("Mother"), "Wife", _("Wife"), "Parent2",
_("Parent2"), "mother", _("mother"), "wife", _("wife"),
"parent2", _("parent2")),
"husband": ("Father", _("Father"), "Husband", _("Husband"),
"Parent1", _("Parent1"), "father", _("father"),
"husband", _("husband"), "parent1", _("parent1")),
"marriage": ("Marriage", _("Marriage"), "marriage", _("marriage")),
"date": ("Date", _("Date"), "date", _("date")),
"place": ("Place", _("Place"), "place", _("place")),
}
lab2col_dict = []
for key in list(column2label.keys()):
for val in column2label[key]:
lab2col_dict.append((val, key))
self.label2column = dict(lab2col_dict)
if default_tag_format:
name = time.strftime(default_tag_format)
tag = self.db.get_tag_from_name(name)
if tag:
self.default_tag = tag
else:
self.default_tag = Tag()
self.default_tag.set_name(name)
else:
self.default_tag = None
def _process(self, count, total, sql):
# ---------------------------------
# Process note
# ---------------------------------
notes = sql.query("""select * from note;""")
for note in notes:
(handle, gid, text, _format, note_type1, note_type2, change,
private) = note
styled_text = [text, []]
markups = sql.query(
"""select * from link where from_handle = ? """
"""and to_type = 'markup';""", handle)
for markup_link in markups:
_from_type, _from_handle, _to_type, to_handle = markup_link
markup_detail = sql.query(
"""select * from markup where handle = ?;""", to_handle)
for markup in markup_detail:
(_mhandle, markup0, markup1, value,
start_stop_list) = markup
ss_list = eval(start_stop_list)
styled_text[1] += [((markup0, markup1), value, ss_list)]
tags = self.get_links(sql, "note", handle, "tag")
g_note = Note()
g_note.unserialize(
(handle, gid, styled_text, _format, (note_type1, note_type2),
change, tags, bool(private)))
self.db.add_note(g_note, self.trans)
count += 1
self.callback(100 * count / total)
# ---------------------------------
# Process event
# ---------------------------------
events = sql.query("""select * from event;""")
for event in events:
(handle, gid, the_type0, the_type1, description, change,
private) = event
note_list = self.get_note_list(sql, "event", handle)
citation_list = self.get_citation_list(sql, "event", handle)
media_list = self.get_media_list(sql, "event", handle)
attribute_list = self.get_attribute_list(sql, "event", handle)
date_handle = self.get_link(sql, "event", handle, "date")
date = self.get_date(sql, date_handle)
place_handle = self.get_link(sql, "event", handle, "place")
place = self.get_place_from_handle(sql, place_handle)
tags = self.get_links(sql, "event", handle, "tag")
data = (handle, gid, (the_type0, the_type1), date, description,
place, citation_list, note_list, media_list,
attribute_list, change, tags, bool(private))
g_event = Event()
g_event.unserialize(data)
self.db.add_event(g_event, self.trans)
count += 1
self.callback(100 * count / total)
# ---------------------------------
# Process person
# ---------------------------------
people = sql.query("""select * from person;""")
for person in people:
if person is None:
continue
(
handle, # 0
gid, # 1
gender, # 2
death_ref_handle, # 5
birth_ref_handle, # 6
change, # 17
private, # 19
) = person
primary_name = self.get_names(sql, "person", handle, True) # one
alternate_names = self.get_names(sql, "person", handle, False)
event_ref_list = self.get_event_ref_list(sql, "person", handle)
family_list = self.get_family_list(sql, "person", handle)
parent_family_list = self.get_parent_family_list(
sql, "person", handle)
media_list = self.get_media_list(sql, "person", handle)
address_list = self.get_address_list(sql,
"person",
handle,
with_parish=False)
attribute_list = self.get_attribute_list(sql, "person", handle)
urls = self.get_url_list(sql, "person", handle)
lds_ord_list = self.get_lds_list(sql, "person", handle)
pcitation_list = self.get_citation_list(sql, "person", handle)
pnote_list = self.get_note_list(sql, "person", handle)
person_ref_list = self.get_person_ref_list(sql, "person", handle)
death_ref_index = lookup(death_ref_handle, event_ref_list)
birth_ref_index = lookup(birth_ref_handle, event_ref_list)
tags = self.get_links(sql, "person", handle, "tag")
data = (
handle, # 0
gid, # 1
gender, # 2
primary_name, # 3
alternate_names, # 4
death_ref_index, # 5
birth_ref_index, # 6
event_ref_list, # 7
family_list, # 8
parent_family_list, # 9
media_list, # 10
address_list, # 11
attribute_list, # 12
urls, # 13
lds_ord_list, # 14
pcitation_list, # 15
pnote_list, # 16
change, # 17
tags,
bool(private), # 19
person_ref_list,
) # 20
g_pers = Person()
g_pers.unserialize(data)
self.db.add_person(g_pers, self.trans)
count += 1
self.callback(100 * count / total)
# ---------------------------------
# Process family
# ---------------------------------
families = sql.query("""select * from family;""")
for family in families:
(handle, gid, father_handle, mother_handle, the_type0, the_type1,
change, private) = family
child_ref_list = self.get_child_ref_list(sql, "family", handle)
event_ref_list = self.get_event_ref_list(sql, "family", handle)
media_list = self.get_media_list(sql, "family", handle)
attribute_list = self.get_attribute_list(sql, "family", handle)
lds_seal_list = self.get_lds_list(sql, "family", handle)
citation_list = self.get_citation_list(sql, "family", handle)
note_list = self.get_note_list(sql, "family", handle)
tags = self.get_links(sql, "family", handle, "tag")
data = (handle, gid, father_handle, mother_handle, child_ref_list,
(the_type0, the_type1), event_ref_list, media_list,
attribute_list, lds_seal_list, citation_list, note_list,
change, tags, private)
g_fam = Family()
g_fam.unserialize(data)
self.db.add_family(g_fam, self.trans)
count += 1
self.callback(100 * count / total)
# ---------------------------------
# Process repository
# ---------------------------------
repositories = sql.query("""select * from repository;""")
for repo in repositories:
(handle, gid, the_type0, the_type1, name, change, private) = repo
note_list = self.get_note_list(sql, "repository", handle)
address_list = self.get_address_list(sql,
"repository",
handle,
with_parish=False)
urls = self.get_url_list(sql, "repository", handle)
tags = self.get_links(sql, "repository", handle, "tag")
data = (handle, gid, (the_type0, the_type1), name, note_list,
address_list, urls, change, tags, private)
g_rep = Repository()
g_rep.unserialize(data)
self.db.add_repository(g_rep, self.trans)
count += 1
self.callback(100 * count / total)
# ---------------------------------
# Process place
# ---------------------------------
places = sql.query("""select * from place;""")
for place in places:
count += 1
(handle, gid, title, value, the_type0, the_type1, code, long, lat,
lang, change, private) = place
# We could look this up by "place_main", but we have the handle:
#main_loc = self.get_main_location(sql, handle, with_parish=True)
alt_loc_list = self.get_location_list(sql,
"place_alt",
handle,
with_parish=True)
urls = self.get_url_list(sql, "place", handle)
media_list = self.get_media_list(sql, "place", handle)
citation_list = self.get_citation_list(sql, "place", handle)
note_list = self.get_note_list(sql, "place", handle)
tags = self.get_links(sql, "place", handle, "tag")
place_type = (the_type0, the_type1)
alt_place_name_list = self.get_alt_place_name_list(sql, handle)
place_ref_list = self.get_place_ref_list(sql, handle)
data = (handle, gid, title, long, lat, place_ref_list,
PlaceName(value=value,
lang=lang).serialize(), alt_place_name_list,
place_type, code, alt_loc_list, urls, media_list,
citation_list, note_list, change, tags, private)
g_plac = Place()
g_plac.unserialize(data)
self.db.commit_place(g_plac, self.trans)
self.callback(100 * count / total)
# ---------------------------------
# Process citation
# ---------------------------------
citations = sql.query("""select * from citation;""")
for citation in citations:
(handle, gid, confidence, page, source_handle, change,
private) = citation
date_handle = self.get_link(sql, "citation", handle, "date")
date = self.get_date(sql, date_handle)
note_list = self.get_note_list(sql, "citation", handle)
media_list = self.get_media_list(sql, "citation", handle)
datamap = self.get_datamap_list(sql, "citation", handle)
tags = self.get_links(sql, "citation", handle, "tag")
data = (handle, gid, date, page, confidence, source_handle,
note_list, media_list, datamap, change, tags, private)
g_cit = Citation()
g_cit.unserialize(data)
self.db.commit_citation(g_cit, self.trans)
count += 1
self.callback(100 * count / total)
# ---------------------------------
# Process source
# ---------------------------------
sources = sql.query("""select * from source;""")
for source in sources:
(handle, gid, title, author, pubinfo, abbrev, change,
private) = source
note_list = self.get_note_list(sql, "source", handle)
media_list = self.get_media_list(sql, "source", handle)
datamap = self.get_datamap_list(sql, "source", handle)
reporef_list = self.get_repository_ref_list(sql, "source", handle)
tags = self.get_links(sql, "source", handle, "tag")
data = (handle, gid, title, author, pubinfo, note_list, media_list,
abbrev, change, datamap, reporef_list, tags, private)
g_src = Source()
g_src.unserialize(data)
self.db.commit_source(g_src, self.trans)
count += 1
self.callback(100 * count / total)
# ---------------------------------
# Process media
# ---------------------------------
media = sql.query("""select * from media;""")
for med in media:
(handle, gid, path, mime, desc, checksum, change, private) = med
attribute_list = self.get_attribute_list(sql, "media", handle)
citation_list = self.get_citation_list(sql, "media", handle)
note_list = self.get_note_list(sql, "media", handle)
date_handle = self.get_link(sql, "media", handle, "date")
date = self.get_date(sql, date_handle)
tags = self.get_links(sql, "media", handle, "tag")
data = (handle, gid, path, mime, desc, checksum, attribute_list,
citation_list, note_list, change, date, tags, private)
g_med = Media()
g_med.unserialize(data)
self.db.commit_media(g_med, self.trans)
count += 1
self.callback(100 * count / total)
# ---------------------------------
# Process tag
# ---------------------------------
tags = sql.query("""select * from tag;""")
for tag in tags:
(handle, name, color, priority, change) = tag
data = (handle, name, color, priority, change)
g_tag = Tag()
g_tag.unserialize(data)
self.db.commit_tag(g_tag, self.trans)
count += 1
self.callback(100 * count / total)
def __init__(self, dbase, user, default_tag_format=None):
self.db = dbase
self.user = user
self.trans = None
self.lineno = 0
self.index = 0
self.fam_count = 0
self.indi_count = 0
self.pref = {} # person ref, internal to this sheet
self.fref = {} # family ref, internal to this sheet
column2label = {
"surname": ("Lastname", "Surname", _("Surname"), "lastname",
"last_name", "surname", _("surname")),
"firstname": ("Firstname", "Given name", _("Given name"), "Given",
_("Given"), "firstname", "first_name", "given_name",
"given name", _("given name"), "given", _("given")),
"callname": ("Callname", "Call name", _("Call name"), "Call",
_("Call"), "callname", "call_name", "call name", "call",
_("call")),
"title": ("Title", _("Person|Title"), "title", _("Person|title")),
"prefix": ("Prefix", _("Prefix"), "prefix", _("prefix")),
"suffix": ("Suffix", _("Suffix"), "suffix", _("suffix")),
"gender": ("Gender", _("Gender"), "gender", _("gender")),
"source": ("Source", _("Source"), "source", _("source")),
"note": ("Note", _("Note"), "note", _("note")),
"birthplace": ("Birthplace", "Birth place", _("Birth place"),
"birthplace", "birth_place", "birth place", _("birth place")),
"birthdate": ("Birthdate", "Birth date", _("Birth date"),
"birthdate", "birth_date", "birth date", _("birth date")),
"birthsource": ("Birthsource", "Birth source", _("Birth source"),
"birthsource", "birth_source", "birth source",
_("birth source")),
"baptismplace": ("Baptismplace", "Baptism place",
_("Baptism place"), "baptismplace", "baptism place",
_("baptism place")),
"baptismdate": ("Baptismdate", "Baptism date", _("Baptism date"),
"baptismdate", "baptism date", _("baptism date")),
"baptismsource": ("Baptismsource", "Baptism source",
_("Baptism source"), "baptismsource", "baptism source",
_("baptism source")),
"burialplace": ("Burialplace", "Burial place", _("Burial place"),
"burialplace", "burial place", _("burial place")),
"burialdate": ("Burialdate", "Burial date", _("Burial date"),
"burialdate", "burial date", _("burial date")),
"burialsource": ("Burialsource", "Burial source",
_("Burial source"), "burialsource", "burial source",
_("burial source")),
"deathplace": ("Deathplace", "Death place", _("Death place"),
"deathplace", "death_place", "death place", _("death place")),
"deathdate": ("Deathdate", "Death date", _("Death date"),
"deathdate", "death_date", "death date", _("death date")),
"deathsource": ("Deathsource", "Death source", _("Death source"),
"deathsource", "death_source", "death source",
_("death source")),
"deathcause": ("Deathcause", "Death cause", _("Death cause"),
"deathcause", "death_cause", "death cause", _("death cause")),
"grampsid": ("Grampsid", "ID", "Gramps id", _("Gramps ID"),
"grampsid", "id", "gramps_id", "gramps id", _("Gramps id")),
"person": ("Person", _("Person"), "person", _("person")),
# ----------------------------------
"child": ("Child", _("Child"), "child", _("child")),
"family": ("Family", _("Family"), "family", _("family")),
# ----------------------------------
"wife": ("Mother", _("Mother"), "Wife", _("Wife"), "Parent2",
_("Parent2"), "mother", _("mother"), "wife", _("wife"),
"parent2", _("parent2")),
"husband": ("Father", _("Father"), "Husband", _("Husband"),
"Parent1", _("Parent1"), "father", _("father"), "husband",
_("husband"), "parent1", _("parent1")),
"marriage": ("Marriage", _("Marriage"), "marriage", _("marriage")),
"date": ("Date", _("Date"), "date", _("date")),
"place": ("Place", _("Place"), "place", _("place")),
}
lab2col_dict = []
for key in list(column2label.keys()):
for val in column2label[key]:
lab2col_dict.append((val, key))
self.label2column = dict(lab2col_dict)
if default_tag_format:
name = time.strftime(default_tag_format)
tag = self.db.get_tag_from_name(name)
if tag:
self.default_tag = tag
else:
self.default_tag = Tag()
self.default_tag.set_name(name)
else:
self.default_tag = None
class CSVParser:
"""Class to read data in CSV format from a file object."""
def __init__(self, dbase, user, default_tag_format=None):
self.db = dbase
self.user = user
self.trans = None
self.lineno = 0
self.index = 0
self.fam_count = 0
self.indi_count = 0
self.place_count = 0
self.pref = {} # person ref, internal to this sheet
self.fref = {} # family ref, internal to this sheet
self.placeref = {}
self.place_types = {}
# Build reverse dictionary, name to type number
for items in PlaceType().get_map().items(): # (0, 'Custom')
self.place_types[items[1]] = items[0]
self.place_types[items[1].lower()] = items[0]
if _(items[1]) != items[1]:
self.place_types[_(items[1])] = items[0]
# Add custom types:
for custom_type in self.db.get_place_types():
self.place_types[custom_type] = 0
self.place_types[custom_type.lower()] = 0
column2label = {
"surname": ("Lastname", "Surname", _("Surname"), "lastname",
"last_name", "surname", _("surname")),
"firstname": ("Firstname", "Given name", _("Given name"), "Given",
_("Given"), "firstname", "first_name", "given_name",
"given name", _("given name"), "given", _("given")),
"callname": ("Callname", "Call name", _("Call name"), "Call",
_("Call"), "callname", "call_name", "call name", "call",
_("call")),
"title": ("Title", _("Person or Place|Title"), "title", _("Person or Place|title")),
"prefix": ("Prefix", _("Prefix"), "prefix", _("prefix")),
"suffix": ("Suffix", _("Suffix"), "suffix", _("suffix")),
"gender": ("Gender", _("Gender"), "gender", _("gender")),
"source": ("Source", _("Source"), "source", _("source")),
"note": ("Note", _("Note"), "note", _("note")),
"birthplace": ("Birthplace", "Birth place", _("Birth place"),
"birthplace", "birth_place", "birth place", _("birth place")),
"birthplace_id": ("Birthplaceid", "Birth place id", _("Birth place id"),
"birthplaceid", "birth_place_id", "birth place id", _("birth place id"),
"birthplace_id"),
"birthdate": ("Birthdate", "Birth date", _("Birth date"),
"birthdate", "birth_date", "birth date", _("birth date")),
"birthsource": ("Birthsource", "Birth source", _("Birth source"),
"birthsource", "birth_source", "birth source",
_("birth source")),
"baptismplace": ("Baptismplace", "Baptism place",
_("Baptism place"), "baptismplace", "baptism place",
_("baptism place")),
"baptismplace_id": ("Baptismplaceid", "Baptism place id",
_("Baptism place id"), "baptismplaceid", "baptism place id",
_("baptism place id"), "baptism_place_id",
"baptismplace_id"),
"baptismdate": ("Baptismdate", "Baptism date", _("Baptism date"),
"baptismdate", "baptism date", _("baptism date")),
"baptismsource": ("Baptismsource", "Baptism source",
_("Baptism source"), "baptismsource", "baptism source",
_("baptism source")),
"burialplace": ("Burialplace", "Burial place", _("Burial place"),
"burialplace", "burial place", _("burial place")),
"burialplace_id": ("Burialplaceid", "Burial place id", _("Burial place id"),
"burialplaceid", "burial place id", _("burial place id"),
"burial_place_id", "burialplace_id"),
"burialdate": ("Burialdate", "Burial date", _("Burial date"),
"burialdate", "burial date", _("burial date")),
"burialsource": ("Burialsource", "Burial source",
_("Burial source"), "burialsource", "burial source",
_("burial source")),
"deathplace": ("Deathplace", "Death place", _("Death place"),
"deathplace", "death_place", "death place", _("death place")),
"deathplace_id": ("Deathplaceid", "Death place id", _("Death place id"),
"deathplaceid", "death_place_id", "death place id", _("death place id"),
"death_place_id", "deathplace_id"),
"deathdate": ("Deathdate", "Death date", _("Death date"),
"deathdate", "death_date", "death date", _("death date")),
"deathsource": ("Deathsource", "Death source", _("Death source"),
"deathsource", "death_source", "death source",
_("death source")),
"deathcause": ("Deathcause", "Death cause", _("Death cause"),
"deathcause", "death_cause", "death cause", _("death cause")),
"grampsid": ("Grampsid", "ID", "Gramps id", _("Gramps ID"),
"grampsid", "id", "gramps_id", "gramps id", _("Gramps id")),
"person": ("Person", _("Person"), "person", _("person")),
# ----------------------------------
"child": ("Child", _("Child"), "child", _("child")),
"family": ("Family", _("Family"), "family", _("family")),
# ----------------------------------
"wife": ("Mother", _("Mother"), "Wife", _("Wife"), "Parent2",
_("Parent2"), "mother", _("mother"), "wife", _("wife"),
"parent2", _("parent2")),
"husband": ("Father", _("Father"), "Husband", _("Husband"),
"Parent1", _("Parent1"), "father", _("father"), "husband",
_("husband"), "parent1", _("parent1")),
"marriage": ("Marriage", _("Marriage"), "marriage", _("marriage")),
"date": ("Date", _("Date"), "date", _("date")),
"place": ("Place", _("Place"), "place", _("place")),
"place_id": ("Placeid", "place id", "Place id", "place_id", "placeid"),
"name": ("Name", _("Name"), "name", _("name")),
"type": ("Type", _("Type"), "type", _("type")),
"latitude": ("Latitude", _("latitude"), "latitude", _("latitude")),
"longitude": ("Longitude", _("Longitude"), "longitude", _("longitude")),
"code": ("Code", _("Code"), "code", _("code")),
"enclosed_by": ("Enclosed by", _("Enclosed by"), "enclosed by", _("enclosed by"),
"enclosed_by", _("enclosed_by"), "Enclosed_by", _("Enclosed_by"),
"enclosedby")
}
lab2col_dict = []
for key in list(column2label.keys()):
for val in column2label[key]:
lab2col_dict.append((val, key))
self.label2column = dict(lab2col_dict)
if default_tag_format:
name = time.strftime(default_tag_format)
tag = self.db.get_tag_from_name(name)
if tag:
self.default_tag = tag
else:
self.default_tag = Tag()
self.default_tag.set_name(name)
else:
self.default_tag = None
def cleanup_column_name(self, column):
"""Handle column aliases for CSV spreadsheet import and SQL."""
return self.label2column.get(column, column)
def read_csv(self, filehandle):
"Read the data from the file and return it as a list."
try:
data = [[r.strip() for r in row] for row in csv.reader(filehandle)]
except csv.Error as err:
self.user.notify_error(_('format error: line %(line)d: %(zero)s') % {
'line' : reader.line_num, 'zero' : err } )
return None
return data
def lookup(self, type_, id_):
"""
Return the object of type type_ with id id_ from db or previously
stored value.
"""
if id_ is None:
return None
if type_ == "family":
if id_.startswith("[") and id_.endswith("]"):
id_ = self.db.fid2user_format(id_[1:-1])
db_lookup = self.db.get_family_from_gramps_id(id_)
if db_lookup is None:
return self.lookup(type_, id_)
else:
return db_lookup
elif id_.lower() in self.fref:
return self.fref[id_.lower()]
else:
return None
elif type_ == "person":
if id_.startswith("[") and id_.endswith("]"):
id_ = self.db.id2user_format(id_[1:-1])
db_lookup = self.db.get_person_from_gramps_id(id_)
if db_lookup is None:
return self.lookup(type_, id_)
else:
return db_lookup
elif id_.lower() in self.pref:
return self.pref[id_.lower()]
else:
return None
elif type_ == "place":
if id_.startswith("[") and id_.endswith("]"):
id_ = self.db.id2user_format(id_[1:-1])
db_lookup = self.db.get_place_from_gramps_id(id_)
if db_lookup is None:
return self.lookup(type_, id_)
else:
return db_lookup
elif id_.lower() in self.placeref:
return self.placeref[id_.lower()]
else:
return None
else:
LOG.warn("invalid lookup type in CSV import: '%s'" % type_)
return None
def storeup(self, type_, id_, object_):
"Store object object_ of type type_ in a dictionary under key id_."
if id_.startswith("[") and id_.endswith("]"):
id_ = id_[1:-1]
#return # do not store gramps people; go look them up
if type_ == "person":
id_ = self.db.id2user_format(id_)
self.pref[id_.lower()] = object_
elif type_ == "family":
id_ = self.db.fid2user_format(id_)
self.fref[id_.lower()] = object_
elif type_ == "place":
id_ = self.db.pid2user_format(id_)
self.placeref[id_.lower()] = object_
else:
LOG.warn("invalid storeup type in CSV import: '%s'" % type_)
def parse(self, filehandle):
"""
Prepare the database and parse the input file.
:param filehandle: open file handle positioned at start of the file
"""
progress_title = _('CSV Import')
with self.user.progress(progress_title,
_('Reading data...'), 1) as step:
data = self.read_csv(filehandle)
with self.user.progress(progress_title,
_('Importing data...'), len(data)) as step:
tym = time.time()
self.db.disable_signals()
with DbTxn(_("CSV import"), self.db, batch=True) as self.trans:
if self.default_tag and self.default_tag.handle is None:
self.db.add_tag(self.default_tag, self.trans)
self._parse_csv_data(data, step)
self.db.enable_signals()
self.db.request_rebuild()
tym = time.time() - tym
# translators: leave all/any {...} untranslated
msg = ngettext('Import Complete: {number_of} second',
'Import Complete: {number_of} seconds', tym
).format(number_of=tym)
LOG.debug(msg)
LOG.debug("New Families: %d" % self.fam_count)
LOG.debug("New Individuals: %d" % self.indi_count)
def _parse_csv_data(self, data, step):
"""Parse each line of the input data and act accordingly."""
self.lineno = 0
self.index = 0
self.fam_count = 0
self.indi_count = 0
self.place_count = 0
self.pref = {} # person ref, internal to this sheet
self.fref = {} # family ref, internal to this sheet
self.placeref = {}
header = None
line_number = 0
for row in data:
step()
line_number += 1
if "".join(row) == "": # no blanks are allowed inside a table
header = None # clear headers, ready for next "table"
continue
######################################
if header is None:
header = [self.cleanup_column_name(r) for r in row]
col = {}
count = 0
for key in header:
col[key] = count
count += 1
continue
# four different kinds of data: person, family, and marriage
if (("marriage" in header) or
("husband" in header) or
("wife" in header)):
self._parse_marriage(line_number, row, col)
elif "family" in header:
self._parse_family(line_number, row, col)
elif "surname" in header:
self._parse_person(line_number, row, col)
elif "place" in header:
self._parse_place(line_number, row, col)
else:
LOG.warn("ignoring line %d" % line_number)
return None
def _parse_marriage(self, line_number, row, col):
"Parse the content of a Marriage,Husband,Wife line."
marriage_ref = rd(line_number, row, col, "marriage")
husband = rd(line_number, row, col, "husband")
wife = rd(line_number, row, col, "wife")
marriagedate = rd(line_number, row, col, "date")
marriageplace = rd(line_number, row, col, "place")
marriageplace_id = rd(line_number, row, col, "place_id")
marriagesource = rd(line_number, row, col, "source")
note = rd(line_number, row, col, "note")
wife = self.lookup("person", wife)
husband = self.lookup("person", husband)
if husband is None and wife is None:
# might have children, so go ahead and add
LOG.warn("no parents on line %d; adding family anyway" %
line_number)
family = self.get_or_create_family(marriage_ref, husband, wife)
# adjust gender, if not already provided
if husband:
# this is just a guess, if unknown
if husband.get_gender() == Person.UNKNOWN:
husband.set_gender(Person.MALE)
self.db.commit_person(husband, self.trans)
if wife:
# this is just a guess, if unknown
if wife.get_gender() == Person.UNKNOWN:
wife.set_gender(Person.FEMALE)
self.db.commit_person(wife, self.trans)
if marriage_ref:
self.storeup("family", marriage_ref.lower(), family)
if marriagesource:
# add, if new
new, marriagesource = self.get_or_create_source(marriagesource)
if marriageplace and marriageplace_id:
raise Error("Error in marriage: can't have a place and place_id")
if marriageplace:
# add, if new
new, marriageplace = self.get_or_create_place(marriageplace)
elif marriageplace_id:
# better exist already, locally or in database:
marriageplace = self.lookup("place", marriageplace_id)
if marriagedate:
marriagedate = _dp.parse(marriagedate)
if marriagedate or marriageplace or marriagesource or note:
# add, if new; replace, if different
new, marriage = self.get_or_create_event(family,
EventType.MARRIAGE, marriagedate,
marriageplace, marriagesource)
if new:
mar_ref = EventRef()
mar_ref.set_reference_handle(marriage.get_handle())
family.add_event_ref(mar_ref)
self.db.commit_family(family, self.trans)
# only add note to event:
# append notes, if previous notes
if note:
previous_notes_list = marriage.get_note_list()
updated_note = False
for note_handle in previous_notes_list:
previous_note = self.db.get_note_from_handle(
note_handle)
if previous_note.type == NoteType.EVENT:
previous_text = previous_note.get()
if note not in previous_text:
note = previous_text + "\n" + note
previous_note.set(note)
self.db.commit_note(previous_note, self.trans)
updated_note = True
break
if not updated_note:
# add new note here
new_note = Note()
new_note.handle = create_id()
new_note.type.set(NoteType.EVENT)
new_note.set(note)
if self.default_tag:
new_note.add_tag(self.default_tag.handle)
self.db.add_note(new_note, self.trans)
marriage.add_note(new_note.handle)
self.db.commit_event(marriage, self.trans)
def _parse_family(self, line_number, row, col):
"Parse the content of a family line"
family_ref = rd(line_number, row, col, "family")
if family_ref is None:
LOG.warn("no family reference found for family on line %d" %
line_number)
return # required
child = rd(line_number, row, col, "child")
source = rd(line_number, row, col, "source")
note = rd(line_number, row, col, "note")
gender = rd(line_number, row, col, "gender")
child = self.lookup("person", child)
family = self.lookup("family", family_ref)
if family is None:
LOG.warn("no matching family reference found for family "
"on line %d" % line_number)
return
if child is None:
LOG.warn("no matching child reference found for family "
"on line %d" % line_number)
return
# is this child already in this family? If so, don't add
LOG.debug("children: %s", [ref.ref for ref in
family.get_child_ref_list()])
LOG.debug("looking for: %s", child.get_handle())
if child.get_handle() not in [ref.ref for ref in
family.get_child_ref_list()]:
# add child to family
LOG.debug(" adding child [%s] to family [%s]",
child.get_gramps_id(), family.get_gramps_id())
childref = ChildRef()
childref.set_reference_handle(child.get_handle())
family.add_child_ref( childref)
self.db.commit_family(family, self.trans)
child.add_parent_family_handle(family.get_handle())
if gender:
# replace
gender = gender.lower()
if gender == gender_map[Person.MALE].lower():
gender = Person.MALE
elif gender == gender_map[Person.FEMALE].lower():
gender = Person.FEMALE
else:
gender = Person.UNKNOWN
child.set_gender(gender)
if source:
# add, if new
dummy_new, source = self.get_or_create_source(source)
self.find_and_set_citation(child, source)
# put note on child
if note:
# append notes, if previous notes
previous_notes_list = child.get_note_list()
updated_note = False
for note_handle in previous_notes_list:
previous_note = self.db.get_note_from_handle(note_handle)
if previous_note.type == NoteType.PERSON:
previous_text = previous_note.get()
if note not in previous_text:
note = previous_text + "\n" + note
previous_note.set(note)
self.db.commit_note(previous_note, self.trans)
updated_note = True
break
if not updated_note:
# add new note here
new_note = Note()
new_note.handle = create_id()
new_note.type.set(NoteType.PERSON)
new_note.set(note)
if self.default_tag:
new_note.add_tag(self.default_tag.handle)
self.db.add_note(new_note, self.trans)
child.add_note(new_note.handle)
self.db.commit_person(child, self.trans)
def _parse_person(self, line_number, row, col):
"Parse the content of a Person line."
surname = rd(line_number, row, col, "surname")
firstname = rd(line_number, row, col, "firstname", "")
callname = rd(line_number, row, col, "callname")
title = rd(line_number, row, col, "title")
prefix = rd(line_number, row, col, "prefix")
suffix = rd(line_number, row, col, "suffix")
gender = rd(line_number, row, col, "gender")
source = rd(line_number, row, col, "source")
note = rd(line_number, row, col, "note")
birthplace = rd(line_number, row, col, "birthplace")
birthplace_id = rd(line_number, row, col, "birthplace_id")
birthdate = rd(line_number, row, col, "birthdate")
birthsource = rd(line_number, row, col, "birthsource")
baptismplace = rd(line_number, row, col, "baptismplace")
baptismplace_id = rd(line_number, row, col, "baptismplace_id")
baptismdate = rd(line_number, row, col, "baptismdate")
baptismsource = rd(line_number, row, col, "baptismsource")
burialplace = rd(line_number, row, col, "burialplace")
burialplace_id = rd(line_number, row, col, "burialplace_id")
burialdate = rd(line_number, row, col, "burialdate")
burialsource = rd(line_number, row, col, "burialsource")
deathplace = rd(line_number, row, col, "deathplace")
deathplace_id = rd(line_number, row, col, "deathplace_id")
deathdate = rd(line_number, row, col, "deathdate")
deathsource = rd(line_number, row, col, "deathsource")
deathcause = rd(line_number, row, col, "deathcause")
grampsid = rd(line_number, row, col, "grampsid")
person_ref = rd(line_number, row, col, "person")
#########################################################
# if this person already exists, don't create them
person = self.lookup("person", person_ref)
if person is None:
if surname is None:
LOG.warn("empty surname for new person on line %d" %
line_number)
surname = ""
# new person
person = self.create_person()
name = Name()
name.set_type(NameType(NameType.BIRTH))
name.set_first_name(firstname)
surname_obj = Surname()
surname_obj.set_surname(surname)
name.add_surname(surname_obj)
person.set_primary_name(name)
else:
name = person.get_primary_name()
#########################################################
if person_ref is not None:
self.storeup("person", person_ref, person)
# replace
if surname is not None:
name.get_primary_surname().set_surname(surname)
if firstname is not None:
name.set_first_name(firstname)
if callname is not None:
name.set_call_name(callname)
if title is not None:
name.set_title(title)
if prefix is not None:
name.get_primary_surname().set_prefix(prefix)
name.group_as = '' # HELP? what should I do here?
if suffix is not None:
name.set_suffix(suffix)
if note is not None:
# append notes, if previous notes
previous_notes_list = person.get_note_list()
updated_note = False
for note_handle in previous_notes_list:
previous_note = self.db.get_note_from_handle(note_handle)
if previous_note.type == NoteType.PERSON:
previous_text = previous_note.get()
if note not in previous_text:
note = previous_text + "\n" + note
previous_note.set(note)
self.db.commit_note(previous_note, self.trans)
updated_note = True
break
if not updated_note:
# add new note here
new_note = Note()
new_note.handle = create_id()
new_note.type.set(NoteType.PERSON)
new_note.set(note)
if self.default_tag:
new_note.add_tag(self.default_tag.handle)
self.db.add_note(new_note, self.trans)
person.add_note(new_note.handle)
if grampsid is not None:
person.gramps_id = grampsid
elif person_ref is not None:
if person_ref.startswith("[") and person_ref.endswith("]"):
person.gramps_id = self.db.id2user_format(person_ref[1:-1])
if (person.get_gender() == Person.UNKNOWN and
gender is not None):
gender = gender.lower()
if gender == gender_map[Person.MALE].lower():
gender = Person.MALE
elif gender == gender_map[Person.FEMALE].lower():
gender = Person.FEMALE
else:
gender = Person.UNKNOWN
person.set_gender(gender)
#########################################################
# add if new, replace if different
# Birth:
if birthdate is not None:
birthdate = _dp.parse(birthdate)
if birthplace and birthplace_id:
raise Error("Error in person: can't have a birthplace and birthplace_id")
if birthplace is not None:
new, birthplace = self.get_or_create_place(birthplace)
elif birthplace_id:
# better exist already, locally or in database:
birthplace = self.lookup("place", birthplace_id)
if birthsource is not None:
new, birthsource = self.get_or_create_source(birthsource)
if birthdate or birthplace or birthsource:
new, birth = self.get_or_create_event(person,
EventType.BIRTH, birthdate,
birthplace, birthsource)
birth_ref = person.get_birth_ref()
if birth_ref is None:
# new
birth_ref = EventRef()
birth_ref.set_reference_handle( birth.get_handle())
person.set_birth_ref( birth_ref)
# Baptism:
if baptismdate is not None:
baptismdate = _dp.parse(baptismdate)
if baptismplace and baptismplace_id:
raise Error("Error in person: can't have a baptismplace and baptismplace_id")
if baptismplace is not None:
new, baptismplace = self.get_or_create_place(baptismplace)
elif baptismplace_id:
# better exist already, locally or in database:
baptismplace = self.lookup("place", baptismplace_id)
if baptismsource is not None:
new, baptismsource = self.get_or_create_source(baptismsource)
if baptismdate or baptismplace or baptismsource:
new, baptism = self.get_or_create_event(person,
EventType.BAPTISM, baptismdate,
baptismplace, baptismsource)
baptism_ref = get_primary_event_ref_from_type(self.db, person,
"Baptism")
if baptism_ref is None:
# new
baptism_ref = EventRef()
baptism_ref.set_reference_handle( baptism.get_handle())
person.add_event_ref( baptism_ref)
# Death:
if deathdate is not None:
deathdate = _dp.parse(deathdate)
if deathplace and deathplace_id:
raise Error("Error in person: can't have a deathplace and deathplace_id")
if deathplace is not None:
new, deathplace = self.get_or_create_place(deathplace)
elif deathplace_id:
# better exist already, locally or in database:
deathplace = self.lookup("place", deathplace_id)
if deathsource is not None:
new, deathsource = self.get_or_create_source(deathsource)
if deathdate or deathplace or deathsource or deathcause:
new, death = self.get_or_create_event(person,
EventType.DEATH, deathdate, deathplace,
deathsource)
if deathcause:
death.set_description(deathcause)
self.db.commit_event(death, self.trans)
death_ref = person.get_death_ref()
if death_ref is None:
# new
death_ref = EventRef()
death_ref.set_reference_handle(death.get_handle())
person.set_death_ref(death_ref)
# Burial:
if burialdate is not None:
burialdate = _dp.parse(burialdate)
if burialplace and burialplace_id:
raise Error("Error in person: can't have a burialplace and burialplace_id")
if burialplace is not None:
new, burialplace = self.get_or_create_place(burialplace)
elif burialplace_id:
# better exist already, locally or in database:
burialplace = self.lookup("place", burialplace_id)
if burialsource is not None:
new, burialsource = self.get_or_create_source(burialsource)
if burialdate or burialplace or burialsource:
new, burial = self.get_or_create_event(person,
EventType.BURIAL, burialdate,
burialplace, burialsource)
burial_ref = get_primary_event_ref_from_type(self.db, person,
"Burial")
if burial_ref is None:
# new
burial_ref = EventRef()
burial_ref.set_reference_handle( burial.get_handle())
person.add_event_ref( burial_ref)
if source:
# add, if new
new, source = self.get_or_create_source(source)
self.find_and_set_citation(person, source)
self.db.commit_person(person, self.trans)
def _parse_place(self, line_number, row, col):
"Parse the content of a Place line."
place_id = rd(line_number, row, col, "place")
place_title = rd(line_number, row, col, "title")
place_name = rd(line_number, row, col, "name")
place_type_str = rd(line_number, row, col, "type")
place_latitude = rd(line_number, row, col, "latitude")
place_longitude = rd(line_number, row, col, "longitude")
place_code = rd(line_number, row, col, "code")
place_enclosed_by_id = rd(line_number, row, col, "enclosed_by")
place_date = rd(line_number, row, col, "date")
#########################################################
# if this place already exists, don't create it
place = self.lookup("place", place_id)
if place is None:
# new place
place = self.create_place()
self.storeup("place", place_id.lower(), place)
if place_title is not None:
place.title = place_title
if place_name is not None:
place.name = PlaceName(value=place_name)
if place_type_str is not None:
place.place_type = self.get_place_type(place_type_str)
if place_latitude is not None:
place.lat = place_latitude
if place_longitude is not None:
place.long = place_longitude
if place_code is not None:
place.code = place_code
if place_enclosed_by_id is not None:
place_enclosed_by = self.lookup("place", place_enclosed_by_id)
if place_enclosed_by is None:
raise Exception("cannot enclose %s in %s as it doesn't exist" % (place.gramps_id, place_enclosed_by_id))
if not place_enclosed_by.handle in place.placeref_list:
placeref = PlaceRef()
placeref.ref = place_enclosed_by.handle
if place_date:
placeref.date = _dp.parse(place_date)
place.placeref_list.append(placeref)
#########################################################
self.db.commit_place(place, self.trans)
def get_place_type(self, place_type_str):
if place_type_str in self.place_types:
return PlaceType((self.place_types[place_type_str], place_type_str))
else:
# New custom type:
return PlaceType((0, place_type_str))
def get_or_create_family(self, family_ref, husband, wife):
"Return the family object for the give family ID."
# if a gramps_id and exists:
LOG.debug("get_or_create_family")
if family_ref.startswith("[") and family_ref.endswith("]"):
id_ = self.db.fid2user_format(family_ref[1:-1])
family = self.db.get_family_from_gramps_id(id_)
if family:
# don't delete, only add
fam_husband_handle = family.get_father_handle()
fam_wife_handle = family.get_mother_handle()
if husband:
if husband.get_handle() != fam_husband_handle:
# this husband is not the same old one! Add him!
family.set_father_handle(husband.get_handle())
if wife:
if wife.get_handle() != fam_wife_handle:
# this wife is not the same old one! Add her!
family.set_mother_handle(wife.get_handle())
LOG.debug(" returning existing family")
return family
# if not, create one:
family = Family()
# was marked with a gramps_id, but didn't exist, so we'll use it:
if family_ref.startswith("[") and family_ref.endswith("]"):
id_ = self.db.fid2user_format(family_ref[1:-1])
family.set_gramps_id(id_)
# add it:
family.set_handle(create_id())
if self.default_tag:
family.add_tag(self.default_tag.handle)
if husband:
family.set_father_handle(husband.get_handle())
husband.add_family_handle(family.get_handle())
if wife:
family.set_mother_handle(wife.get_handle())
wife.add_family_handle(family.get_handle())
if husband and wife:
family.set_relationship(FamilyRelType.MARRIED)
self.db.add_family(family, self.trans)
if husband:
self.db.commit_person(husband, self.trans)
if wife:
self.db.commit_person(wife, self.trans)
self.fam_count += 1
return family
def get_or_create_event(self, object_, type_, date=None, place=None,
source=None):
""" Add or find a type event on object """
# first, see if it exists
LOG.debug("get_or_create_event")
ref_list = object_.get_event_ref_list()
LOG.debug("refs: %s", ref_list)
# look for a match, and possible correction
for ref in ref_list:
event = self.db.get_event_from_handle(ref.ref)
LOG.debug(" compare event type %s == %s", int(event.get_type()),
type_)
if int(event.get_type()) == type_:
# Match! Let's update
if date:
event.set_date_object(date)
if place:
event.set_place_handle(place.get_handle())
if source:
self.find_and_set_citation(event, source)
self.db.commit_event(event, self.trans)
LOG.debug(" returning existing event")
return (0, event)
# else create it:
LOG.debug(" creating event")
event = Event()
if type_:
event.set_type(EventType(type_))
if date:
event.set_date_object(date)
if place:
event.set_place_handle(place.get_handle())
if source:
self.find_and_set_citation(event, source)
self.db.add_event(event, self.trans)
return (1, event)
def create_person(self):
""" Used to create a new person we know doesn't exist """
person = Person()
if self.default_tag:
person.add_tag(self.default_tag.handle)
self.db.add_person(person, self.trans)
self.indi_count += 1
return person
def create_place(self):
""" Used to create a new person we know doesn't exist """
place = Place()
if self.default_tag:
place.add_tag(self.default_tag.handle)
self.db.add_place(place, self.trans)
self.place_count += 1
return place
def get_or_create_place(self, place_name):
"Return the requested place object tuple-packed with a new indicator."
LOG.debug("get_or_create_place: looking for: %s", place_name)
for place_handle in self.db.iter_place_handles():
place = self.db.get_place_from_handle(place_handle)
place_title = place_displayer.display(self.db, place)
if place_title == place_name:
return (0, place)
place = Place()
place.set_title(place_name)
place.name = PlaceName(value=place_name)
self.db.add_place(place, self.trans)
return (1, place)
def get_or_create_source(self, source_text):
"Return the requested source object tuple-packed with a new indicator."
source_list = self.db.get_source_handles(sort_handles=False)
LOG.debug("get_or_create_source: list: %s", source_list)
LOG.debug("get_or_create_source: looking for: %s", source_text)
for source_handle in source_list:
source = self.db.get_source_from_handle(source_handle)
if source.get_title() == source_text:
LOG.debug(" returning existing source")
return (0, source)
LOG.debug(" creating source")
source = Source()
source.set_title(source_text)
self.db.add_source(source, self.trans)
return (1, source)
def find_and_set_citation(self, obj, source):
# look for the source in the existing citations for the object
LOG.debug("find_and_set_citation: looking for source: %s",
source.get_gramps_id())
for citation_handle in obj.get_citation_list():
citation = self.db.get_citation_from_handle(citation_handle)
LOG.debug("find_and_set_citation: existing citation: %s",
citation.get_gramps_id())
poss_source = self.db.get_source_from_handle(
citation.get_reference_handle())
LOG.debug(" compare source %s == %s", source.get_gramps_id(),
poss_source.get_gramps_id())
if poss_source.get_handle() == source.get_handle():
# The source is already cited
LOG.debug(" source already cited")
return
# we couldn't find an appropriate citation, so we have to create one.
citation = Citation()
LOG.debug(" creating citation")
citation.set_reference_handle(source.get_handle())
self.db.add_citation(citation, self.trans)
LOG.debug(" created citation, citation %s %s" %
(citation, citation.get_gramps_id()))
obj.add_citation(citation.get_handle())
def exportData(database,
filename,
error_dialog=None,
option_box=None,
callback=None):
if not callable(callback):
callback = lambda percent: None # dummy
with OpenFileOrStdout(filename, encoding="utf-8") as fp:
total = (len(database.note_map) + len(database.person_map) +
len(database.event_map) + len(database.family_map) +
len(database.repository_map) + len(database.place_map) +
len(database.media_map) + len(database.citation_map) +
len(database.source_map) + len(database.tag_map))
count = 0.0
# ---------------------------------
# Notes
# ---------------------------------
for handle in database.note_map.keys():
serial = database.note_map[handle]
write_line(fp, Note.create(serial))
count += 1
callback(100 * count / total)
# ---------------------------------
# Event
# ---------------------------------
for handle in database.event_map.keys():
serial = database.event_map[handle]
write_line(fp, Event.create(serial))
count += 1
callback(100 * count / total)
# ---------------------------------
# Person
# ---------------------------------
for handle in database.person_map.keys():
serial = database.person_map[handle]
write_line(fp, Person.create(serial))
count += 1
callback(100 * count / total)
# ---------------------------------
# Family
# ---------------------------------
for handle in database.family_map.keys():
serial = database.family_map[handle]
write_line(fp, Family.create(serial))
count += 1
callback(100 * count / total)
# ---------------------------------
# Repository
# ---------------------------------
for handle in database.repository_map.keys():
serial = database.repository_map[handle]
write_line(fp, Repository.create(serial))
count += 1
callback(100 * count / total)
# ---------------------------------
# Place
# ---------------------------------
for handle in database.place_map.keys():
serial = database.place_map[handle]
write_line(fp, Place.create(serial))
count += 1
callback(100 * count / total)
# ---------------------------------
# Source
# ---------------------------------
for handle in database.source_map.keys():
serial = database.source_map[handle]
write_line(fp, Source.create(serial))
count += 1
callback(100 * count / total)
# ---------------------------------
# Citation
# ---------------------------------
for handle in database.citation_map.keys():
serial = database.citation_map[handle]
write_line(fp, Citation.create(serial))
count += 1
callback(100 * count / total)
# ---------------------------------
# Media
# ---------------------------------
for handle in database.media_map.keys():
serial = database.media_map[handle]
write_line(fp, MediaObject.create(serial))
count += 1
callback(100 * count / total)
# ---------------------------------
# Tag
# ---------------------------------
for handle in database.tag_map.keys():
serial = database.tag_map[handle]
write_line(fp, Tag.create(serial))
count += 1
callback(100 * count / total)
return True
def __init__(self, dbase, user, default_tag_format=None):
self.db = dbase
self.user = user
self.trans = None
self.lineno = 0
self.index = 0
self.fam_count = 0
self.indi_count = 0
self.place_count = 0
self.pref = {} # person ref, internal to this sheet
self.fref = {} # family ref, internal to this sheet
self.placeref = {}
self.place_types = {}
# Build reverse dictionary, name to type number
for items in PlaceType().get_map().items(): # (0, 'Custom')
self.place_types[items[1]] = items[0]
self.place_types[items[1].lower()] = items[0]
if _(items[1]) != items[1]:
self.place_types[_(items[1])] = items[0]
# Add custom types:
for custom_type in self.db.get_place_types():
self.place_types[custom_type] = 0
self.place_types[custom_type.lower()] = 0
column2label = {
"surname": ("Lastname", "Surname", _("Surname"), "lastname",
"last_name", "surname", _("surname")),
"firstname": ("Firstname", "Given name", _("Given name"), "Given",
_("Given"), "firstname", "first_name", "given_name",
"given name", _("given name"), "given", _("given")),
"callname":
("Callname", "Call name", _("Call name"), "Call", _("Call"),
"callname", "call_name", "call name", "call", _("call")),
"title": ("Title", _("Person or Place|Title"), "title",
_("Person or Place|title")),
"prefix": ("Prefix", _("Prefix"), "prefix", _("prefix")),
"suffix": ("Suffix", _("Suffix"), "suffix", _("suffix")),
"gender": ("Gender", _("Gender"), "gender", _("gender")),
"source": ("Source", _("Source"), "source", _("source")),
"note": ("Note", _("Note"), "note", _("note")),
"birthplace":
("Birthplace", "Birth place", _("Birth place"), "birthplace",
"birth_place", "birth place", _("birth place")),
"birthplace_id": ("Birthplaceid", "Birth place id",
_("Birth place id"), "birthplaceid",
"birth_place_id", "birth place id",
_("birth place id"), "birthplace_id"),
"birthdate": ("Birthdate", "Birth date", _("Birth date"),
"birthdate", "birth_date", "birth date",
_("birth date")),
"birthsource": ("Birthsource", "Birth source", _("Birth source"),
"birthsource", "birth_source", "birth source",
_("birth source")),
"baptismplace":
("Baptismplace", "Baptism place", _("Baptism place"),
"baptismplace", "baptism place", _("baptism place")),
"baptismplace_id":
("Baptismplaceid", "Baptism place id",
_("Baptism place id"), "baptismplaceid", "baptism place id",
_("baptism place id"), "baptism_place_id", "baptismplace_id"),
"baptismdate": ("Baptismdate", "Baptism date", _("Baptism date"),
"baptismdate", "baptism date", _("baptism date")),
"baptismsource": ("Baptismsource", "Baptism source",
_("Baptism source"), "baptismsource",
"baptism source", _("baptism source")),
"burialplace": ("Burialplace", "Burial place", _("Burial place"),
"burialplace", "burial place", _("burial place")),
"burialplace_id":
("Burialplaceid", "Burial place id",
_("Burial place id"), "burialplaceid", "burial place id",
_("burial place id"), "burial_place_id", "burialplace_id"),
"burialdate": ("Burialdate", "Burial date", _("Burial date"),
"burialdate", "burial date", _("burial date")),
"burialsource": ("Burialsource", "Burial source",
_("Burial source"), "burialsource",
"burial source", _("burial source")),
"deathplace": ("Deathplace", "Death place", _("Death place"),
"deathplace", "death_place", "death place",
_("death place")),
"deathplace_id":
("Deathplaceid", "Death place id", _("Death place id"),
"deathplaceid", "death_place_id", "death place id",
_("death place id"), "death_place_id", "deathplace_id"),
"deathdate": ("Deathdate", "Death date", _("Death date"),
"deathdate", "death_date", "death date",
_("death date")),
"deathsource": ("Deathsource", "Death source", _("Death source"),
"deathsource", "death_source", "death source",
_("death source")),
"deathcause": ("Deathcause", "Death cause", _("Death cause"),
"deathcause", "death_cause", "death cause",
_("death cause")),
"grampsid": ("Grampsid", "ID", "Gramps id", _("Gramps ID"),
"grampsid", "id", "gramps_id", "gramps id",
_("Gramps id")),
"person": ("Person", _("Person"), "person", _("person")),
# ----------------------------------
"child": ("Child", _("Child"), "child", _("child")),
"family": ("Family", _("Family"), "family", _("family")),
# ----------------------------------
"wife": ("Mother", _("Mother"), "Wife", _("Wife"), "Parent2",
_("Parent2"), "mother", _("mother"), "wife", _("wife"),
"parent2", _("parent2")),
"husband": ("Father", _("Father"), "Husband", _("Husband"),
"Parent1", _("Parent1"), "father", _("father"),
"husband", _("husband"), "parent1", _("parent1")),
"marriage": ("Marriage", _("Marriage"), "marriage", _("marriage")),
"date": ("Date", _("Date"), "date", _("date")),
"place": ("Place", _("Place"), "place", _("place")),
"place_id": ("Placeid", "place id", "Place id", "place_id",
"placeid"),
"name": ("Name", _("Name"), "name", _("name")),
"type": ("Type", _("Type"), "type", _("type")),
"latitude": ("Latitude", _("latitude"), "latitude", _("latitude")),
"longitude": ("Longitude", _("Longitude"), "longitude",
_("longitude")),
"code": ("Code", _("Code"), "code", _("code")),
"enclosed_by": ("Enclosed by", _("Enclosed by"), "enclosed by",
_("enclosed by"), "enclosed_by", _("enclosed_by"),
"Enclosed_by", _("Enclosed_by"), "enclosedby")
}
lab2col_dict = []
for key in list(column2label.keys()):
for val in column2label[key]:
lab2col_dict.append((val, key))
self.label2column = dict(lab2col_dict)
if default_tag_format:
name = time.strftime(default_tag_format)
tag = self.db.get_tag_from_name(name)
if tag:
self.default_tag = tag
else:
self.default_tag = Tag()
self.default_tag.set_name(name)
else:
self.default_tag = None
class CSVParser(object):
"""Class to read data in CSV format from a file object."""
def __init__(self, dbase, user, default_tag_format=None):
self.db = dbase
self.user = user
self.trans = None
self.lineno = 0
self.index = 0
self.fam_count = 0
self.indi_count = 0
self.place_count = 0
self.pref = {} # person ref, internal to this sheet
self.fref = {} # family ref, internal to this sheet
self.placeref = {}
self.place_types = {}
# Build reverse dictionary, name to type number
for items in PlaceType().get_map().items(): # (0, 'Custom')
self.place_types[items[1]] = items[0]
self.place_types[items[1].lower()] = items[0]
if _(items[1]) != items[1]:
self.place_types[_(items[1])] = items[0]
# Add custom types:
for custom_type in self.db.get_place_types():
self.place_types[custom_type] = 0
self.place_types[custom_type.lower()] = 0
column2label = {
"surname": ("Lastname", "Surname", _("Surname"), "lastname",
"last_name", "surname", _("surname")),
"firstname": ("Firstname", "Given name", _("Given name"), "Given",
_("Given"), "firstname", "first_name", "given_name",
"given name", _("given name"), "given", _("given")),
"callname":
("Callname", "Call name", _("Call name"), "Call", _("Call"),
"callname", "call_name", "call name", "call", _("call")),
"title": ("Title", _("Person or Place|Title"), "title",
_("Person or Place|title")),
"prefix": ("Prefix", _("Prefix"), "prefix", _("prefix")),
"suffix": ("Suffix", _("Suffix"), "suffix", _("suffix")),
"gender": ("Gender", _("Gender"), "gender", _("gender")),
"source": ("Source", _("Source"), "source", _("source")),
"note": ("Note", _("Note"), "note", _("note")),
"birthplace":
("Birthplace", "Birth place", _("Birth place"), "birthplace",
"birth_place", "birth place", _("birth place")),
"birthplace_id": ("Birthplaceid", "Birth place id",
_("Birth place id"), "birthplaceid",
"birth_place_id", "birth place id",
_("birth place id"), "birthplace_id"),
"birthdate": ("Birthdate", "Birth date", _("Birth date"),
"birthdate", "birth_date", "birth date",
_("birth date")),
"birthsource": ("Birthsource", "Birth source", _("Birth source"),
"birthsource", "birth_source", "birth source",
_("birth source")),
"baptismplace":
("Baptismplace", "Baptism place", _("Baptism place"),
"baptismplace", "baptism place", _("baptism place")),
"baptismplace_id":
("Baptismplaceid", "Baptism place id",
_("Baptism place id"), "baptismplaceid", "baptism place id",
_("baptism place id"), "baptism_place_id", "baptismplace_id"),
"baptismdate": ("Baptismdate", "Baptism date", _("Baptism date"),
"baptismdate", "baptism date", _("baptism date")),
"baptismsource": ("Baptismsource", "Baptism source",
_("Baptism source"), "baptismsource",
"baptism source", _("baptism source")),
"burialplace": ("Burialplace", "Burial place", _("Burial place"),
"burialplace", "burial place", _("burial place")),
"burialplace_id":
("Burialplaceid", "Burial place id",
_("Burial place id"), "burialplaceid", "burial place id",
_("burial place id"), "burial_place_id", "burialplace_id"),
"burialdate": ("Burialdate", "Burial date", _("Burial date"),
"burialdate", "burial date", _("burial date")),
"burialsource": ("Burialsource", "Burial source",
_("Burial source"), "burialsource",
"burial source", _("burial source")),
"deathplace": ("Deathplace", "Death place", _("Death place"),
"deathplace", "death_place", "death place",
_("death place")),
"deathplace_id":
("Deathplaceid", "Death place id", _("Death place id"),
"deathplaceid", "death_place_id", "death place id",
_("death place id"), "death_place_id", "deathplace_id"),
"deathdate": ("Deathdate", "Death date", _("Death date"),
"deathdate", "death_date", "death date",
_("death date")),
"deathsource": ("Deathsource", "Death source", _("Death source"),
"deathsource", "death_source", "death source",
_("death source")),
"deathcause": ("Deathcause", "Death cause", _("Death cause"),
"deathcause", "death_cause", "death cause",
_("death cause")),
"grampsid": ("Grampsid", "ID", "Gramps id", _("Gramps ID"),
"grampsid", "id", "gramps_id", "gramps id",
_("Gramps id")),
"person": ("Person", _("Person"), "person", _("person")),
# ----------------------------------
"child": ("Child", _("Child"), "child", _("child")),
"family": ("Family", _("Family"), "family", _("family")),
# ----------------------------------
"wife": ("Mother", _("Mother"), "Wife", _("Wife"), "Parent2",
_("Parent2"), "mother", _("mother"), "wife", _("wife"),
"parent2", _("parent2")),
"husband": ("Father", _("Father"), "Husband", _("Husband"),
"Parent1", _("Parent1"), "father", _("father"),
"husband", _("husband"), "parent1", _("parent1")),
"marriage": ("Marriage", _("Marriage"), "marriage", _("marriage")),
"date": ("Date", _("Date"), "date", _("date")),
"place": ("Place", _("Place"), "place", _("place")),
"place_id": ("Placeid", "place id", "Place id", "place_id",
"placeid"),
"name": ("Name", _("Name"), "name", _("name")),
"type": ("Type", _("Type"), "type", _("type")),
"latitude": ("Latitude", _("latitude"), "latitude", _("latitude")),
"longitude": ("Longitude", _("Longitude"), "longitude",
_("longitude")),
"code": ("Code", _("Code"), "code", _("code")),
"enclosed_by": ("Enclosed by", _("Enclosed by"), "enclosed by",
_("enclosed by"), "enclosed_by", _("enclosed_by"),
"Enclosed_by", _("Enclosed_by"), "enclosedby")
}
lab2col_dict = []
for key in list(column2label.keys()):
for val in column2label[key]:
lab2col_dict.append((val, key))
self.label2column = dict(lab2col_dict)
if default_tag_format:
name = time.strftime(default_tag_format)
tag = self.db.get_tag_from_name(name)
if tag:
self.default_tag = tag
else:
self.default_tag = Tag()
self.default_tag.set_name(name)
else:
self.default_tag = None
def cleanup_column_name(self, column):
"""Handle column aliases for CSV spreadsheet import and SQL."""
return self.label2column.get(column, column)
def read_csv(self, filehandle):
"Read the data from the file and return it as a list."
try:
data = [[r.strip() for r in row] for row in csv.reader(filehandle)]
except csv.Error as err:
self.user.notify_error(
_('format error: line %(line)d: %(zero)s') % {
'line': reader.line_num,
'zero': err
})
return None
return data
def lookup(self, type_, id_):
"""
Return the object of type type_ with id id_ from db or previously
stored value.
"""
if id_ is None:
return None
if type_ == "family":
if id_.startswith("[") and id_.endswith("]"):
id_ = self.db.fid2user_format(id_[1:-1])
db_lookup = self.db.get_family_from_gramps_id(id_)
if db_lookup is None:
return self.lookup(type_, id_)
else:
return db_lookup
elif id_.lower() in self.fref:
return self.fref[id_.lower()]
else:
return None
elif type_ == "person":
if id_.startswith("[") and id_.endswith("]"):
id_ = self.db.id2user_format(id_[1:-1])
db_lookup = self.db.get_person_from_gramps_id(id_)
if db_lookup is None:
return self.lookup(type_, id_)
else:
return db_lookup
elif id_.lower() in self.pref:
return self.pref[id_.lower()]
else:
return None
elif type_ == "place":
if id_.startswith("[") and id_.endswith("]"):
id_ = self.db.id2user_format(id_[1:-1])
db_lookup = self.db.get_place_from_gramps_id(id_)
if db_lookup is None:
return self.lookup(type_, id_)
else:
return db_lookup
elif id_.lower() in self.placeref:
return self.placeref[id_.lower()]
else:
return None
else:
LOG.warn("invalid lookup type in CSV import: '%s'" % type_)
return None
def storeup(self, type_, id_, object_):
"Store object object_ of type type_ in a dictionary under key id_."
if id_.startswith("[") and id_.endswith("]"):
id_ = id_[1:-1]
#return # do not store gramps people; go look them up
if type_ == "person":
id_ = self.db.id2user_format(id_)
self.pref[id_.lower()] = object_
elif type_ == "family":
id_ = self.db.fid2user_format(id_)
self.fref[id_.lower()] = object_
elif type_ == "place":
id_ = self.db.pid2user_format(id_)
self.placeref[id_.lower()] = object_
else:
LOG.warn("invalid storeup type in CSV import: '%s'" % type_)
def parse(self, filehandle):
"""
Prepare the database and parse the input file.
:param filehandle: open file handle positioned at start of the file
"""
progress_title = _('CSV Import')
with self.user.progress(progress_title, _('Reading data...'),
1) as step:
data = self.read_csv(filehandle)
with self.user.progress(progress_title, _('Importing data...'),
len(data)) as step:
tym = time.time()
self.db.disable_signals()
with DbTxn(_("CSV import"), self.db, batch=True) as self.trans:
if self.default_tag and self.default_tag.handle is None:
self.db.add_tag(self.default_tag, self.trans)
self._parse_csv_data(data, step)
self.db.enable_signals()
self.db.request_rebuild()
tym = time.time() - tym
# translators: leave all/any {...} untranslated
msg = ngettext('Import Complete: {number_of} second',
'Import Complete: {number_of} seconds',
tym).format(number_of=tym)
LOG.debug(msg)
LOG.debug("New Families: %d" % self.fam_count)
LOG.debug("New Individuals: %d" % self.indi_count)
def _parse_csv_data(self, data, step):
"""Parse each line of the input data and act accordingly."""
self.lineno = 0
self.index = 0
self.fam_count = 0
self.indi_count = 0
self.place_count = 0
self.pref = {} # person ref, internal to this sheet
self.fref = {} # family ref, internal to this sheet
self.placeref = {}
header = None
line_number = 0
for row in data:
step()
line_number += 1
if "".join(row) == "": # no blanks are allowed inside a table
header = None # clear headers, ready for next "table"
continue
######################################
if header is None:
header = [self.cleanup_column_name(r) for r in row]
col = {}
count = 0
for key in header:
col[key] = count
count += 1
continue
# four different kinds of data: person, family, and marriage
if (("marriage" in header) or ("husband" in header)
or ("wife" in header)):
self._parse_marriage(line_number, row, col)
elif "family" in header:
self._parse_family(line_number, row, col)
elif "surname" in header:
self._parse_person(line_number, row, col)
elif "place" in header:
self._parse_place(line_number, row, col)
else:
LOG.warn("ignoring line %d" % line_number)
return None
def _parse_marriage(self, line_number, row, col):
"Parse the content of a Marriage,Husband,Wife line."
marriage_ref = rd(line_number, row, col, "marriage")
husband = rd(line_number, row, col, "husband")
wife = rd(line_number, row, col, "wife")
marriagedate = rd(line_number, row, col, "date")
marriageplace = rd(line_number, row, col, "place")
marriageplace_id = rd(line_number, row, col, "place_id")
marriagesource = rd(line_number, row, col, "source")
note = rd(line_number, row, col, "note")
wife = self.lookup("person", wife)
husband = self.lookup("person", husband)
if husband is None and wife is None:
# might have children, so go ahead and add
LOG.warn("no parents on line %d; adding family anyway" %
line_number)
family = self.get_or_create_family(marriage_ref, husband, wife)
# adjust gender, if not already provided
if husband:
# this is just a guess, if unknown
if husband.get_gender() == Person.UNKNOWN:
husband.set_gender(Person.MALE)
self.db.commit_person(husband, self.trans)
if wife:
# this is just a guess, if unknown
if wife.get_gender() == Person.UNKNOWN:
wife.set_gender(Person.FEMALE)
self.db.commit_person(wife, self.trans)
if marriage_ref:
self.storeup("family", marriage_ref.lower(), family)
if marriagesource:
# add, if new
new, marriagesource = self.get_or_create_source(marriagesource)
if marriageplace and marriageplace_id:
raise Error("Error in marriage: can't have a place and place_id")
if marriageplace:
# add, if new
new, marriageplace = self.get_or_create_place(marriageplace)
elif marriageplace_id:
# better exist already, locally or in database:
marriageplace = self.lookup("place", marriageplace_id)
if marriagedate:
marriagedate = _dp.parse(marriagedate)
if marriagedate or marriageplace or marriagesource or note:
# add, if new; replace, if different
new, marriage = self.get_or_create_event(family,
EventType.MARRIAGE,
marriagedate,
marriageplace,
marriagesource)
if new:
mar_ref = EventRef()
mar_ref.set_reference_handle(marriage.get_handle())
family.add_event_ref(mar_ref)
self.db.commit_family(family, self.trans)
# only add note to event:
# append notes, if previous notes
if note:
previous_notes_list = marriage.get_note_list()
updated_note = False
for note_handle in previous_notes_list:
previous_note = self.db.get_note_from_handle(note_handle)
if previous_note.type == NoteType.EVENT:
previous_text = previous_note.get()
if note not in previous_text:
note = previous_text + "\n" + note
previous_note.set(note)
self.db.commit_note(previous_note, self.trans)
updated_note = True
break
if not updated_note:
# add new note here
new_note = Note()
new_note.handle = create_id()
new_note.type.set(NoteType.EVENT)
new_note.set(note)
if self.default_tag:
new_note.add_tag(self.default_tag.handle)
self.db.add_note(new_note, self.trans)
marriage.add_note(new_note.handle)
self.db.commit_event(marriage, self.trans)
def _parse_family(self, line_number, row, col):
"Parse the content of a family line"
family_ref = rd(line_number, row, col, "family")
if family_ref is None:
LOG.warn("no family reference found for family on line %d" %
line_number)
return # required
child = rd(line_number, row, col, "child")
source = rd(line_number, row, col, "source")
note = rd(line_number, row, col, "note")
gender = rd(line_number, row, col, "gender")
child = self.lookup("person", child)
family = self.lookup("family", family_ref)
if family is None:
LOG.warn("no matching family reference found for family "
"on line %d" % line_number)
return
if child is None:
LOG.warn("no matching child reference found for family "
"on line %d" % line_number)
return
# is this child already in this family? If so, don't add
LOG.debug("children: %s",
[ref.ref for ref in family.get_child_ref_list()])
LOG.debug("looking for: %s", child.get_handle())
if child.get_handle() not in [
ref.ref for ref in family.get_child_ref_list()
]:
# add child to family
LOG.debug(" adding child [%s] to family [%s]",
child.get_gramps_id(), family.get_gramps_id())
childref = ChildRef()
childref.set_reference_handle(child.get_handle())
family.add_child_ref(childref)
self.db.commit_family(family, self.trans)
child.add_parent_family_handle(family.get_handle())
if gender:
# replace
gender = gender.lower()
if gender == gender_map[Person.MALE].lower():
gender = Person.MALE
elif gender == gender_map[Person.FEMALE].lower():
gender = Person.FEMALE
else:
gender = Person.UNKNOWN
child.set_gender(gender)
if source:
# add, if new
dummy_new, source = self.get_or_create_source(source)
self.find_and_set_citation(child, source)
# put note on child
if note:
# append notes, if previous notes
previous_notes_list = child.get_note_list()
updated_note = False
for note_handle in previous_notes_list:
previous_note = self.db.get_note_from_handle(note_handle)
if previous_note.type == NoteType.PERSON:
previous_text = previous_note.get()
if note not in previous_text:
note = previous_text + "\n" + note
previous_note.set(note)
self.db.commit_note(previous_note, self.trans)
updated_note = True
break
if not updated_note:
# add new note here
new_note = Note()
new_note.handle = create_id()
new_note.type.set(NoteType.PERSON)
new_note.set(note)
if self.default_tag:
new_note.add_tag(self.default_tag.handle)
self.db.add_note(new_note, self.trans)
child.add_note(new_note.handle)
self.db.commit_person(child, self.trans)
def _parse_person(self, line_number, row, col):
"Parse the content of a Person line."
surname = rd(line_number, row, col, "surname")
firstname = rd(line_number, row, col, "firstname", "")
callname = rd(line_number, row, col, "callname")
title = rd(line_number, row, col, "title")
prefix = rd(line_number, row, col, "prefix")
suffix = rd(line_number, row, col, "suffix")
gender = rd(line_number, row, col, "gender")
source = rd(line_number, row, col, "source")
note = rd(line_number, row, col, "note")
birthplace = rd(line_number, row, col, "birthplace")
birthplace_id = rd(line_number, row, col, "birthplace_id")
birthdate = rd(line_number, row, col, "birthdate")
birthsource = rd(line_number, row, col, "birthsource")
baptismplace = rd(line_number, row, col, "baptismplace")
baptismplace_id = rd(line_number, row, col, "baptismplace_id")
baptismdate = rd(line_number, row, col, "baptismdate")
baptismsource = rd(line_number, row, col, "baptismsource")
burialplace = rd(line_number, row, col, "burialplace")
burialplace_id = rd(line_number, row, col, "burialplace_id")
burialdate = rd(line_number, row, col, "burialdate")
burialsource = rd(line_number, row, col, "burialsource")
deathplace = rd(line_number, row, col, "deathplace")
deathplace_id = rd(line_number, row, col, "deathplace_id")
deathdate = rd(line_number, row, col, "deathdate")
deathsource = rd(line_number, row, col, "deathsource")
deathcause = rd(line_number, row, col, "deathcause")
grampsid = rd(line_number, row, col, "grampsid")
person_ref = rd(line_number, row, col, "person")
#########################################################
# if this person already exists, don't create them
person = self.lookup("person", person_ref)
if person is None:
if surname is None:
LOG.warn("empty surname for new person on line %d" %
line_number)
surname = ""
# new person
person = self.create_person()
name = Name()
name.set_type(NameType(NameType.BIRTH))
name.set_first_name(firstname)
surname_obj = Surname()
surname_obj.set_surname(surname)
name.add_surname(surname_obj)
person.set_primary_name(name)
else:
name = person.get_primary_name()
#########################################################
if person_ref is not None:
self.storeup("person", person_ref, person)
# replace
if surname is not None:
name.get_primary_surname().set_surname(surname)
if firstname is not None:
name.set_first_name(firstname)
if callname is not None:
name.set_call_name(callname)
if title is not None:
name.set_title(title)
if prefix is not None:
name.get_primary_surname().set_prefix(prefix)
name.group_as = '' # HELP? what should I do here?
if suffix is not None:
name.set_suffix(suffix)
if note is not None:
# append notes, if previous notes
previous_notes_list = person.get_note_list()
updated_note = False
for note_handle in previous_notes_list:
previous_note = self.db.get_note_from_handle(note_handle)
if previous_note.type == NoteType.PERSON:
previous_text = previous_note.get()
if note not in previous_text:
note = previous_text + "\n" + note
previous_note.set(note)
self.db.commit_note(previous_note, self.trans)
updated_note = True
break
if not updated_note:
# add new note here
new_note = Note()
new_note.handle = create_id()
new_note.type.set(NoteType.PERSON)
new_note.set(note)
if self.default_tag:
new_note.add_tag(self.default_tag.handle)
self.db.add_note(new_note, self.trans)
person.add_note(new_note.handle)
if grampsid is not None:
person.gramps_id = grampsid
elif person_ref is not None:
if person_ref.startswith("[") and person_ref.endswith("]"):
person.gramps_id = self.db.id2user_format(person_ref[1:-1])
if (person.get_gender() == Person.UNKNOWN and gender is not None):
gender = gender.lower()
if gender == gender_map[Person.MALE].lower():
gender = Person.MALE
elif gender == gender_map[Person.FEMALE].lower():
gender = Person.FEMALE
else:
gender = Person.UNKNOWN
person.set_gender(gender)
#########################################################
# add if new, replace if different
# Birth:
if birthdate is not None:
birthdate = _dp.parse(birthdate)
if birthplace and birthplace_id:
raise Error(
"Error in person: can't have a birthplace and birthplace_id")
if birthplace is not None:
new, birthplace = self.get_or_create_place(birthplace)
elif birthplace_id:
# better exist already, locally or in database:
birthplace = self.lookup("place", birthplace_id)
if birthsource is not None:
new, birthsource = self.get_or_create_source(birthsource)
if birthdate or birthplace or birthsource:
new, birth = self.get_or_create_event(person, EventType.BIRTH,
birthdate, birthplace,
birthsource)
birth_ref = person.get_birth_ref()
if birth_ref is None:
# new
birth_ref = EventRef()
birth_ref.set_reference_handle(birth.get_handle())
person.set_birth_ref(birth_ref)
# Baptism:
if baptismdate is not None:
baptismdate = _dp.parse(baptismdate)
if baptismplace and baptismplace_id:
raise Error(
"Error in person: can't have a baptismplace and baptismplace_id"
)
if baptismplace is not None:
new, baptismplace = self.get_or_create_place(baptismplace)
elif baptismplace_id:
# better exist already, locally or in database:
baptismplace = self.lookup("place", baptismplace_id)
if baptismsource is not None:
new, baptismsource = self.get_or_create_source(baptismsource)
if baptismdate or baptismplace or baptismsource:
new, baptism = self.get_or_create_event(person, EventType.BAPTISM,
baptismdate, baptismplace,
baptismsource)
baptism_ref = get_primary_event_ref_from_type(
self.db, person, "Baptism")
if baptism_ref is None:
# new
baptism_ref = EventRef()
baptism_ref.set_reference_handle(baptism.get_handle())
person.add_event_ref(baptism_ref)
# Death:
if deathdate is not None:
deathdate = _dp.parse(deathdate)
if deathplace and deathplace_id:
raise Error(
"Error in person: can't have a deathplace and deathplace_id")
if deathplace is not None:
new, deathplace = self.get_or_create_place(deathplace)
elif deathplace_id:
# better exist already, locally or in database:
deathplace = self.lookup("place", deathplace_id)
if deathsource is not None:
new, deathsource = self.get_or_create_source(deathsource)
if deathdate or deathplace or deathsource or deathcause:
new, death = self.get_or_create_event(person, EventType.DEATH,
deathdate, deathplace,
deathsource)
if deathcause:
death.set_description(deathcause)
self.db.commit_event(death, self.trans)
death_ref = person.get_death_ref()
if death_ref is None:
# new
death_ref = EventRef()
death_ref.set_reference_handle(death.get_handle())
person.set_death_ref(death_ref)
# Burial:
if burialdate is not None:
burialdate = _dp.parse(burialdate)
if burialplace and burialplace_id:
raise Error(
"Error in person: can't have a burialplace and burialplace_id")
if burialplace is not None:
new, burialplace = self.get_or_create_place(burialplace)
elif burialplace_id:
# better exist already, locally or in database:
burialplace = self.lookup("place", burialplace_id)
if burialsource is not None:
new, burialsource = self.get_or_create_source(burialsource)
if burialdate or burialplace or burialsource:
new, burial = self.get_or_create_event(person, EventType.BURIAL,
burialdate, burialplace,
burialsource)
burial_ref = get_primary_event_ref_from_type(
self.db, person, "Burial")
if burial_ref is None:
# new
burial_ref = EventRef()
burial_ref.set_reference_handle(burial.get_handle())
person.add_event_ref(burial_ref)
if source:
# add, if new
new, source = self.get_or_create_source(source)
self.find_and_set_citation(person, source)
self.db.commit_person(person, self.trans)
def _parse_place(self, line_number, row, col):
"Parse the content of a Place line."
place_id = rd(line_number, row, col, "place")
place_title = rd(line_number, row, col, "title")
place_name = rd(line_number, row, col, "name")
place_type_str = rd(line_number, row, col, "type")
place_latitude = rd(line_number, row, col, "latitude")
place_longitude = rd(line_number, row, col, "longitude")
place_code = rd(line_number, row, col, "code")
place_enclosed_by_id = rd(line_number, row, col, "enclosed_by")
place_date = rd(line_number, row, col, "date")
#########################################################
# if this place already exists, don't create it
place = self.lookup("place", place_id)
if place is None:
# new place
place = self.create_place()
self.storeup("place", place_id.lower(), place)
if place_title is not None:
place.title = place_title
if place_name is not None:
place.name = PlaceName(value=place_name)
if place_type_str is not None:
place.place_type = self.get_place_type(place_type_str)
if place_latitude is not None:
place.lat = place_latitude
if place_longitude is not None:
place.long = place_longitude
if place_code is not None:
place.code = place_code
if place_enclosed_by_id is not None:
place_enclosed_by = self.lookup("place", place_enclosed_by_id)
if place_enclosed_by is None:
raise Exception("cannot enclose %s in %s as it doesn't exist" %
(place.gramps_id, place_enclosed_by_id))
if not place_enclosed_by.handle in place.placeref_list:
placeref = PlaceRef()
placeref.ref = place_enclosed_by.handle
if place_date:
placeref.date = _dp.parse(place_date)
place.placeref_list.append(placeref)
#########################################################
self.db.commit_place(place, self.trans)
def get_place_type(self, place_type_str):
if place_type_str in self.place_types:
return PlaceType(
(self.place_types[place_type_str], place_type_str))
else:
# New custom type:
return PlaceType((0, place_type_str))
def get_or_create_family(self, family_ref, husband, wife):
"Return the family object for the give family ID."
# if a gramps_id and exists:
LOG.debug("get_or_create_family")
if family_ref.startswith("[") and family_ref.endswith("]"):
id_ = self.db.fid2user_format(family_ref[1:-1])
family = self.db.get_family_from_gramps_id(id_)
if family:
# don't delete, only add
fam_husband_handle = family.get_father_handle()
fam_wife_handle = family.get_mother_handle()
if husband:
if husband.get_handle() != fam_husband_handle:
# this husband is not the same old one! Add him!
family.set_father_handle(husband.get_handle())
if wife:
if wife.get_handle() != fam_wife_handle:
# this wife is not the same old one! Add her!
family.set_mother_handle(wife.get_handle())
LOG.debug(" returning existing family")
return family
# if not, create one:
family = Family()
# was marked with a gramps_id, but didn't exist, so we'll use it:
if family_ref.startswith("[") and family_ref.endswith("]"):
id_ = self.db.fid2user_format(family_ref[1:-1])
family.set_gramps_id(id_)
# add it:
family.set_handle(create_id())
if self.default_tag:
family.add_tag(self.default_tag.handle)
if husband:
family.set_father_handle(husband.get_handle())
husband.add_family_handle(family.get_handle())
if wife:
family.set_mother_handle(wife.get_handle())
wife.add_family_handle(family.get_handle())
if husband and wife:
family.set_relationship(FamilyRelType.MARRIED)
self.db.add_family(family, self.trans)
if husband:
self.db.commit_person(husband, self.trans)
if wife:
self.db.commit_person(wife, self.trans)
self.fam_count += 1
return family
def get_or_create_event(self,
object_,
type_,
date=None,
place=None,
source=None):
""" Add or find a type event on object """
# first, see if it exists
LOG.debug("get_or_create_event")
ref_list = object_.get_event_ref_list()
LOG.debug("refs: %s", ref_list)
# look for a match, and possible correction
for ref in ref_list:
event = self.db.get_event_from_handle(ref.ref)
LOG.debug(" compare event type %s == %s", int(event.get_type()),
type_)
if int(event.get_type()) == type_:
# Match! Let's update
if date:
event.set_date_object(date)
if place:
event.set_place_handle(place.get_handle())
if source:
self.find_and_set_citation(event, source)
self.db.commit_event(event, self.trans)
LOG.debug(" returning existing event")
return (0, event)
# else create it:
LOG.debug(" creating event")
event = Event()
if type_:
event.set_type(EventType(type_))
if date:
event.set_date_object(date)
if place:
event.set_place_handle(place.get_handle())
if source:
self.find_and_set_citation(event, source)
self.db.add_event(event, self.trans)
return (1, event)
def create_person(self):
""" Used to create a new person we know doesn't exist """
person = Person()
if self.default_tag:
person.add_tag(self.default_tag.handle)
self.db.add_person(person, self.trans)
self.indi_count += 1
return person
def create_place(self):
""" Used to create a new person we know doesn't exist """
place = Place()
if self.default_tag:
place.add_tag(self.default_tag.handle)
self.db.add_place(place, self.trans)
self.place_count += 1
return place
def get_or_create_place(self, place_name):
"Return the requested place object tuple-packed with a new indicator."
LOG.debug("get_or_create_place: looking for: %s", place_name)
for place_handle in self.db.iter_place_handles():
place = self.db.get_place_from_handle(place_handle)
place_title = place_displayer.display(self.db, place)
if place_title == place_name:
return (0, place)
place = Place()
place.set_title(place_name)
place.name = PlaceName(value=place_name)
self.db.add_place(place, self.trans)
return (1, place)
def get_or_create_source(self, source_text):
"Return the requested source object tuple-packed with a new indicator."
source_list = self.db.get_source_handles(sort_handles=False)
LOG.debug("get_or_create_source: list: %s", source_list)
LOG.debug("get_or_create_source: looking for: %s", source_text)
for source_handle in source_list:
source = self.db.get_source_from_handle(source_handle)
if source.get_title() == source_text:
LOG.debug(" returning existing source")
return (0, source)
LOG.debug(" creating source")
source = Source()
source.set_title(source_text)
self.db.add_source(source, self.trans)
return (1, source)
def find_and_set_citation(self, obj, source):
# look for the source in the existing citations for the object
LOG.debug("find_and_set_citation: looking for source: %s",
source.get_gramps_id())
for citation_handle in obj.get_citation_list():
citation = self.db.get_citation_from_handle(citation_handle)
LOG.debug("find_and_set_citation: existing citation: %s",
citation.get_gramps_id())
poss_source = self.db.get_source_from_handle(
citation.get_reference_handle())
LOG.debug(" compare source %s == %s", source.get_gramps_id(),
poss_source.get_gramps_id())
if poss_source.get_handle() == source.get_handle():
# The source is already cited
LOG.debug(" source already cited")
return
# we couldn't find an appropriate citation, so we have to create one.
citation = Citation()
LOG.debug(" creating citation")
citation.set_reference_handle(source.get_handle())
self.db.add_citation(citation, self.trans)
LOG.debug(" created citation, citation %s %s" %
(citation, citation.get_gramps_id()))
obj.add_citation(citation.get_handle())
def applyTagClicked(self, button) :
progress = None
rows = self.treeSelection.count_selected_rows()
tag_name = str(self.tagcombo.get_active_text())
# start the db transaction
with DbTxn("Tag not related", self.db) as transaction:
tag = self.db.get_tag_from_name(tag_name)
if not tag:
# create the tag if it doesn't already exist
tag = Tag()
tag.set_name(tag_name)
tag.set_priority(self.db.get_number_of_tags())
tag_handle = self.db.add_tag(tag, transaction)
else:
tag_handle = tag.get_handle()
# if more than 1 person is selected, use a progress indicator
if rows > 1:
progress = ProgressMeter(self.title,_('Starting'),
parent=self.window)
progress.set_pass(
# translators: leave all/any {...} untranslated
#TRANS: no singular form needed, as rows is always > 1
ngettext("Setting tag for {number_of} person",
"Setting tag for {number_of} people",
rows).format(number_of=rows),
rows)
# iterate through all of the selected rows
(model, paths) = self.treeSelection.get_selected_rows()
for path in paths:
if progress:
progress.step()
# for the current row, get the GID and the person from the database
iter = self.model.get_iter(path)
personGid = self.model.get_value(iter, 1)
person = self.db.get_person_from_gramps_id(personGid)
# add the tag to the person
person.add_tag(tag_handle)
# save this change
self.db.commit_person(person, transaction)
# refresh the tags column
self.treeView.set_model(None)
for path in paths:
iter = self.model.get_iter(path)
personGid = self.model.get_value(iter, 1)
person = self.db.get_person_from_gramps_id(personGid)
self.model.set_value(iter, 3, self.get_tag_list(person))
self.treeView.set_model(self.model)
self.treeView.expand_all()
if progress:
progress.close()
def __init__(self, dbase, user, default_tag_format=None):
self.db = dbase
self.user = user
self.trans = None
self.lineno = 0
self.index = 0
self.fam_count = 0
self.indi_count = 0
self.place_count = 0
self.pref = {} # person ref, internal to this sheet
self.fref = {} # family ref, internal to this sheet
self.placeref = {}
self.place_types = {}
# Build reverse dictionary, name to type number
for items in PlaceType().get_map().items(): # (0, 'Custom')
self.place_types[items[1]] = items[0]
self.place_types[items[1].lower()] = items[0]
if _(items[1]) != items[1]:
self.place_types[_(items[1])] = items[0]
# Add custom types:
for custom_type in self.db.get_place_types():
self.place_types[custom_type] = 0
self.place_types[custom_type.lower()] = 0
column2label = {
"surname": ("Lastname", "Surname", _("Surname"), "lastname",
"last_name", "surname", _("surname")),
"firstname": ("Firstname", "Given name", _("Given name"), "Given",
_("Given"), "firstname", "first_name", "given_name",
"given name", _("given name"), "given", _("given")),
"callname": ("Callname", "Call name", _("Call name"), "Call",
_("Call"), "callname", "call_name", "call name", "call",
_("call")),
"title": ("Title", _("Person or Place|Title"), "title", _("Person or Place|title")),
"prefix": ("Prefix", _("Prefix"), "prefix", _("prefix")),
"suffix": ("Suffix", _("Suffix"), "suffix", _("suffix")),
"gender": ("Gender", _("Gender"), "gender", _("gender")),
"source": ("Source", _("Source"), "source", _("source")),
"note": ("Note", _("Note"), "note", _("note")),
"birthplace": ("Birthplace", "Birth place", _("Birth place"),
"birthplace", "birth_place", "birth place", _("birth place")),
"birthplace_id": ("Birthplaceid", "Birth place id", _("Birth place id"),
"birthplaceid", "birth_place_id", "birth place id", _("birth place id"),
"birthplace_id"),
"birthdate": ("Birthdate", "Birth date", _("Birth date"),
"birthdate", "birth_date", "birth date", _("birth date")),
"birthsource": ("Birthsource", "Birth source", _("Birth source"),
"birthsource", "birth_source", "birth source",
_("birth source")),
"baptismplace": ("Baptismplace", "Baptism place",
_("Baptism place"), "baptismplace", "baptism place",
_("baptism place")),
"baptismplace_id": ("Baptismplaceid", "Baptism place id",
_("Baptism place id"), "baptismplaceid", "baptism place id",
_("baptism place id"), "baptism_place_id",
"baptismplace_id"),
"baptismdate": ("Baptismdate", "Baptism date", _("Baptism date"),
"baptismdate", "baptism date", _("baptism date")),
"baptismsource": ("Baptismsource", "Baptism source",
_("Baptism source"), "baptismsource", "baptism source",
_("baptism source")),
"burialplace": ("Burialplace", "Burial place", _("Burial place"),
"burialplace", "burial place", _("burial place")),
"burialplace_id": ("Burialplaceid", "Burial place id", _("Burial place id"),
"burialplaceid", "burial place id", _("burial place id"),
"burial_place_id", "burialplace_id"),
"burialdate": ("Burialdate", "Burial date", _("Burial date"),
"burialdate", "burial date", _("burial date")),
"burialsource": ("Burialsource", "Burial source",
_("Burial source"), "burialsource", "burial source",
_("burial source")),
"deathplace": ("Deathplace", "Death place", _("Death place"),
"deathplace", "death_place", "death place", _("death place")),
"deathplace_id": ("Deathplaceid", "Death place id", _("Death place id"),
"deathplaceid", "death_place_id", "death place id", _("death place id"),
"death_place_id", "deathplace_id"),
"deathdate": ("Deathdate", "Death date", _("Death date"),
"deathdate", "death_date", "death date", _("death date")),
"deathsource": ("Deathsource", "Death source", _("Death source"),
"deathsource", "death_source", "death source",
_("death source")),
"deathcause": ("Deathcause", "Death cause", _("Death cause"),
"deathcause", "death_cause", "death cause", _("death cause")),
"grampsid": ("Grampsid", "ID", "Gramps id", _("Gramps ID"),
"grampsid", "id", "gramps_id", "gramps id", _("Gramps id")),
"person": ("Person", _("Person"), "person", _("person")),
# ----------------------------------
"child": ("Child", _("Child"), "child", _("child")),
"family": ("Family", _("Family"), "family", _("family")),
# ----------------------------------
"wife": ("Mother", _("Mother"), "Wife", _("Wife"), "Parent2",
_("Parent2"), "mother", _("mother"), "wife", _("wife"),
"parent2", _("parent2")),
"husband": ("Father", _("Father"), "Husband", _("Husband"),
"Parent1", _("Parent1"), "father", _("father"), "husband",
_("husband"), "parent1", _("parent1")),
"marriage": ("Marriage", _("Marriage"), "marriage", _("marriage")),
"date": ("Date", _("Date"), "date", _("date")),
"place": ("Place", _("Place"), "place", _("place")),
"place_id": ("Placeid", "place id", "Place id", "place_id", "placeid"),
"name": ("Name", _("Name"), "name", _("name")),
"type": ("Type", _("Type"), "type", _("type")),
"latitude": ("Latitude", _("latitude"), "latitude", _("latitude")),
"longitude": ("Longitude", _("Longitude"), "longitude", _("longitude")),
"code": ("Code", _("Code"), "code", _("code")),
"enclosed_by": ("Enclosed by", _("Enclosed by"), "enclosed by", _("enclosed by"),
"enclosed_by", _("enclosed_by"), "Enclosed_by", _("Enclosed_by"),
"enclosedby")
}
lab2col_dict = []
for key in list(column2label.keys()):
for val in column2label[key]:
lab2col_dict.append((val, key))
self.label2column = dict(lab2col_dict)
if default_tag_format:
name = time.strftime(default_tag_format)
tag = self.db.get_tag_from_name(name)
if tag:
self.default_tag = tag
else:
self.default_tag = Tag()
self.default_tag.set_name(name)
else:
self.default_tag = None
