def find_or_make_source(self):
""" Find or create a source.
returns handle to source."""
for hndl in self.dbstate.db.get_source_handles():
if self.dbstate.db.get_raw_source_data(hndl)[2] == 'GeoNames':
return hndl
# No source found, lets add one with associated repo and note
repo = Repository()
repo.set_name("www.geonames.org")
rtype = RepositoryType(RepositoryType.WEBSITE)
repo.set_type(rtype)
url = Url()
url.set_path('http://www.geonames.org/')
url.set_description(_('GeoNames web site'))
url.set_type(UrlType(UrlType.WEB_HOME))
repo.add_url(url)
url = Url()
url.set_path('*****@*****.**')
url.set_description(_('GeoNames author'))
url.set_type(UrlType(UrlType.EMAIL))
repo.add_url(url)
note_txt = StyledText(
_('GeoNames was founded by Marc Wick. You can reach him at '))
note_txt += StyledText('*****@*****.**' + '\n')
note_txt += StyledText(
_('GeoNames is a project of Unxos GmbH, Weingartenstrasse 8,'
' 8708 Männedorf, Switzerland.\nThis work is licensed under a '))
note_txt += linkst(
_('Creative Commons Attribution 3.0 License'),
'https://creativecommons.org/licenses/by/3.0/legalcode')
new_note = Note()
new_note.set_styledtext(note_txt)
new_note.set_type(NoteType.REPO)
src = Source()
src.title = 'GeoNames'
src.author = 'Marc Wick'
repo_ref = RepoRef()
mtype = SourceMediaType(SourceMediaType.ELECTRONIC)
repo_ref.set_media_type(mtype)
with DbTxn(
_("Add Souce/Repo/Note (%s)") % "GeoNames",
self.dbstate.db) as trans:
self.dbstate.db.add_note(new_note, trans)
repo.add_note(new_note.get_handle())
self.dbstate.db.add_repository(repo, trans)
repo_ref.set_reference_handle(repo.handle)
src.add_repo_reference(repo_ref)
self.dbstate.db.add_source(src, trans)
return src.handle
def find_or_make_source(self):
""" Find or create a source.
returns handle to source."""
for hndl in self.dbstate.db.get_source_handles():
if self.dbstate.db.get_raw_source_data(hndl)[2] == 'GeoNames':
return hndl
# No source found, lets add one with associated repo and note
repo = Repository()
repo.set_name("www.geonames.org")
rtype = RepositoryType(RepositoryType.WEBSITE)
repo.set_type(rtype)
url = Url()
url.set_path('http://www.geonames.org/')
url.set_description(_('GeoNames web site'))
url.set_type(UrlType(UrlType.WEB_HOME))
repo.add_url(url)
url = Url()
url.set_path('*****@*****.**')
url.set_description(_('GeoNames author'))
url.set_type(UrlType(UrlType.EMAIL))
repo.add_url(url)
note_txt = StyledText(_(
'GeoNames was founded by Marc Wick. You can reach him at '))
note_txt += StyledText('*****@*****.**' + '\n')
note_txt += StyledText(_(
'GeoNames is a project of Unxos GmbH, Weingartenstrasse 8,'
' 8708 Männedorf, Switzerland.\nThis work is licensed under a '))
note_txt += linkst(
_('Creative Commons Attribution 3.0 License'),
'https://creativecommons.org/licenses/by/3.0/legalcode')
new_note = Note()
new_note.set_styledtext(note_txt)
new_note.set_type(NoteType.REPO)
src = Source()
src.title = 'GeoNames'
src.author = 'Marc Wick'
repo_ref = RepoRef()
mtype = SourceMediaType(SourceMediaType.ELECTRONIC)
repo_ref.set_media_type(mtype)
with DbTxn(_("Add Souce/Repo/Note (%s)") % "GeoNames",
self.dbstate.db) as trans:
self.dbstate.db.add_note(new_note, trans)
repo.add_note(new_note.get_handle())
self.dbstate.db.add_repository(repo, trans)
repo_ref.set_reference_handle(repo.handle)
src.add_repo_reference(repo_ref)
self.dbstate.db.add_source(src, trans)
return src.handle
def _add_source(self,repos=None):
# Add a Source
with DbTxn("Add Source and Citation", self._db) as tran:
source = Source()
if repos is not None:
repo_ref = RepoRef()
repo_ref.set_reference_handle(repos.get_handle())
source.add_repo_reference(repo_ref)
self._db.add_source(source, tran)
self._db.commit_source(source, tran)
citation = Citation()
citation.set_reference_handle(source.get_handle())
self._db.add_citation(citation, tran)
self._db.commit_citation(citation, tran)
return citation
def _add_source(self, repos=None):
# Add a Source
with DbTxn("Add Source and Citation", self._db) as tran:
source = Source()
if repos is not None:
repo_ref = RepoRef()
repo_ref.set_reference_handle(repos.get_handle())
source.add_repo_reference(repo_ref)
self._db.add_source(source, tran)
self._db.commit_source(source, tran)
citation = Citation()
citation.set_reference_handle(source.get_handle())
self._db.add_citation(citation, tran)
self._db.commit_citation(citation, tran)
return citation
def addSource(sourceName, attribs, reftag, repository):
snote = addNote(attribs[3], NoteType.SOURCE)
sidno = db.find_next_source_gramps_id()
source = Source()
source.set_gramps_id(sidno)
source.set_title(sourceName)
source.set_author(attribs[0])
source.set_publication_info(attribs[1])
source.set_abbreviation(attribs[2])
source.add_note(snote.get_handle())
if reftag != None:
source.add_tag(reftag.get_handle())
repoRef = RepoRef()
repoRef.set_reference_handle(repository.get_handle())
source.add_repo_reference(repoRef)
source.set_change_time(chgtime)
with DbTxn(_("Add Source"), db) as trans:
shandle = db.add_source(source, trans)
return source
def buildSource(self, sidno, stitle, tag, repository, shandle, attribs):
if shandle == None:
shandle = id.create_id() # 26-merkkinen tunniste
# repositoryType = RepositoryType()
source = Source()
source.set_handle(shandle)
source.set_gramps_id(sidno)
source.set_title(stitle)
source.set_author(attribs[0])
source.set_publication_info(attribs[1])
source.set_abbreviation(attribs[2])
source.add_tag(tag.get_handle())
repoRef = RepoRef()
repoRef.set_reference_handle(repository.get_handle())
source.add_repo_reference(repoRef)
source.set_change_time(self.chgtime)
# source.set_color("#000000")
# print ("Source ") ; print(source.to_struct())
return ([source, shandle])
def importSourceHierarchies(db, filename, user):
def findNextRidno(ridstrt):
with DbTxn(_("Find next ridno"), db):
prefix_save = db.get_repository_prefix()
db.set_repository_id_prefix(ridstrt + '%05d')
next_ridno = db.find_next_repository_gramps_id()
LOG.debug('Next ridno = ' + next_ridno)
db.set_repository_id_prefix(prefix_save)
return next_ridno
def findNextSidno(ridno):
with DbTxn(_("Find next sidno"), db):
prefix_save = db.get_source_prefix()
db.set_source_id_prefix(ridno + '%05d')
next_sidno = db.find_next_source_gramps_id()
LOG.debug('Next sidno = ' + next_sidno)
db.set_source_id_prefix(prefix_save)
return next_sidno
def checkTagExistence(otext):
with DbTxn(_("Read Tag"), db):
tag = db.get_tag_from_name(otext)
if tag != None:
LOG.debug('Tag found by name, no duplicates: ' + otext + ' ' +
tag.get_name())
else:
tag = Tag()
tag.set_name(otext)
tag.set_color("#EF2929")
with DbTxn(_("Add Tag"), db) as trans:
thandle = db.add_tag(tag, trans)
LOG.debug('Tag added: ' + tag.get_name() + ' ' + thandle)
return tag
fdir = os.path.dirname(filename)
fh = logging.FileHandler(fdir + '\\sourceimport.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
LOG.addHandler(fh)
LOG.info(" fdir = " + fdir)
cout = fdir + "\\sresult.csv"
LOG.debug('ini file handling')
config = configman.register_manager("importsources")
'''
config.register("options.repositoryidrng", "1000")
config.register("options.repositoryincr", "1")
config.register("options.sourceidrng", "1000")
config.register("options.sourceidincr", "1")
'''
config.register("options.refstring", "r")
config.load()
config.save()
'''
repository_idrange = int(config.get('options.repositoryidrng'))
repository_incr = int(config.get('options.repositoryincr'))
source_idrange = int(config.get('options.sourceidrng'))
source_idincr = int(config.get('options.sourceidincr'))
'''
refstr = config.get('options.refstring')
'''
repository_idno = 0
source_idno = 0
'''
t_count = 0
r_count = 0
s_count = 0
c_count = 0
u_count = 0
ridno = None
sidno = None
reftext = 'Referenssi'
reftag = checkTagExistence(reftext)
chgtime = int(time.time())
LOG.info(" chgtime = " + str(chgtime))
try:
with open(cout, 'w', newline='\n', encoding="utf-8-sig") as csv_out:
# csv_out.write(u'\uFEFF') # utf-8-bom
r_writer = csv.writer(csv_out, delimiter=';')
with open(filename, 'r', encoding="utf-8-sig") as t_in:
rhandle = None
t_dialect = csv.Sniffer().sniff(t_in.read(1024))
t_in.seek(0)
t_reader = csv.reader(t_in, t_dialect)
LOG.info('CSV input file delimiter is ' + t_dialect.delimiter)
for row in t_reader:
rectype = row[
0] # Record type = Gramps object id prefix character
# LOG.debug('Row type: -' + row[0] + '-')
if rectype == '#':
LOG.debug('Comment row: ' + row[0])
c_count += 1
r_writer.writerow([
row[0], row[1], row[2], row[3], row[4], row[5],
row[6], row[7]
])
else:
idno = row[
2] # Possibly previously assigned Gramps object id
handle = row[3].strip(
'"'
) # Possibly previously assigned Gramps object handle
otext = row[4].strip()
LOG.debug('Handle = ' + handle)
'''
if rectype == 'T':
LOG.debug('Tag row: ' + row[0])
thandle = ''
t_count += 1
recobj = row[1] # Tag related to repositories or sources
tag = None
with DbTxn(_("Read Tag"), db):
tag = db.get_tag_from_name(otext)
if tag != None:
LOG.info('Tag found by name, no duplicates: ' + otext + ' ' + tag.get_name())
thandle = tag.get_handle()
else:
tag = Tag()
tag.set_name(otext)
tag.set_change_time(chgtime)
tag.set_color("#EF2929")
with DbTxn(_("Add Tag"), db) as trans:
thandle = db.add_tag(tag, trans)
LOG.info('Tag added: ' + tag.get_name() + ' ' + thandle)
tags[recobj] = tag
try:
r_writer.writerow([rectype, recobj, '', '"' + thandle + '"', otext, '', '', '', ''])
except IOError:
LOG.error('Error writing T-csv ' + IOError.strerror)
'''
if rectype == 'R':
LOG.debug('Repository row: ' + row[0])
rhandle = ''
# source_idno = 0
r_count += 1
repotype = row[1] # repository type number
if idno == '':
if refstr == 'r':
# repository type based numbering should be applied but not supplied by Gramps
ridno = findNextRidno(rectype + refstr +
'9')
# repository_idno = repository_idno + repository_incr
# ridno = rectype + refstr + str(int(repotype) * repository_idrange + repository_idno)
else:
ridno = db.find_next_repository_gramps_id()
else:
ridno = idno
LOG.debug('Ridno = ' + str(ridno))
repository = Repository()
if handle != '':
with DbTxn(_("Read Repository"), db) as trans:
repository = db.get_repository_from_handle(
handle)
if repository == None:
LOG.error(
'Repository NOT found by handle: '
+ handle + ' ' + otext)
raise GrampsImportError(
'Repository NOT found by handle: ',
handle + '/' + otext)
repositoryType = RepositoryType()
repositoryType.set(int(repotype))
repository.set_type(repositoryType)
repository.set_gramps_id(ridno)
repository.set_name(otext)
repository.set_change_time(chgtime)
if reftag != None:
repository.add_tag(reftag.get_handle())
if handle == '':
with DbTxn(_("Add Repository"), db) as trans:
rhandle = db.add_repository(
repository, trans)
else:
with DbTxn(_("Update Repository"),
db) as trans:
db.commit_repository(repository, trans)
rhandle = handle
try:
r_writer.writerow([
rectype, repotype, ridno,
'"' + rhandle + '"', otext, '', '', '', ''
])
except IOError:
LOG.error('Error writing R-csv ' +
IOError.strerror)
elif rectype == 'S':
LOG.debug('Source row: ' + row[0])
shandle = ''
sidno = ''
s_count += 1
attribs = (row[5], row[6], row[7])
if idno == '':
LOG.debug('Ridno for sidno = ' + str(ridno))
if refstr == 'r':
# repository type based numbering should be applied but not supplied by Gramps
sidno = findNextSidno(rectype + refstr +
'9')
# source_idno = source_idno + source_idincr
# sidno = rectype + refstr + str((int(repotype) * repository_idrange + repository_idno) * source_idrange + source_idno)
else:
sidno = db.find_next_source_gramps_id()
else:
sidno = idno
LOG.debug('Sidno = ' + str(sidno))
source = Source()
if handle != '':
with DbTxn(_("Read Source"), db) as trans:
source = db.get_source_from_handle(handle)
if source == None:
LOG.error(
'Source NOT found by handle: ' +
handle + ' ' + otext)
raise GrampsImportError(
'Source NOT found by handle: ',
handle + '/' + otext)
source.set_gramps_id(sidno)
source.set_title(otext)
source.set_author(attribs[0])
source.set_publication_info(attribs[1])
source.set_abbreviation(attribs[2])
if reftag != None:
source.add_tag(reftag.get_handle())
repoRef = RepoRef()
repoRef.set_reference_handle(rhandle)
source.add_repo_reference(repoRef)
source.set_change_time(chgtime)
if handle == '':
with DbTxn(_("Add Source"), db) as trans:
shandle = db.add_source(source, trans)
else:
with DbTxn(_("Update Source"), db) as trans:
db.commit_source(source, trans)
shandle = handle
try:
r_writer.writerow([
rectype, '', sidno, '"' + shandle + '"',
otext, attribs[0], attribs[1], attribs[2],
''
])
except IOError:
LOG.error('Error writing S-csv ' +
IOError.strerror)
else:
u_count += 1
LOG.debug('Unknown rectype: ' + rectype)
raise GrampsImportError('Unknown record type ' +
rectype)
except:
exc = sys.exc_info()[0]
LOG.error('*** Something went really wrong! ', exc)
return ImportInfo({_('Results'): _('Something went really wrong ')})
results = {
_('Results'):
_('Input file handled.'),
_(' Repositories '):
str(r_count),
_(' Sources '):
str(s_count),
_(' Comments '):
str(c_count),
_(' Unknown types '):
str(u_count),
_(' Total '):
str(t_count + r_count + s_count + c_count + u_count)
}
LOG.info('Input file handled.')
LOG.info(' Repositories ' + str(r_count))
LOG.info(' Sources ' + str(s_count))
LOG.info(' Comments ' + str(c_count))
LOG.info(' Unknown types ' + str(u_count))
LOG.info(' Total ' +
str(t_count + r_count + s_count + c_count + u_count))
db.enable_signals()
db.request_rebuild()
return ImportInfo(results)
def run(self):
objects = []
skipped = 0
total_objects = 0
total_notes = 0
matching_notes = 0
citations_added = 0
sources_added = 0
repos_added = 0
with self.user.progress(_("Generating citations"), '',
self.db.get_total()) as step:
repos = self.get_repos()
sources = self.get_sources()
notes_to_remove = {}
for classname, obj in self.yield_objects(step):
total_objects += 1
obj.classname = classname
obj.notes = []
obj.citations = []
current_citations = None
for note, m, notehandle in self.find_matching_notes(obj):
matching_notes += 1
# get a list of current citations; this is in the loop to avoid computing the set for all objects
if current_citations is None:
current_citations = set()
for type, h in obj.get_referenced_citation_handles(
): # type = 'Citation'
citation = self.db.get_citation_from_handle(h)
page = citation.get_page()
current_citations.add(
(page, citation.source_handle))
source = sources[m.sourcetitle]
source.set_title(m.sourcetitle)
if (m.citationpage, source.handle) in current_citations:
skipped += 1
continue # already has this citation, skip
repo = repos[m.reponame]
repo.set_name(m.reponame)
source.repo = repo
citation = Citation()
citation.set_page(m.citationpage)
citation.source = source
citation.note = m.details
obj.citations.append(citation)
obj.notes.append(notehandle)
notelines = note.get().splitlines()
if len(notelines) == 1:
notes_to_remove[notehandle] = m
if obj.notes: objects.append(obj)
with DbTxn(_("Add citations"), self.db) as trans:
for obj in objects:
self.log("Object {}".format(obj))
for citation in obj.citations:
source = citation.source
repo = source.repo
citation_handle = self.db.add_citation(citation, trans)
source_handle = self.db.add_source(source, trans)
repo_handle = self.db.add_repository(repo, trans)
self.log("- Adding citation: {}".format(
citation.get_page()))
if not hasattr(repo, "in_db"):
repos_added += 1
self.log("- Adding repo: {}".format(
repo.get_name()))
repo.in_db = True
if not hasattr(source, "in_db"):
sources_added += 1
self.log("- Adding source: {}".format(
source.get_title()))
source.in_db = True
note = Note()
note.set(citation.note)
note.set_type(NoteType.LINK)
note_handle = self.db.add_note(note, trans)
citation.add_note(note_handle)
citation.newnote = note
citation.set_reference_handle(source_handle)
if not source.has_repo_reference(repo_handle):
reporef = RepoRef()
reporef.set_reference_handle(repo_handle)
source.add_repo_reference(reporef)
obj.add_citation(citation_handle)
citations_added += 1
for notehandle in obj.notes:
if notehandle in notes_to_remove:
obj.remove_note(notehandle)
commit_func = self.primary_objects[
obj.primary_object_class]['commit_func']
commit_func(obj.primary_object, trans)
for citation in obj.citations:
self.db.commit_citation(citation, trans)
self.db.commit_source(citation.source, trans)
self.db.commit_repository(source.repo, trans)
self.db.commit_note(citation.newnote, trans)
for notehandle, m in notes_to_remove.items():
self.log("Removing note: {} {}".format(
repr(notehandle), m.line))
self.db.remove_note(notehandle, trans)
log = []
log.append(
_("Total objects processed: {total_objects}").format(
total_objects=total_objects))
log.append("- " + _("Total notes: {total_notes}").format(
total_notes=self.total_notes))
log.append("- " + _("Matching notes: {matching_notes}").format(
matching_notes=matching_notes))
log.append("- " +
_("Unique matching notes: {unique_matching_notes}"
).format(unique_matching_notes=len(notes_to_remove)))
log.append("- " +
_("Skipped notes: {skipped}").format(skipped=skipped))
log.append("- " + _("Citations added: {citations_added}").format(
citations_added=citations_added))
log.append("- " + _("Sources added: {sources_added}").format(
sources_added=sources_added))
log.append("- " + _("Repositories added: {repos_added}").format(
repos_added=repos_added))
msg = "\n".join(log)
msg += "\n\n" + _("You can undo all changes with Ctrl-Z.")
self.log(msg)
OkDialog(_("All events processed"),
msg,
parent=self.uistate.window)
