def importData(dbase, filename, user):
"""Function called by Gramps to import data on persons in CSV format."""
if dbase.get_feature("skip-import-additions"): # don't add source or tags
parser = CSVParser(dbase, user, None)
else:
parser = CSVParser(dbase, user,
(config.get('preferences.tag-on-import-format') if
config.get('preferences.tag-on-import') else None))
try:
with open(filename, 'rb') as filehandle:
line = filehandle.read(3)
if line == codecs.BOM_UTF8:
filehandle.seek(0)
filehandle = TextIOWrapper(filehandle,
encoding='utf_8_sig',
errors='replace',
newline='')
else: # just open with OS encoding
filehandle.seek(0)
filehandle = TextIOWrapper(filehandle,
errors='replace',
newline='')
parser.parse(filehandle)
except EnvironmentError as err:
user.notify_error(_("%s could not be opened\n") % filename, str(err))
return
return ImportInfo({_("Results"): _("done")})
def _importData(database, filename, user):
"""
Imports the files corresponding to the specified Database, Filename & User.
"""
data = ProgenOptions(database, filename, user)
if data.fail: # 'Cancel' button pressed
return
data = libprogen.ProgenParser(database, filename, user, data.option)
try:
info = libprogen.ProgenParser.parse_progen_file(data)
except libprogen.ProgenError as msg:
user.notify_error(_("Pro-Gen data error"), str(msg))
return
except IOError as msg:
user.notify_error(_("%s could not be opened") % filename, str(msg))
return
if info: # successful import
# display qualified/standard statistic window
if user.uistate:
InfoDialog(_('Import Statistics'),
info.info_text(),
parent=user.parent)
else:
return ImportInfo({_("Results"): _("done")})
def importData(database, filename, user):
"""Function called by Gramps to import data on persons in VCard format."""
parser = VCardParser(database)
try:
with OpenFileOrStdin(filename) as filehandle:
parser.parse(filehandle, user)
except EnvironmentError as msg:
user.notify_error(_("%s could not be opened\n") % filename, str(msg))
return
except GrampsImportError as msg:
user.notify_error(_("%s could not be opened\n") % filename, str(msg))
return
return ImportInfo({_("Results"): _("done")})
def importData(dbase, filename, user):
"""Function called by Gramps to import data on persons in CSV format."""
if dbase.get_feature("skip-import-additions"): # don't add source or tags
parser = CSVParser(dbase, user, None)
else:
parser = CSVParser(dbase, user, (config.get('preferences.tag-on-import-format') if
config.get('preferences.tag-on-import') else None))
try:
with open(filename, 'r') as filehandle:
parser.parse(filehandle)
except EnvironmentError as err:
user.notify_error(_("%s could not be opened\n") % filename, str(err))
return
return ImportInfo({_("Results"): _("done")})
def importData(database, filename, user):
"""Function called by Gramps to import data on persons in VCard format."""
parser = VCardParser(database)
try:
with OpenFileOrStdin(filename) as filehandle:
parser.parse(filehandle, user)
except EnvironmentError as msg:
user.notify_error(_("%s could not be opened\n") % filename, str(msg))
return
except GrampsImportError as msg:
user.notify_error(_("%s could not be opened\n") % filename, str(msg))
return
## a "VCARD import report" happens in VCardParser so this is not needed:
## (but the imports_test.py unittest currently requires it, so here it is)
return ImportInfo({_("Results"): _("done")})
def importData(database, filename, user):
global callback
try:
g = GeneWebParser(database, filename)
except IOError as msg:
user.notify_error(_("%s could not be opened\n") % filename, str(msg))
return
try:
status = g.parse_geneweb_file()
except IOError as msg:
errmsg = _("%s could not be opened\n") % filename
user.notify_error(errmsg, str(msg))
return
return ImportInfo({_("Results"): _("done")})
def importSourceHierarchies(db, filename, user):
def findNextRidno(ridstrt):
with DbTxn(_("Find next ridno"), db):
prefix_save = db.get_repository_prefix()
db.set_repository_id_prefix(ridstrt + '%05d')
next_ridno = db.find_next_repository_gramps_id()
LOG.debug('Next ridno = ' + next_ridno)
db.set_repository_id_prefix(prefix_save)
return next_ridno
def findNextSidno(ridno):
with DbTxn(_("Find next sidno"), db):
prefix_save = db.get_source_prefix()
db.set_source_id_prefix(ridno + '%05d')
next_sidno = db.find_next_source_gramps_id()
LOG.debug('Next sidno = ' + next_sidno)
db.set_source_id_prefix(prefix_save)
return next_sidno
def checkTagExistence(otext):
with DbTxn(_("Read Tag"), db):
tag = db.get_tag_from_name(otext)
if tag != None:
LOG.debug('Tag found by name, no duplicates: ' + otext + ' ' +
tag.get_name())
else:
tag = Tag()
tag.set_name(otext)
tag.set_color("#EF2929")
with DbTxn(_("Add Tag"), db) as trans:
thandle = db.add_tag(tag, trans)
LOG.debug('Tag added: ' + tag.get_name() + ' ' + thandle)
return tag
fdir = os.path.dirname(filename)
fh = logging.FileHandler(fdir + '\\sourceimport.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
LOG.addHandler(fh)
LOG.info(" fdir = " + fdir)
cout = fdir + "\\sresult.csv"
LOG.debug('ini file handling')
config = configman.register_manager("importsources")
'''
config.register("options.repositoryidrng", "1000")
config.register("options.repositoryincr", "1")
config.register("options.sourceidrng", "1000")
config.register("options.sourceidincr", "1")
'''
config.register("options.refstring", "r")
config.load()
config.save()
'''
repository_idrange = int(config.get('options.repositoryidrng'))
repository_incr = int(config.get('options.repositoryincr'))
source_idrange = int(config.get('options.sourceidrng'))
source_idincr = int(config.get('options.sourceidincr'))
'''
refstr = config.get('options.refstring')
'''
repository_idno = 0
source_idno = 0
'''
t_count = 0
r_count = 0
s_count = 0
c_count = 0
u_count = 0
ridno = None
sidno = None
reftext = 'Referenssi'
reftag = checkTagExistence(reftext)
chgtime = int(time.time())
LOG.info(" chgtime = " + str(chgtime))
try:
with open(cout, 'w', newline='\n', encoding="utf-8-sig") as csv_out:
# csv_out.write(u'\uFEFF') # utf-8-bom
r_writer = csv.writer(csv_out, delimiter=';')
with open(filename, 'r', encoding="utf-8-sig") as t_in:
rhandle = None
t_dialect = csv.Sniffer().sniff(t_in.read(1024))
t_in.seek(0)
t_reader = csv.reader(t_in, t_dialect)
LOG.info('CSV input file delimiter is ' + t_dialect.delimiter)
for row in t_reader:
rectype = row[
0] # Record type = Gramps object id prefix character
# LOG.debug('Row type: -' + row[0] + '-')
if rectype == '#':
LOG.debug('Comment row: ' + row[0])
c_count += 1
r_writer.writerow([
row[0], row[1], row[2], row[3], row[4], row[5],
row[6], row[7]
])
else:
idno = row[
2] # Possibly previously assigned Gramps object id
handle = row[3].strip(
'"'
) # Possibly previously assigned Gramps object handle
otext = row[4].strip()
LOG.debug('Handle = ' + handle)
'''
if rectype == 'T':
LOG.debug('Tag row: ' + row[0])
thandle = ''
t_count += 1
recobj = row[1] # Tag related to repositories or sources
tag = None
with DbTxn(_("Read Tag"), db):
tag = db.get_tag_from_name(otext)
if tag != None:
LOG.info('Tag found by name, no duplicates: ' + otext + ' ' + tag.get_name())
thandle = tag.get_handle()
else:
tag = Tag()
tag.set_name(otext)
tag.set_change_time(chgtime)
tag.set_color("#EF2929")
with DbTxn(_("Add Tag"), db) as trans:
thandle = db.add_tag(tag, trans)
LOG.info('Tag added: ' + tag.get_name() + ' ' + thandle)
tags[recobj] = tag
try:
r_writer.writerow([rectype, recobj, '', '"' + thandle + '"', otext, '', '', '', ''])
except IOError:
LOG.error('Error writing T-csv ' + IOError.strerror)
'''
if rectype == 'R':
LOG.debug('Repository row: ' + row[0])
rhandle = ''
# source_idno = 0
r_count += 1
repotype = row[1] # repository type number
if idno == '':
if refstr == 'r':
# repository type based numbering should be applied but not supplied by Gramps
ridno = findNextRidno(rectype + refstr +
'9')
# repository_idno = repository_idno + repository_incr
# ridno = rectype + refstr + str(int(repotype) * repository_idrange + repository_idno)
else:
ridno = db.find_next_repository_gramps_id()
else:
ridno = idno
LOG.debug('Ridno = ' + str(ridno))
repository = Repository()
if handle != '':
with DbTxn(_("Read Repository"), db) as trans:
repository = db.get_repository_from_handle(
handle)
if repository == None:
LOG.error(
'Repository NOT found by handle: '
+ handle + ' ' + otext)
raise GrampsImportError(
'Repository NOT found by handle: ',
handle + '/' + otext)
repositoryType = RepositoryType()
repositoryType.set(int(repotype))
repository.set_type(repositoryType)
repository.set_gramps_id(ridno)
repository.set_name(otext)
repository.set_change_time(chgtime)
if reftag != None:
repository.add_tag(reftag.get_handle())
if handle == '':
with DbTxn(_("Add Repository"), db) as trans:
rhandle = db.add_repository(
repository, trans)
else:
with DbTxn(_("Update Repository"),
db) as trans:
db.commit_repository(repository, trans)
rhandle = handle
try:
r_writer.writerow([
rectype, repotype, ridno,
'"' + rhandle + '"', otext, '', '', '', ''
])
except IOError:
LOG.error('Error writing R-csv ' +
IOError.strerror)
elif rectype == 'S':
LOG.debug('Source row: ' + row[0])
shandle = ''
sidno = ''
s_count += 1
attribs = (row[5], row[6], row[7])
if idno == '':
LOG.debug('Ridno for sidno = ' + str(ridno))
if refstr == 'r':
# repository type based numbering should be applied but not supplied by Gramps
sidno = findNextSidno(rectype + refstr +
'9')
# source_idno = source_idno + source_idincr
# sidno = rectype + refstr + str((int(repotype) * repository_idrange + repository_idno) * source_idrange + source_idno)
else:
sidno = db.find_next_source_gramps_id()
else:
sidno = idno
LOG.debug('Sidno = ' + str(sidno))
source = Source()
if handle != '':
with DbTxn(_("Read Source"), db) as trans:
source = db.get_source_from_handle(handle)
if source == None:
LOG.error(
'Source NOT found by handle: ' +
handle + ' ' + otext)
raise GrampsImportError(
'Source NOT found by handle: ',
handle + '/' + otext)
source.set_gramps_id(sidno)
source.set_title(otext)
source.set_author(attribs[0])
source.set_publication_info(attribs[1])
source.set_abbreviation(attribs[2])
if reftag != None:
source.add_tag(reftag.get_handle())
repoRef = RepoRef()
repoRef.set_reference_handle(rhandle)
source.add_repo_reference(repoRef)
source.set_change_time(chgtime)
if handle == '':
with DbTxn(_("Add Source"), db) as trans:
shandle = db.add_source(source, trans)
else:
with DbTxn(_("Update Source"), db) as trans:
db.commit_source(source, trans)
shandle = handle
try:
r_writer.writerow([
rectype, '', sidno, '"' + shandle + '"',
otext, attribs[0], attribs[1], attribs[2],
''
])
except IOError:
LOG.error('Error writing S-csv ' +
IOError.strerror)
else:
u_count += 1
LOG.debug('Unknown rectype: ' + rectype)
raise GrampsImportError('Unknown record type ' +
rectype)
except:
exc = sys.exc_info()[0]
LOG.error('*** Something went really wrong! ', exc)
return ImportInfo({_('Results'): _('Something went really wrong ')})
results = {
_('Results'):
_('Input file handled.'),
_(' Repositories '):
str(r_count),
_(' Sources '):
str(s_count),
_(' Comments '):
str(c_count),
_(' Unknown types '):
str(u_count),
_(' Total '):
str(t_count + r_count + s_count + c_count + u_count)
}
LOG.info('Input file handled.')
LOG.info(' Repositories ' + str(r_count))
LOG.info(' Sources ' + str(s_count))
LOG.info(' Comments ' + str(c_count))
LOG.info(' Unknown types ' + str(u_count))
LOG.info(' Total ' +
str(t_count + r_count + s_count + c_count + u_count))
db.enable_signals()
db.request_rebuild()
return ImportInfo(results)
def importReferenceSources(db, filename, user):
def findNextRidno(ridstrt):
with DbTxn(_("Find next ridno"), db):
prefix_save = db.get_repository_prefix()
db.set_repository_id_prefix(ridstrt + '%04d')
next_ridno = db.find_next_repository_gramps_id()
LOG.debug('Next ridno = ' + next_ridno)
db.set_repository_id_prefix(prefix_save)
return next_ridno
def findNextSidno(ridno):
with DbTxn(_("Find next sidno"), db):
prefix_save = db.get_source_prefix()
db.set_source_id_prefix(ridno + '%04d')
next_sidno = db.find_next_source_gramps_id()
LOG.debug('Next sidno = ' + next_sidno)
db.set_source_id_prefix(prefix_save)
return next_sidno
def addRepository(repositoryName, repositoryUrl, reftag):
ridno = db.find_next_repository_gramps_id()
repository = Repository()
repositoryType = RepositoryType()
repositoryType.set(RepositoryType.ARCHIVE)
repository.set_type(repositoryType)
repository.set_gramps_id(ridno)
repository.set_name(repositoryName)
urlList = []
# for urlPath in repositoryUrls:
url = Url()
url.set_path(repositoryUrl)
url.set_description('Sshy')
url.set_type(UrlType.WEB_HOME)
urlList.append(url)
repository.set_url_list(urlList)
repository.set_change_time(chgtime)
if reftag != None:
repository.add_tag(reftag.get_handle())
with DbTxn(_("Add Repository"), db) as trans:
rhandle = db.add_repository(repository, trans)
return repository
def addSource(sourceName, attribs, reftag, repository):
snote = addNote(attribs[3], NoteType.SOURCE)
sidno = db.find_next_source_gramps_id()
source = Source()
source.set_gramps_id(sidno)
source.set_title(sourceName)
source.set_author(attribs[0])
source.set_publication_info(attribs[1])
source.set_abbreviation(attribs[2])
source.add_note(snote.get_handle())
if reftag != None:
source.add_tag(reftag.get_handle())
repoRef = RepoRef()
repoRef.set_reference_handle(repository.get_handle())
source.add_repo_reference(repoRef)
source.set_change_time(chgtime)
with DbTxn(_("Add Source"), db) as trans:
shandle = db.add_source(source, trans)
return source
def addNote(ntext, ntype):
nidno = db.find_next_note_gramps_id()
note = Note(ntext)
note.set_gramps_id(nidno)
note.set_type(ntype)
if reftag != None:
note.add_tag(reftag.get_handle())
note.set_change_time(chgtime)
with DbTxn(_("Add Note"), db) as trans:
nhandle = db.add_note(note, trans)
LOG.debug('Note added: ' + ntext + ' ' + nhandle)
return note
def checkTagExistence(otext):
with DbTxn(_("Read Tag"), db):
tag = db.get_tag_from_name(otext)
if tag != None:
LOG.debug('Tag found by name, no duplicates: ' + otext + ' ' +
tag.get_name())
else:
tag = Tag()
tag.set_name(otext)
tag.set_color("#EF2929")
with DbTxn(_("Add Tag"), db) as trans:
thandle = db.add_tag(tag, trans)
LOG.debug('Tag added: ' + tag.get_name() + ' ' + thandle)
return tag
fdir = os.path.dirname(filename)
fh = logging.FileHandler(fdir + '\\sourceimport.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
LOG.addHandler(fh)
LOG.info(" fdir = " + fdir)
# cout = fdir + "\\sresult.csv"
LOG.debug('ini file handling')
config = configman.register_manager("importsources")
config.register("options.refstring", "r")
config.load()
config.save()
refstr = config.get('options.refstring')
r_count = 0
s_count = 0
u_count = 0
reftext = 'Referenssi'
reftag = checkTagExistence(reftext)
chgtime = int(time.time())
LOG.info(" chgtime = " + str(chgtime))
try:
with open(filename, 'r', encoding="utf-8") as t_in:
t_dialect = csv.Sniffer().sniff(t_in.read(2048))
t_dialect.delimiter = ";"
t_in.seek(0)
t_reader = csv.reader(t_in, t_dialect)
LOG.info('CSV input file delimiter is ' + t_dialect.delimiter)
for row in t_reader:
rectype = row[
0] # Record type = Gramps object id prefix character
# otext = row[4].strip()
if rectype == 'Arkisto':
repoName = row[1]
LOG.debug('New repository: ' + repoName)
r_count += 1
# repotype = row[1] # repository type number
repoUrl = row[3]
LOG.debug("===========================> " + repoUrl)
repository = addRepository(repoName, repoUrl, reftag)
elif rectype == 'Lähde':
sourceName = row[1]
LOG.debug('Source row: ' + sourceName)
s_count += 1
sourceNote = 'Sshy: ' + row[4]
attribs = (
"", "", "", sourceNote
) # Author, Publication-info, Abbreviation, Note-text
LOG.debug('New source: ' + sourceName)
addSource(sourceName, attribs, reftag, repository)
else:
u_count += 1
LOG.debug('Unknown rectype: ' + rectype)
# raise GrampsImportError('Unknown record type ' + rectype)
except:
exc = sys.exc_info()[0]
LOG.error('*** Something went really wrong! ', exc)
return ImportInfo({_('Results'): _('Something went really wrong ')})
results = {
_('Results'): _('Input file handled.'),
_(' Repositories '): str(r_count),
_(' Sources '): str(s_count)
}
LOG.info('Input file handled.')
LOG.info(' Repositories ' + str(r_count))
LOG.info(' Sources ' + str(s_count))
db.enable_signals()
db.request_rebuild()
return ImportInfo(results)
def importPlaceHierarchy(db, filename, user):
def findNextPidno(pidstrt):
with DbTxn(_("Find next pidno"), db):
db.set_repository_id_prefix(pidstrt + '%04d')
next_pidno = db.find_next_place_gramps_id()
LOG.debug('Next pidno = ' + next_pidno)
db.set_repository_id_prefix('R%04d')
return next_pidno
def findNextSidno(ridno):
with DbTxn(_("Find next sidno"), db):
db.set_place_id_prefix(ridno + '%03d')
next_sidno = db.find_next_place_gramps_id()
LOG.debug('Next sidno = ' + next_sidno)
db.set_place_id_prefix('S%04d')
return next_sidno
def findPlace(id, handle, name):
pid = ''
phandle = ''
place = None
if handle != '':
with DbTxn(_("Read place"), db) as trans:
place = db.get_place_from_handle(handle)
if place != None:
LOG.info('Place read by handle: ' + handle + ' ' +
place.get_name())
else:
LOG.error('Place NOT found by handle: ' + handle + ' ' +
pname)
raise GrampsImportError('Place NOT found by handle: ',
handle + '/' + pname)
return place
def addPlace(pname, ptype, refPlace=None):
place = Place()
placeName = PlaceName()
placeName.set_value(pname)
place.set_name(placeName)
# place.set_change_time(chgtime)
place.set_type(ptype)
place.add_tag(tags[ptype])
if refPlace != None:
placeRef = PlaceRef()
placeRef.set_reference_handle(refPlace.get_handle())
place.add_placeref(placeRef)
# tag.set_color("#EF2929")
with DbTxn(_("Add Place"), db) as trans:
phandle = db.add_place(place, trans)
LOG.debug('Place added: ' + place.get_name().get_value() + ' ' +
phandle)
return place
fdir = os.path.dirname(filename)
'''
fh = logging.FileHandler(fdir + '\\placeimport.log')
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
LOG.addHandler(fh)
'''
LOG.info(" fdir = " + fdir)
cout = fdir + "\\result0.csv"
LOG.debug('ini file handling')
'''
config = configman.register_manager("importplaces")
config.register("options.repositoryidrng", "1000")
config.register("options.repositoryincr", "1")
config.register("options.placeidrng", "1000")
config.register("options.placeidincr", "1")
config.register("options.refstring", "r")
config.load()
config.save()
repository_idrange = int(config.get('options.repositoryidrng'))
repository_incr = int(config.get('options.repositoryincr'))
place_idrange = int(config.get('options.placeidrng'))
place_idincr = int(config.get('options.placeidincr'))
refstr = config.get('options.refstring')
repository_idno = 0
place_idno = 0
'''
h_count = 0 # Header line count
t_count = 0 # Tag count
p_count = 0 # Province count (lääni)
c_count = 0 # Community count (kunta)
v_count = 0 # Village count (kylä)
u_count = 0 # Unknown row count
pidno = None
sidno = None
country = None
province = None
municipality = None
village = None
placeTypes = {"lääni": 1, "kunta": 2, "kylä": 3}
tags = {
PlaceType.COUNTRY: "Referenssivaltio",
PlaceType.PROVINCE: "Referenssilääni",
PlaceType.MUNICIPALITY: "Referenssikunta",
PlaceType.VILLAGE: "Referenssikylä"
} # Dictionary recordtype: tag
chgtime = int(time.time())
LOG.info(" chgtime = " + str(chgtime))
try:
with open(cout, 'w', newline='\n', encoding="utf-8") as csv_out:
r_writer = csv.writer(csv_out, delimiter=';')
with open(filename, 'r', encoding="utf-8") as t_in:
rhandle = None
phandle = None
t_dialect = csv.Sniffer().sniff(t_in.read(1024))
t_in.seek(0)
t_reader = csv.reader(t_in, t_dialect)
LOG.info('CSV input file delimiter is ' + t_dialect.delimiter)
for row in t_reader:
rectype = row[
0] # Record type = Gramps place type name (fi_FI)
LOG.debug('Row type: -' + row[0] + '-')
if rectype == 'type':
LOG.debug('First row: ' + row[0])
h_count += 1
r_writer.writerow([
row[0], row[1], row[2], row[3], row[4], row[5],
row[6], row[7]
])
else:
idno = row[
1] # Possibly previously assigned Gramps object id
handle = row[2].strip(
'"'
) # Possibly previously assigned Gramps object handle
pname = row[3].strip()
LOG.debug("Rectype " + rectype + ' name = ' + pname)
if rectype == 'valtio':
result = findPlace(idno, handle, pname)
if result != None:
country = result
else:
country = addPlace(pname, PlaceType.COUNTRY)
if handle != '':
with DbTxn(_("Update Country"), db) as trans:
db.commit_place(country, trans)
chandle = country.get_handle()
cidno = country.get_gramps_id()
try:
r_writer.writerow([
row[0], row[1], cidno, chandle, row[4],
row[5], row[6], row[7]
])
except IOError:
LOG.error('Error writing country-csv ' +
IOError.strerror)
elif rectype == 'lääni':
LOG.debug('Province row: ' + pname)
p_count += 1
result = findPlace(pidno, phandle, pname)
if result != None:
province = result
LOG.info(
'Province found by name, no duplicates: ' +
pname + ' ' +
province.get_name().get_value())
else:
province = addPlace(pname, PlaceType.PROVINCE,
country)
if handle != '':
with DbTxn(_("Update Province"), db) as trans:
db.commit_place(province, trans)
phandle = province.get_handle()
pidno = province.get_gramps_id()
try:
r_writer.writerow([
row[0], row[1], pidno, phandle, row[4],
row[5], row[6], row[7]
])
except IOError:
LOG.error('Error writing province-csv ' +
IOError.strerror)
elif rectype == 'kunta':
LOG.debug('Municipality row: ' + pname)
c_count += 1
result = findPlace(pidno, phandle, pname)
if result != None:
municipality = result
else:
municipality = addPlace(
pname, PlaceType.MUNICIPALITY, province)
if handle != '':
with DbTxn(_("Update Municipality"),
db) as trans:
db.commit_place(municipality, trans)
mhandle = municipality.get_handle()
midno = municipality.get_gramps_id()
try:
r_writer.writerow([
row[0], row[1], midno, mhandle, row[4],
row[5], row[6], row[7]
])
except IOError:
LOG.error('Error writing municipality-csv ' +
IOError.strerror)
elif rectype == 'kylä':
LOG.debug('Village row: ' + pname)
v_count += 1
result = findPlace(pidno, phandle, pname)
if result != None:
village = result
else:
village = addPlace(pname, PlaceType.VILLAGE,
municipality)
if handle != '':
with DbTxn(_("Update Village"), db) as trans:
db.commit_place(village, trans)
vhandle = village.get_handle()
vidno = village.get_gramps_id()
try:
r_writer.writerow([
row[0], row[1], vidno, vhandle, row[4],
row[5], row[6], row[7]
])
except IOError:
LOG.error('Error writing village-csv ' +
IOError.strerror)
else:
u_count += 1
LOG.error('Unknown rectype: ' + rectype)
raise GrampsImportError('Unknown record type ' +
rectype)
except:
exc = sys.exc_info()[0]
LOG.error('*** Something went really wrong! ', exc)
return ImportInfo({_('Results'): _('Something went really wrong ')})
results = {
_('Results'):
_('Input file handled.'),
_(' Tags '):
str(t_count),
_(' Provinces '):
str(p_count),
_(' Communities '):
str(c_count),
_(' Villages '):
str(v_count),
_(' Unknown types '):
str(u_count),
_(' Total '):
str(t_count + p_count + c_count + v_count + u_count)
}
LOG.info('Input file handled.')
LOG.info(' Tags ' + str(t_count))
LOG.info(' Provinces ' + str(p_count))
LOG.info(' Communities ' + str(c_count))
LOG.info(' Villages ' + str(v_count))
LOG.info(' Unknown types ' + str(u_count))
LOG.info(' Total ' +
str(t_count + p_count + c_count + v_count + u_count))
db.enable_signals()
db.request_rebuild()
return ImportInfo(results)
def importData(database, filename, user):
"""
Try to handle ANSEL encoded files that are not really ANSEL encoded
"""
if DbMixin not in database.__class__.__bases__:
database.__class__.__bases__ = (DbMixin,) + \
database.__class__.__bases__
try:
# Opening in utf-8 with universal newline to allow cr, lf, and crlf
# If the file is really UTF16 or a varient, the next block code will not
# find anything even if it is there, but this is ok since it won't be
# ANSEL, or is inconsistent...
with open(filename,
"r",
encoding='utf-8',
errors='replace',
newline=None) as ifile:
ansel = False
gramps = False
for index in range(50):
# Treat the file as though it is UTF-8 since this is the more
# modern option; and anyway it doesn't really matter as we are
# only trying to detect a CHAR or SOUR line which is only
# 7-bit ASCII anyway, and we ignore anything that can't be
# translated.
line = ifile.readline()
line = line.split()
if len(line) == 0:
break
if len(line) > 2 and line[1][0:4] == 'CHAR' \
and line[2] == "ANSEL":
ansel = True
if len(line) > 2 and line[1][0:4] == 'SOUR' \
and line[2] == "GRAMPS":
gramps = True
except IOError:
return
if not gramps and ansel and user.uistate:
top = Glade()
code = top.get_object('codeset')
code.set_active(0)
dialog = top.toplevel
dialog.set_transient_for(user.uistate.window)
dialog.run()
enc = ['ANSEL', 'ANSEL', 'ANSI', 'ASCII', 'UTF-8']
code_set = enc[code.get_active()]
dialog.destroy()
else:
code_set = ""
assert (isinstance(code_set, str))
try:
ifile = open(filename, "rb")
stage_one = libgedcom.GedcomStageOne(ifile)
stage_one.parse()
if code_set:
stage_one.set_encoding(code_set)
ifile.seek(0)
if database.get_feature(
"skip-import-additions"): # don't add source or tags
gedparse = libgedcom.GedcomParser(database, ifile, filename, user,
stage_one, None, None)
else:
gedparse = libgedcom.GedcomParser(
database, ifile, filename, user, stage_one,
config.get('preferences.default-source'),
(config.get('preferences.tag-on-import-format')
if config.get('preferences.tag-on-import') else None))
except IOError as msg:
user.notify_error(_("%s could not be opened\n") % filename, str(msg))
return
except GedcomError as msg:
user.notify_error(
_("Invalid GEDCOM file"),
_("%s could not be imported") % filename + "\n" + str(msg))
return
try:
read_only = database.readonly
database.readonly = False
gedparse.parse_gedcom_file(False)
database.readonly = read_only
ifile.close()
except IOError as msg:
msg = _("%s could not be opened\n") % filename
user.notify_error(msg, str(msg))
return
except DbError as msg:
user.notify_db_error(str(msg.value))
return
except GedcomError as msg:
user.notify_error(_('Error reading GEDCOM file'), str(msg))
return
## a "GEDCOM import report" happens in GedcomParser so this is not needed:
## (but the imports_test.py unittest currently requires it, so here it is)
return ImportInfo({_("Results"): _("done")})
def importData(database, filename, user):
"""
Try to handle ANSEL encoded files that are not really ANSEL encoded
"""
if DbMixin not in database.__class__.__bases__:
database.__class__.__bases__ = (DbMixin,) + \
database.__class__.__bases__
try:
with open(filename, "rb") as ifile:
ansel = False
gramps = False
for index in range(50):
# Treat the file as though it is UTF-8 since this is the more modern
# option; and anyway it doesn't really matter as we are only trying to
# detect a CHAR or SOUR line which is only 7-bit ASCII anyway, and we
# ignore anything that can't be translated.
line = ifile.readline()
line = line.decode(encoding='utf-8', errors='replace')
line = line.split()
if len(line) == 0:
break
if len(line
) > 2 and line[1][0:4] == 'CHAR' and line[2] == "ANSEL":
ansel = True
if len(line) > 2 and line[1][0:4] == 'SOUR' and line[
2] == "GRAMPS":
gramps = True
except IOError:
return
if not gramps and ansel:
top = Glade()
code = top.get_object('codeset')
code.set_active(0)
dialog = top.toplevel
dialog.run()
enc = ['ANSEL', 'ANSEL', 'ANSI', 'ASCII', 'UTF-8']
code_set = enc[code.get_active()]
dialog.destroy()
else:
code_set = ""
assert (isinstance(code_set, str))
try:
ifile = open(filename, "rb")
stage_one = libgedcom.GedcomStageOne(ifile)
stage_one.parse()
if code_set:
stage_one.set_encoding(code_set)
ifile.seek(0)
if database.get_feature(
"skip-import-additions"): # don't add source or tags
gedparse = libgedcom.GedcomParser(database, ifile, filename, user,
stage_one, None, None)
else:
gedparse = libgedcom.GedcomParser(
database, ifile, filename, user, stage_one,
config.get('preferences.default-source'),
(config.get('preferences.tag-on-import-format')
if config.get('preferences.tag-on-import') else None))
except IOError as msg:
user.notify_error(_("%s could not be opened\n") % filename, str(msg))
return
except GedcomError as msg:
user.notify_error(
_("Invalid GEDCOM file"),
_("%s could not be imported") % filename + "\n" + str(msg))
return
try:
read_only = database.readonly
database.readonly = False
gedparse.parse_gedcom_file(False)
database.readonly = read_only
ifile.close()
except IOError as msg:
msg = _("%s could not be opened\n") % filename
user.notify_error(msg, str(msg))
return
except DbError as msg:
user.notify_db_error(str(msg.value))
return
except GedcomError as msg:
user.notify_error(_('Error reading GEDCOM file'), str(msg))
return
return ImportInfo({_("Results"): _("done")})
def importPlaceHierarchy(db, filename, user):
all_countries = [] # Countries in the database
states_of_current_country = [] # States in the current country
municipalities_of_current_state = [] # Municipalities in the current state
villages_of_current_municipality = [
] # Villages in the current municipality
h_count = 0 # Header line count
c_count = 0 # Country count (valtio)
p_count = 0 # State count (lääni)
m_count = 0 # Municipality count (kunta)
v_count = 0 # Village count (kylä)
u_count = 0 # Unknown row count
def get_countries(db):
countries = []
for handle in db.find_place_child_handles(''):
place = db.get_place_from_handle(handle)
if int(place.get_type()) == PlaceType.COUNTRY:
countries.append(place)
return countries
def findNextPidno(pidstrt):
with DbTxn(_("Find next pidno"), db):
db.set_repository_id_prefix(pidstrt + '%04d')
next_pidno = db.find_next_place_gramps_id()
LOG.debug('Next pidno = ' + next_pidno)
db.set_repository_id_prefix('R%04d')
return next_pidno
def findPlace(pid, handle, pname):
place = None
if pid != '':
with DbTxn(_("Read place"), db):
place = db.get_place_from_id(pid)
if place != None:
LOG.info('Place read by id: ' + handle + ' ' +
place.get_name().get_value())
else:
LOG.error('Place NOT found by id: ' + handle + ' ' + pname)
raise GrampsImportError('Place NOT found by id: ',
pid + '/' + pname)
return place
def checkPlaceDuplicate(pname, old_places):
place = None
if len(old_places) > 0:
for old_place in old_places:
LOG.debug('Comparing ' + pname + ' with ' +
old_place.get_name().get_value())
if old_place.get_name().get_value() == pname:
# LOG.debug('Found match ' + pname + ' with ' + place.get_name().get_value() + ' of type ' + place.__class__.__name__ )
return old_place
return None
def addPlace(pname, ptype, refPlace=None, plang='fi'):
place = Place()
placeName = PlaceName()
placeName.set_value(pname)
placeName.set_language(plang)
place.set_name(placeName)
# place.set_change_time(chgtime)
place.set_type(ptype)
place.add_tag(tags[ptype])
if refPlace != None:
placeRef = PlaceRef()
placeRef.set_reference_handle(refPlace.get_handle())
place.add_placeref(placeRef)
# tag.set_color("#EF2929")
with DbTxn(_("Add Place"), db) as trans:
phandle = db.add_place(place, trans)
LOG.debug('Place added: ' + place.get_name().get_value() + ' ' +
phandle)
return place
def checkTagExistence(otext):
with DbTxn(_("Read Tag"), db):
tag = db.get_tag_from_name(otext)
if tag != None:
LOG.debug('Tag found by name, no duplicates: ' + otext + ' ' +
tag.get_name())
else:
tag = Tag()
tag.set_name(otext)
tag.set_color("#EF2929")
with DbTxn(_("Add Tag"), db) as trans:
thandle = db.add_tag(tag, trans)
LOG.debug('Tag added: ' + tag.get_name() + ' ' + thandle)
return tag
chgtime = int(time.time())
LOG.info(" chgtime = " + str(chgtime))
fdir = os.path.dirname(filename)
'''
fh = logging.FileHandler(fdir + '\\placeimport.log')
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
LOG.addHandler(fh)
'''
LOG.info(" fdir = " + fdir)
cout = fdir + "\\result0.csv"
LOG.debug('ini file handling')
'''
config = configman.register_manager("importplaces")
config.register("options.repositoryidrng", "1000")
config.register("options.repositoryincr", "1")
config.register("options.placeidrng", "1000")
config.register("options.placeidincr", "1")
config.register("options.refstring", "r")
config.load()
config.save()
repository_idrange = int(config.get('options.repositoryidrng'))
repository_incr = int(config.get('options.repositoryincr'))
place_idrange = int(config.get('options.placeidrng'))
place_idincr = int(config.get('options.placeidincr'))
refstr = config.get('options.refstring')
place_idno = 0
'''
tags = {}
# Dictionary tagtypes placetype: tag name
tagTypes = {
PlaceType.COUNTRY: "Referenssivaltio",
PlaceType.STATE: "Referenssilääni",
PlaceType.MUNICIPALITY: "Referenssikunta",
PlaceType.VILLAGE: "Referenssikylä"
}
for key, value in tagTypes.items():
tags[key] = checkTagExistence(value).get_handle()
all_countries = get_countries(db)
try:
with open(cout, 'w', newline='\n', encoding="utf-8") as csv_out:
r_writer = csv.writer(csv_out, delimiter=';')
with open(filename, 'r', encoding="utf-8") as t_in:
# rhandle = None
phandle = None
t_dialect = csv.Sniffer().sniff(t_in.read(1024))
t_in.seek(0)
t_reader = csv.reader(t_in, t_dialect)
LOG.info('CSV input file delimiter is ' + t_dialect.delimiter)
for row in t_reader:
rectype = row[
0] # Record type = Gramps place type name (fi_FI)
# LOG.debug('Row type: -' + row[0] + '-')
if rectype == 'type':
LOG.debug('First row: ' + row[0])
h_count += 1
r_writer.writerow([
row[0], row[1], row[2], row[3], row[4], row[5],
row[6], row[7]
])
else:
idno = row[
1] # Possibly previously assigned Gramps object id
handle = row[2].strip(
'"'
) # Possibly previously assigned Gramps object handle
pname = row[3].strip()
# LOG.debug("Rectype " + rectype +' name = ' + pname)
if rectype == 'valtio':
LOG.debug('Country row: ' + pname)
c_count += 1
result = None
if handle != '':
result = findPlace(idno, handle, pname)
else:
result = checkPlaceDuplicate(
pname, all_countries)
if result != None:
LOG.debug('Country row is a duplicate of ' +
result.get_name().get_value() +
' and updates the existing one)')
country = result
# &TODO: some updating
with DbTxn(_("Update Country"), db) as trans:
db.commit_place(country, trans)
else:
country = addPlace(pname, PlaceType.COUNTRY)
states_of_current_country = []
for cname, handle in db.find_backlink_handles(
country.get_handle(), ['Place']):
states_of_current_country.append(
db.get_place_from_handle(handle))
chandle = country.get_handle()
cidno = country.get_gramps_id()
try:
r_writer.writerow([
row[0], row[1], cidno, chandle, row[4],
row[5], row[6], row[7]
])
except IOError:
LOG.error('Error writing country-csv ' +
IOError.strerror)
LOG.debug('Old states ' +
str(len(states_of_current_country)))
elif rectype == 'lääni':
LOG.debug('State row: ' + pname)
result = None
p_count += 1
if handle != '':
result = findPlace(idno, handle, pname)
else:
result = checkPlaceDuplicate(
pname, states_of_current_country)
if result != None:
LOG.debug('State row is a duplicate of ' +
result.get_name().get_value() +
' and updates the existing one)')
state = result
# &TODO: some updating
with DbTxn(_("Update State"), db) as trans:
db.commit_place(state, trans)
else:
state = addPlace(pname, PlaceType.STATE,
country)
states_of_current_country.append(state)
municipalities_of_current_state = []
for cname, handle in db.find_backlink_handles(
state.get_handle(), ['Place']):
municipalities_of_current_state.append(
db.get_place_from_handle(handle))
phandle = state.get_handle()
pidno = state.get_gramps_id()
try:
r_writer.writerow([
row[0], row[1], pidno, phandle, row[4],
row[5], row[6], row[7]
])
except IOError:
LOG.error('Error writing state-csv ' +
IOError.strerror)
LOG.debug(
'Old municipalities ' +
str(len(municipalities_of_current_state)))
elif rectype == 'kunta':
LOG.debug('Municipality row: ' + pname)
m_count += 1
result = None
if handle != '':
result = findPlace(idno, handle, pname)
else:
result = checkPlaceDuplicate(
pname, municipalities_of_current_state)
if result != None:
LOG.debug(
'Municipality row is a duplicate of ' +
result.get_name().get_value() +
' and updates the existing one)')
municipality = result
# &TODO: some updating
with DbTxn(_("Update Municipality"),
db) as trans:
db.commit_place(municipality, trans)
else:
municipality = addPlace(
pname, PlaceType.MUNICIPALITY, state)
municipalities_of_current_state.append(
municipality)
villages_of_current_municipality = []
for cname, handle in db.find_backlink_handles(
municipality.get_handle(), ['Place']):
villages_of_current_municipality.append(
db.get_place_from_handle(handle))
mhandle = municipality.get_handle()
midno = municipality.get_gramps_id()
try:
r_writer.writerow([
row[0], row[1], midno, mhandle, row[4],
row[5], row[6], row[7]
])
except IOError:
LOG.error('Error writing municipality-csv ' +
IOError.strerror)
LOG.debug(
'Old municipalities ' +
str(len(municipalities_of_current_state)))
elif rectype == 'kylä':
LOG.debug('Village row: ' + pname)
v_count += 1
if handle != '':
result = findPlace(idno, handle, pname)
else:
result = checkPlaceDuplicate(
pname, villages_of_current_municipality)
if result != None:
LOG.debug('Village row is a duplicate of ' +
result.get_name().get_value() +
' and updates the existing one)')
village = result
# &TODO: some updating
with DbTxn(_("Update Village"), db) as trans:
db.commit_place(village, trans)
else:
village = addPlace(pname, PlaceType.VILLAGE,
municipality)
villages_of_current_municipality.append(
village)
vhandle = village.get_handle()
vidno = village.get_gramps_id()
try:
r_writer.writerow([
row[0], row[1], vidno, vhandle, row[4],
row[5], row[6], row[7]
])
except IOError:
LOG.error('Error writing village-csv ' +
IOError.strerror)
LOG.debug(
'Old villages ' +
str(len(villages_of_current_municipality)))
else:
u_count += 1
LOG.error('Unknown rectype: ' + rectype)
raise GrampsImportError('Unknown record type ' +
rectype)
except:
exc = sys.exc_info()[0]
LOG.error('*** Something went really wrong! ', exc)
return ImportInfo({_('Results'): _('Something went really wrong ')})
results = {
_('Results'):
_('Input file handled.'),
_(' Countries '):
str(c_count),
_(' States '):
str(p_count),
_(' Municipalities '):
str(m_count),
_(' Villages '):
str(v_count),
_(' Unknown types '):
str(u_count),
_(' Total '):
str(c_count + p_count + m_count + v_count + u_count)
}
LOG.info('Input file handled.')
LOG.info(' Countries ' + str(c_count))
LOG.info(' States ' + str(p_count))
LOG.info(' Municipalities ' + str(m_count))
LOG.info(' Villages ' + str(v_count))
LOG.info(' Unknown types ' + str(u_count))
LOG.info(' Total ' +
str(c_count + p_count + m_count + v_count + u_count))
db.enable_signals()
db.request_rebuild()
return ImportInfo(results)
def importPlaceHierarchies(db, filename, user):
all_countries = [] # Countries in the database
municipalities_of_current_country = [] # Municipalities in the current state
villages_of_current_municipality = [] # Villages in the current municipality
farms_of_current_village = []
buildings_of_current_farm = []
h_count = 0 # Header line count
country_count = 0 # Country count (valtio)
municipality_count = 0 # Municipality count (kunta)
village_count = 0 # Village count (kylä)
farm_count = 0 # farm count (tila)
building_count = 0
unknown_count = 0 # Unknown row count
def get_level0(db):
for handle in db.find_place_child_handles(''):
place = db.get_place_from_handle(handle)
if int(place.get_type()) == PlaceType.COUNTRY:
all_countries.append(place)
return all_countries
def parseNames(pname, plang='fi'):
pnames = pname.split(',')
LOG.debug('Place name %s split into %d pieces' % (pname, len(pnames)))
priName = (pnames[0].strip(), plang)
altNames = []
if len(pnames) > 1:
del pnames[0]
for aname in pnames:
altName = PlaceName()
anameparts = aname.split(':')
if len(anameparts) == 1:
altName.set_value(anameparts[0].strip())
altName.set_language('se')
altNames.append(altName)
elif len(anameparts) == 2:
altName.set_value(anameparts[1].strip())
altName.set_language(anameparts[0].strip())
altNames.append(altName)
else:
LOG.error('Pieleen meni? %s' % aname)
return priName, altNames
def findPlace(pid, pname):
place = None
if pid != '':
with DbTxn(_("Read place"), db):
place = db.get_place_from_id(pid)
if place != None:
LOG.info('Place read by id: ' + pid + ' ' + place.get_name().get_value())
else:
LOG.error('Place NOT found by id: ' + pid + ' ' + pname)
raise GrampsImportError('Place NOT found by id: ', pid + '/' + pname)
return place
def checkPlaceDuplicate(pname, old_places):
if len(old_places) > 0:
for old_place in old_places:
LOG.debug('Comparing ' + pname + ' with ' + old_place.get_name().get_value())
if old_place.get_name().get_value() == pname:
# LOG.debug('Found match ' + pname + ' with ' + place.get_name().get_value() + ' of type ' + place.__class__.__name__ )
return old_place
return None
def addPlace(priName, altNames, ptype, refPlace=None):
place = Place()
placeName = PlaceName()
placeName.set_value(priName[0])
placeName.set_language(priName[1])
place.set_name(placeName)
if len(altNames) > 0:
place.set_alternative_names(altNames)
# place.set_change_time(chgtime)
place.set_type(ptype)
# place.add_tag(tags[ptype])
place.add_tag(reftag)
if refPlace != None:
placeRef = PlaceRef()
placeRef.set_reference_handle(refPlace.get_handle())
place.add_placeref(placeRef)
# tag.set_color("#EF2929")
with DbTxn(_("Add Place"), db) as trans:
phandle = db.add_place(place, trans)
LOG.debug('Place added: ' + place.get_name().get_value() + ' ' + phandle)
return place
def checkTagExistence(otext):
with DbTxn(_("Read Tag"), db):
tag = db.get_tag_from_name(otext)
if tag != None:
LOG.debug('Tag found by name, no duplicates: ' + otext + ' ' + tag.get_name())
else:
tag = Tag()
tag.set_name(otext)
tag.set_color("#EF2929")
with DbTxn(_("Add Tag"), db) as trans:
thandle = db.add_tag(tag, trans)
LOG.debug('Tag added: ' + tag.get_name() + ' ' + thandle)
return tag
def findNextPidno(pidstrt):
with DbTxn(_("Find next pidno"), db):
prefix_save = db.get_place_prefix()
db.set_place_id_prefix(pidstrt + '%05d')
next_pidno = db.find_next_place_gramps_id()
LOG.debug('Next pidno = ' + next_pidno)
db.set_place_id_prefix(prefix_save)
return next_pidno
chgtime = int(time.time())
LOG.info(" chgtime = " + str(chgtime))
fdir = os.path.dirname(filename)
fh = logging.FileHandler(fdir + '\\placeimport.log')
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
LOG.addHandler(fh)
LOG.info(" fdir = " + fdir)
cout = fdir + "\\presult.csv"
LOG.debug('ini file handling')
config = configman.register_manager("importplaces")
config.register("options.refstring", "r")
config.load()
config.save()
# refstr = config.get('options.refstring')
reftag = checkTagExistence('Referenssi').get_handle()
'''
tags = {}
# Dictionary tagtypes placetype: tag name
tagTypes = {PlaceType.COUNTRY: "Referenssivaltio",
PlaceType.STATE: "Referenssilääni",
PlaceType.MUNICIPALITY: "Referenssikunta",
PlaceType.VILLAGE: "Referenssikylä",
PlaceType.FARM: "Referenssitila",
PlaceType.BUILDING: "Referenssitalo"}
for key, value in tagTypes.items():
tags[key] = checkTagExistence(value).get_handle()
'''
all_countries = get_level0(db)
recno = 0
try:
with open(cout, 'w', newline = '\n', encoding="utf-8-sig") as csv_out:
r_writer = csv.writer(csv_out, delimiter=';')
with open(filename, 'r', encoding="utf-8-sig") as t_in:
# rhandle = None
phandle = None
t_dialect = csv.Sniffer().sniff(t_in.read(1024))
t_in.seek(0)
t_reader = csv.reader(t_in, t_dialect)
LOG.info('CSV input file delimiter is ' + t_dialect.delimiter)
for row in t_reader:
recno += 1
rectype = row[5] # Record type = Gramps place type name (fi_FI)
# LOG.debug('Row type: -' + row[0] + '-')
if recno == 1:
LOG.debug('First row: ' + row[0])
h_count += 1
r_writer.writerow([row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7]])
else:
idno = row[4] # Possibly previously assigned Gramps object id
handle = row[7].strip('"') # Possibly previously assigned Gramps object handle
if rectype in ('Valtio', 'Country'):
priName, altNames = parseNames(row[0])
country_count += 1
result = None
if idno != '':
result = findPlace(idno, priName[0])
else:
result = checkPlaceDuplicate(priName[0], all_countries)
if result != None:
LOG.debug('Country row is a duplicate of ' + result.get_name().get_value() + ' and updates the existing one)')
country = result
country.set_alternative_names(altNames)
with DbTxn(_("Update Country"), db) as trans:
db.commit_place(country, trans)
else:
country = addPlace(priName, altNames, PlaceType.COUNTRY)
municipalities_of_current_country = []
for cname, handle in db.find_backlink_handles(country.get_handle(), ['Place']):
municipalities_of_current_country.append(db.get_place_from_handle(handle))
chandle = country.get_handle()
cidno = country.get_gramps_id()
try:
r_writer.writerow([row[0], row[1], row[2], row[3], cidno, row[5], row[6], chandle])
except IOError:
LOG.error('Error writing country-csv ' + IOError.strerror)
LOG.debug('Old municipalities ' + str(len(municipalities_of_current_country)))
elif rectype in ('Kunta', 'Municipality', 'Kaupunki', 'Town'):
priName, altNames = parseNames(row[1])
LOG.debug('Municipality/Town row: ' + priName[0])
municipality_count += 1
result = None
if idno != '':
result = findPlace(idno, priName[0])
else:
result = checkPlaceDuplicate(priName[0], municipalities_of_current_country)
if result != None:
LOG.debug('Municipality row is a duplicate of ' + result.get_name().get_value() + ' and updates the existing one)')
municipality = result
municipality.set_alternative_names(altNames)
with DbTxn(_("Update Municipality"), db) as trans:
db.commit_place(municipality, trans)
else:
if rectype in ('Kunta', 'Municipality'):
municipality = addPlace(priName, altNames, PlaceType.MUNICIPALITY, country)
else:
municipality = addPlace(priName, altNames, PlaceType.TOWN, country)
municipalities_of_current_country.append(municipality)
villages_of_current_municipality = []
for vname, handle in db.find_backlink_handles(municipality.get_handle(), ['Place']):
villages_of_current_municipality.append(db.get_place_from_handle(handle))
mhandle = municipality.get_handle()
midno = municipality.get_gramps_id()
try:
r_writer.writerow([row[0], row[1], row[2], row[3], midno, row[5], row[6], mhandle])
except IOError:
LOG.error('Error writing municipality-csv ' + IOError.strerror)
LOG.debug('Old municipalities ' + str(len(municipalities_of_current_country)))
elif rectype in ('Kylä', 'Village'):
priName, altNames = parseNames(row[2])
LOG.debug('Village row: ' + priName[0])
village_count += 1
if idno != '':
result = findPlace(idno, priName[0])
else:
result = checkPlaceDuplicate(priName[0], villages_of_current_municipality)
if result != None:
LOG.debug('Village row is a duplicate of ' + result.get_name().get_value() + ' and updates the existing one)')
village = result
village.set_alternative_names(altNames)
with DbTxn(_("Update Village"), db) as trans:
db.commit_place(village, trans)
else:
village = addPlace(priName, altNames, PlaceType.VILLAGE, municipality)
villages_of_current_municipality.append(village)
farms_of_current_village = []
for fname, handle in db.find_backlink_handles(village.get_handle(), ['Place']):
farms_of_current_village.append(db.get_place_from_handle(handle))
vhandle = village.get_handle()
vidno = village.get_gramps_id()
try:
r_writer.writerow([row[0], row[1], row[2], row[3], vidno, row[5], row[6], vhandle])
except IOError:
LOG.error('Error writing village-csv ' + IOError.strerror)
LOG.debug('Old villages ' + str(len(villages_of_current_municipality)))
elif rectype in ('Tila', 'Farm'):
priName, altNames = parseNames(row[3])
LOG.debug('Farme row: ' + priName[0])
result = None
farm_count += 1
if handle != '':
result = findPlace(idno, priName[0])
else:
result = checkPlaceDuplicate(priName[0], farms_of_current_village)
if result != None:
LOG.debug('Farm row is a duplicate of ' + result.get_name().get_value() + ' and updates the existing one)')
farm = result
farm.set_alternative_names(altNames)
with DbTxn(_("Update Farm"), db) as trans:
db.commit_place(farm, trans)
else:
farm = addPlace(priName, altNames, PlaceType.FARM, village)
farms_of_current_village.append(farm)
buildings_of_current_farm = []
for bname, handle in db.find_backlink_handles(farm.get_handle(), ['Place']):
buildings_of_current_farm.append(db.get_place_from_handle(handle))
fhandle = farm.get_handle()
fidno = farm.get_gramps_id()
try:
r_writer.writerow([row[0], row[1], row[2], row[3], fidno, row[5], row[6], fhandle])
except IOError:
LOG.error('Error writing farm-csv ' + IOError.strerror)
LOG.debug('Old farmss ' + str(len(farms_of_current_village)))
elif rectype in ('Rakennus', 'Building'):
priName, altNames = parseNames(row[4])
LOG.debug('Building row: ' + priName[0])
result = None
building_count += 1
if handle != '':
result = findPlace(idno, priName[0])
else:
result = checkPlaceDuplicate(priName[0], buildings_of_current_farm)
if result != None:
LOG.debug('Building row is a duplicate of ' + result.get_name().get_value() + ' and updates the existing one)')
building = result
# &TODO: some updating
with DbTxn(_("Update Farm"), db) as trans:
db.commit_place(building, trans)
else:
building = addPlace(priName, altNames, PlaceType.BUILDING, farm)
buildings_of_current_farm.append(building)
bhandle = building.get_handle()
bidno = building.get_gramps_id()
try:
r_writer.writerow([row[0], row[1], row[2], row[3], bidno, row[5], row[6], bhandle])
except IOError:
LOG.error('Error writing building-csv ' + IOError.strerror)
LOG.debug('Old buildings ' + str(len(buildings_of_current_farm)))
else:
unknown_count += 1
LOG.error('Unknown rectype on line ' + str(recno) + ': ' + rectype)
raise GrampsImportError('Unknown record type ' + rectype)
except:
exc = sys.exc_info()[0]
LOG.error('*** Something went really wrong! ', exc )
return ImportInfo({_('Results'): _('Something went really wrong ')})
results = { _('Results'): _('Input file handled.')
, _(' Countries '): str(country_count)
, _(' Municipalities '): str(municipality_count)
, _(' Villages '): str(village_count)
, _(' Farms '): str(farm_count)
, _(' Buildings '): str(building_count)
, _(' Unknown types '): str(unknown_count)
, _(' Total '): str(country_count + municipality_count + village_count + farm_count + building_count + unknown_count) }
LOG.info('Input file handled, ' + str(recno) + ' rows')
LOG.info(' Countries ' + str(country_count))
LOG.info(' Municipalities ' + str(municipality_count))
LOG.info(' Villages ' + str(village_count))
LOG.info(' Farms ' + str(farm_count))
LOG.info(' Buildings ' + str(building_count))
LOG.info(' Unknown types ' + str(unknown_count))
LOG.info(' Total ' + str(country_count + municipality_count + village_count + farm_count + building_count + unknown_count))
db.enable_signals()
db.request_rebuild()
return ImportInfo(results)
def importSourceHierarchies(db, filename, user):
def findNextRidno(ridstrt):
with DbTxn(_("Find next ridno"), db):
prefix_save = db.get_repository_prefix()
db.set_repository_id_prefix(ridstrt + '%04d')
next_ridno = db.find_next_repository_gramps_id()
LOG.debug('Next ridno = ' + next_ridno)
db.set_repository_id_prefix(prefix_save)
return next_ridno
def findNextSidno(ridno):
with DbTxn(_("Find next sidno"), db):
prefix_save = db.get_source_prefix()
db.set_source_id_prefix(ridno + '%04d')
next_sidno = db.find_next_source_gramps_id()
LOG.debug('Next sidno = ' + next_sidno)
db.set_source_id_prefix(prefix_save)
return next_sidno
def addRepository(repositoryName, reftag):
ridno = db.find_next_repository_gramps_id()
repository = Repository()
repositoryType = RepositoryType()
repositoryType.set(RepositoryType.ARCHIVE)
repository.set_type(repositoryType)
repository.set_gramps_id(ridno)
repository.set_name(repositoryName)
repository.set_url_list(())
repository.set_change_time(chgtime)
if reftag != None:
repository.add_tag(reftag.get_handle())
with DbTxn(_("Add Repository"), db) as trans:
rhandle = db.add_repository(repository, trans)
return repository
def addSource(sourceName, attribs, reftag, repository):
snote = addNote(attribs[3], NoteType.SOURCE)
sidno = db.find_next_source_gramps_id()
source = Source()
source.set_gramps_id(sidno)
source.set_title(sourceName)
source.set_author(attribs[0])
source.set_publication_info(attribs[1])
source.set_abbreviation(attribs[2])
source.add_note(snote.get_handle())
if reftag != None:
source.add_tag(reftag.get_handle())
repoRef = RepoRef()
repoRef.set_reference_handle(repository.get_handle())
source.add_repo_reference(repoRef)
source.set_change_time(chgtime)
with DbTxn(_("Add Source"), db) as trans:
shandle = db.add_source(source, trans)
return source
def addNote(ntext, ntype):
nidno = db.find_next_note_gramps_id()
note = Note(ntext)
note.set_gramps_id(nidno)
note.set_type(ntype)
if reftag != None:
note.add_tag(reftag.get_handle())
note.set_change_time(chgtime)
with DbTxn(_("Add Note"), db) as trans:
nhandle = db.add_note(note, trans)
LOG.debug('Note added: ' + ntext + ' ' + nhandle)
return note
def checkTagExistence(otext):
with DbTxn(_("Read Tag"), db):
tag = db.get_tag_from_name(otext)
if tag != None:
LOG.debug('Tag found by name, no duplicates: ' + otext + ' ' +
tag.get_name())
else:
tag = Tag()
tag.set_name(otext)
tag.set_color("#EF2929")
with DbTxn(_("Add Tag"), db) as trans:
thandle = db.add_tag(tag, trans)
LOG.debug('Tag added: ' + tag.get_name() + ' ' + thandle)
return tag
fdir = os.path.dirname(filename)
fh = logging.FileHandler(fdir + '\\sourceimport.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setLevel(logging.INFO)
fh.setFormatter(formatter)
LOG.addHandler(fh)
LOG.info(" fdir = " + fdir)
LOG.debug('ini file handling')
config = configman.register_manager("importsources")
'''
config.register("options.repositoryidrng", "1000")
config.register("options.repositoryincr", "1")
config.register("options.sourceidrng", "1000")
config.register("options.sourceidincr", "1")
'''
config.register("options.refstring", "r")
config.load()
config.save()
r_count = 0
s_count = 0
reftag = checkTagExistence('Referenssi')
chgtime = int(time.time())
LOG.info(" chgtime = " + str(chgtime))
try:
currRepoName = ''
with open(filename, 'r', encoding="utf-8-sig") as t_in:
t_dialect = csv.Sniffer().sniff(t_in.read(1024))
t_dialect.delimiter = ";"
t_in.seek(0)
t_reader = csv.reader(t_in, t_dialect)
LOG.info('CSV input file delimiter is ' + t_dialect.delimiter)
global repository
for row in t_reader:
repoName = row[0].strip() # Repository name
if repoName != currRepoName:
currRepoName = repoName
LOG.debug('New repository: ' + currRepoName)
repository = addRepository(currRepoName, reftag)
r_count += 1
sourceName = repoName + " " + row[1].strip(
) + " " + row[2].strip()
attribs = (repoName, "", row[3], row[4])
LOG.debug('New source: ' + sourceName)
addSource(sourceName, attribs, reftag, repository)
s_count += 1
# LOG.debug('Unknown rectype: ' + rectype)
# raise GrampsImportError('Unknown record type ' + rectype)
except:
exc = sys.exc_info()[0]
LOG.error('*** Something went really wrong! ', exc)
return ImportInfo({_('Results'): _('Something went really wrong ')})
results = {
_('Results'): _('Input file handled.'),
_(' Repositories '): str(r_count),
_(' Sources '): str(s_count)
}
LOG.info('Input file handled.')
LOG.info(' Repositories ' + str(r_count))
LOG.info(' Sources ' + str(s_count))
db.enable_signals()
db.request_rebuild()
return ImportInfo(results)