def background_load_to_neo4j(username,filename):
''' Imports gramps xml data to database '''
upload_folder = get_upload_folder(username)
pathname = os.path.join(upload_folder,filename)
metaname = pathname+".meta"
logname = pathname+".log"
update_metafile(metaname, progress={})
steps = []
try:
os.makedirs(upload_folder, exist_ok=True)
set_meta(username,filename,status=STATUS_LOADING)
this_thread = threading.current_thread()
this_thread.progress = {}
counts = gramps_loader.analyze_xml(username, filename)
update_metafile(metaname,counts=counts,progress={})
threading.Thread(target=lambda: i_am_alive(metaname,this_thread),name="i_am_alive for " + filename).start()
steps,batch_id = gramps_loader.xml_to_neo4j(pathname,username)
for step in steps:
print(step)
if not batch_id:
raise RuntimeError("Run Failed, missing batch_id")
if os.path.exists(metaname):
set_meta(username,filename,
batch_id=batch_id,
status=STATUS_DONE)
msg = "{}:\nStored the file {} from user {} to neo4j".format(util.format_timestamp(),pathname,username)
msg += "\nBatch id: {}".format(batch_id)
msg += "\nLog file: {}".format(logname)
msg += "\n"
for step in steps:
msg += "\n{}".format(step)
msg += "\n"
open(logname,"w", encoding='utf-8').write(msg)
email.email_admin(
"Stk: Gramps XML file stored",
msg )
syslog.log(type="storing to database complete",file=filename,user=username)
except:
traceback.print_exc()
res = traceback.format_exc()
set_meta(username,filename,status=STATUS_FAILED)
msg = "{}:\nStoring the file {} from user {} to database FAILED".format(util.format_timestamp(),pathname,username)
msg += "\nLog file: {}".format(logname)
msg += "\n" + res
for step in steps:
msg += "\n{}".format(step)
msg += "\n"
open(logname,"w", encoding='utf-8').write(msg)
email.email_admin(
"Stk: Gramps XML file storing FAILED",
msg )
syslog.log(type="gramps store to database failed",file=filename,user=username)
def process_gedcom(args, transform_module, task_name=''):
LOG.info("------ Ajo '%s' alkoi %s ------", task_name,
util.format_timestamp())
transform_module.initialize(args)
if hasattr(transform_module, "process"):
with Output(args) as f:
transform_module.process(args, f)
else:
try:
# 1st traverse
if hasattr(transform_module, "phase1"):
for gedline in read_gedcom(args):
transform_module.phase1(args, gedline)
# Intermediate processing of collected data
if hasattr(transform_module, "phase2"):
transform_module.phase2(args)
do_phase4 = hasattr(transform_module, "phase4")
if hasattr(transform_module, "phase3"):
# 2nd traverse "phase3"
with Output(args) as f:
f.transform_name = transform_module.__name__
f.display_changes = args.display_changes
for gedline in read_gedcom(args):
if do_phase4 and gedline.tag == "TRLR":
f.original_line = ""
transform_module.phase4(args, f)
f.original_line = gedline.line.strip()
f.saved_line = ""
transform_module.phase3(args, gedline, f)
except FileNotFoundError as err:
LOG.error("Ohjelma päättyi virheeseen {}: {}".format(
type(err).__name__, str(err)))
LOG.info("------ Ajo '%s' päättyi %s ------", task_name,
util.format_timestamp())
def gedcom_save(gedcom):
filename1 = gedcom_utils.gedcom_fullname(gedcom)
filename2 = filename1 + "-temp"
newname = util.generate_name(filename1)
os.rename(filename1,newname)
os.rename(filename2,filename1)
gedcom_utils.history_append(filename1,"\n{}:".format(util.format_timestamp()))
gedcom_utils.history_append(filename1,_("File {} saved as {}").format(filename1,newname))
gedcom_utils.history_append(filename1,_("File {} saved as {}").format(filename2,filename1))
logger.info(f'-> bp.gedcom.routes.gedcom_save f="{os.path.basename(newname)}"')
rsp = dict(newname=os.path.basename(newname))
return jsonify(rsp)
def from_node(cls, node):
''' Convert a Neo4j node to Batch object.
'''
obj = cls()
obj.uniq_id = node.id
obj.user = node.get('user', "")
obj.file = node.get('file', None)
obj.id = node.get('id', None)
obj.status = node.get('status', "")
obj.timestamp = node.get('timestamp', 0)
obj.upload = format_timestamp(obj.timestamp)
obj.auditor = node.get('auditor', None)
return obj
def gedcom_revert(gedcom,version):
filename1 = gedcom_utils.gedcom_fullname(gedcom)
filename2 = gedcom_utils.gedcom_fullname(version)
newname = util.generate_name(filename1)
if os.path.exists(filename1) and os.path.exists(filename2):
os.rename(filename1,newname)
os.rename(filename2,filename1)
gedcom_utils.history_append(filename1,"\n{}:".format(util.format_timestamp()))
gedcom_utils.history_append(filename1,_("File {} saved as {}").format(filename1,newname))
gedcom_utils.history_append(filename1,_("File {} saved as {}").format(filename2,filename1))
rsp = dict(newname=os.path.basename(newname))
else:
rsp = dict(status="Error")
return jsonify(rsp)
def log(type,**kwargs):
""" Create a Syslog event node with given arguments and timestamp.
"""
try:
user=current_user.username
except:
user = kwargs.get("user","")
timestamp = time.time()
timestr=util.format_timestamp(timestamp)
msg = json.dumps(kwargs)
try:
shareds.driver.session().run(syslog_cypher_write, type=type, user=user,
msg=msg, time=timestamp, timestr=timestr)
except Exception:
traceback.print_exc()
def emit(self, line):
''' Process an input line '''
#if self.display_changes and self.original_line and \
if self.args.display_changes and self.original_line is not None:
if self.original_line == "" and self.saved_line != "":
print('{:>36} --> {}'.format(self.saved_line, self.saved_line))
print('{:>36} --> {}'.format(self.original_line, line))
self.saved_line = ""
elif line.strip() != self.original_line:
print('{:>36} --> {}'.format(self.original_line, line))
else:
self.saved_line = self.original_line
self.original_line = ""
if line.startswith("1 CHAR"):
# this is probably unnecessary because the result should always be
# the same as in the input file, i.e. the line is not modified
if self.output_encoding == "UTF-8":
line = "1 CHAR UTF-8"
elif self.output_encoding == "ISO8859-1":
line = "1 CHAR ANSI"
else:
line = "1 CHAR {}".format(self.output_encoding)
self.f.write(line + "\n")
if self.log:
#TODO: Should follow a setting from gedder.py
self.log = False
args = sys.argv[1:]
try:
v = " v." + VERSION
except NameError:
v = ""
self.emit("1 NOTE _TRANSFORM{} {}".format(v, self.transform_name))
self.emit("2 CONT args:")
for name, value in vars(self.args).items():
self.emit("1 CONT - {}={}".format(name, value))
self.emit("2 CONT _COMMAND {} {}".\
format(os.path.basename(sys.argv[0]), " ".join(args)))
user = getpass.getuser()
if not user:
user = "******"
datestring = util.format_timestamp()
self.emit("2 CONT _USER {}".format(user))
self.emit("2 CONT _DATE {}".format(datestring))
if self.new_name:
self.emit("2 CONT _SAVEDFILE " + self.new_name)
def from_node(cls, node):
''' Convert a Neo4j node to an Audit object.
<Node id=439060 labels={'Audit'}
properties={'auditor': 'juha', 'id': '2020-01-03.001',
'user': '******', 'timestamp': 1578940247182}>
'''
obj = cls()
obj.uniq_id = node.id
obj.user = node.get('user', "")
obj.auditor = node.get('auditor')
obj.id = node.get('id', None)
#obj.status = node.get('status', "")
obj.timestamp = node.get('timestamp', 0)
if obj.timestamp:
obj.updated = format_timestamp(obj.timestamp/1000.)
return obj
def gedcom_upload():
# code from: http://flask.pocoo.org/docs/1.0/patterns/fileuploads/
def allowed_file(filename):
return '.' in filename and \
filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
gedcom_folder = gedcom_utils.get_gedcom_folder()
# check if the post request has the file part
if 'file' not in request.files:
flash(_('Choose a GEDCOM file to upload'), category='flash_warning')
return redirect(url_for('.gedcom_list'))
file = request.files['file']
# if user does not select file, browser also
# submit an empty part without filename
if file.filename == '': # pragma: no cover
flash(_('Choose a GEDCOM file to upload'), category='flash_warning')
return redirect(url_for('.gedcom_list'))
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
os.makedirs(gedcom_folder, exist_ok=True)
fullname = os.path.join(gedcom_folder, filename)
if os.path.exists(fullname):
flash(_('This GEDCOM file already exists'), category='flash_error')
return redirect(url_for('.gedcom_list'))
file.save(fullname)
desc = request.form['desc']
encoding = util.guess_encoding(fullname)
metadata = {
'desc':desc,
'encoding':encoding,
'upload_time':util.format_timestamp(),
'size':os.stat(fullname).st_size,
}
gedcom_utils.save_metadata(filename, metadata)
gedcom_utils.history_init(fullname)
syslog.log(type="uploaded a gedcom",gedcom=file.filename)
logger.info(f'-> bp.gedcom.routes.gedcom_upload n={os.stat(fullname).st_size/1024.}kb')
return redirect(url_for('.gedcom_info',gedcom=filename))
def process_gedcom(arglist, transform_module):
""" Implements a mechanism for Gedcom transforms.
Returns a dictionary:
- stdout result texts for the user log page
- stderr errors texts for the user log page
- oldname original name of gedcom file
- logfile log file name
The transform_module is assumed to contain the following methods:
- initialize
- transform: implements the actual transformation for a single line block ("item")
- fixlines: preprocesses the Gedcom contents (list of lines/strings)
- add_args: adds the transform-specific arguments (ArgumentParser style)
See sukujutut.py as an example
"""
msg = _("Transform '{}' started at {}").format(transform_module.name,
util.format_timestamp())
LOG.info(f"------ {msg} ------")
parser = ArgumentParser()
# parser.add_argument('transform', help="Name of the transform (Python module)")
parser.add_argument('input_gedcom',
help=_("Name of the input GEDCOM file"))
parser.add_argument('--logfile',
help=_("Name of the log file"),
default="_LOGFILE")
# parser.add_argument('--output_gedcom', help="Name of the output GEDCOM file; this file will be created/overwritten" )
parser.add_argument('--display-changes',
action='store_true',
help=_('Display changed rows'))
parser.add_argument('--dryrun',
action='store_true',
help=_('Do not produce an output file'))
parser.add_argument('--nolog',
action='store_true',
help=_('Do not produce a log in the output file'))
parser.add_argument('--encoding',
type=str,
default="UTF-8",
choices=["UTF-8", "UTF-8-SIG", "ISO8859-1"],
help=_("Encoding of the input GEDCOM"))
transform_module.add_args(parser)
args = parser.parse_args(arglist)
args.output_gedcom = None
args.nolog = True # replaced by history file
gedcom_utils.history_append(args.input_gedcom, "\n" + msg)
gedcom_utils.history_append_args(args)
# You may deny stdout redirect by setting GEDCOM_REDIRECT_SYSOUT=False in config.py
if not 'GEDCOM_REDIRECT_SYSOUT' in globals():
GEDCOM_REDIRECT_SYSOUT = True
try:
gedcom_utils.init_log(args.logfile)
with Output(args) as out:
out.original_line = None
out.transform_name = transform_module.__name__
if GEDCOM_REDIRECT_SYSOUT:
saved_stdout = sys.stdout
saved_stderr = sys.stderr
sys.stdout = io.StringIO()
sys.stderr = io.StringIO()
if args.dryrun:
old_name = ""
else:
old_name = out.new_name
print(f"<h3>------ {msg} ------</h3>")
t = transformer.Transformer(
transform_module=transform_module,
display_callback=gedcom_utils.display_changes,
options=args)
""" Create a Gedcom transformer g from Transformer t and execute
the transformations.
The resulting Items are in the list g.items
"""
g = t.transform_file(args.input_gedcom)
g.print_items(out)
print("<div>")
print("<b>------ " + ngettext(
"Total {num} change", "Total {num} changes",
num=t.num_changes).format(num=t.num_changes) + "</b>")
#print(_("------ Number of changes: {}").format(t.num_changes))
except:
traceback.print_exc()
finally:
if old_name:
gedcom_utils.history_append(
args.input_gedcom,
_('File saved as {}').format(args.input_gedcom))
gedcom_utils.history_append(
args.input_gedcom,
_("Old file saved as {}").format(old_name))
else:
gedcom_utils.history_append(
args.input_gedcom,
_("File saved as {}").format(args.input_gedcom + "temp"))
msg = _("Transform '{}' ended at {}").format(
transform_module.name, util.format_timestamp())
gedcom_utils.history_append(args.input_gedcom, msg)
print(f"<h3>------ {msg} ------</h3>")
print("</div>")
output = None
errors = None
if GEDCOM_REDIRECT_SYSOUT:
output = sys.stdout.getvalue()
errors = sys.stderr.getvalue()
sys.stdout = saved_stdout
sys.stderr = saved_stderr
if old_name:
old_basename = os.path.basename(old_name)
else:
old_basename = ""
if errors and old_basename:
os.rename(old_name, args.input_gedcom)
old_basename = ""
rsp = dict(stdout=output,
stderr=errors,
oldname=old_basename,
logfile=args.logfile)
if hasattr(transform_module, "output_format") \
and transform_module.output_format == "plain_text":
rsp["plain_text"] = True
return rsp
def get_nameobject(self):
''' Returns the latest object of type PersonName_v1 '''
if self.current_index >= 0:
return self.rows[self.current_index]
if __name__ == '__main__':
''' Test set '''
from .ged_output import Output
logging.basicConfig(filename='example.log',
level=logging.DEBUG,
format='%(levelname)s:%(message)s')
LOG.info("------ Ajo '%s' alkoi %s", "Testi",
util.format_timestamp() + " ------")
# One person with two NAME lines
my_record_1 = GedcomRecord(GedcomLine('0 @I1@ INDI'))
my_name = PersonName_v1(
GedcomLine(
'1 NAME Amalia Saima* (Sanni) Erikint./Raitala os. von Krats/Ericsdr.'
))
my_record_1.add_member(my_name)
my_name.add_line(GedcomLine(('2', 'GIVN', 'Saimi')))
my_name.add_line(GedcomLine('3 SOUR Äidiltä'))
my_name.add_line(GedcomLine('2 SURN Raitala'))
my_name.add_line(GedcomLine('2 NOTE Kummin kaima'))
my_record_2 = GedcomRecord(GedcomLine('0 @I2@ INDI'))
my_name = PersonName_v1(GedcomLine('1 NAME vauva//Ericsdr.'))
my_record_2.add_member(my_name)
def history_init(gedcom_fname):
''' Initialize history file. '''
history_file_name = gedcom_fname + "-history"
open(history_file_name,"w").write("{}: Uploaded {}\n".format(util.format_timestamp(),gedcom_fname))
def list_uploads(username):
''' Gets a list of uploaded files and their process status.
Also db Batches without upload file are included in the list.
'''
# 1. List Batches, their status and Person count
batches = {}
result = shareds.driver.session().run(Cypher_batch.get_user_batch_names,
user=username)
for record in result:
# <Record batch='2019-08-12.001' timestamp=None persons=1949>
batch = record['batch']
batches[batch] = (record['status'], record['persons'])
# 2. List uploaded files
upload_folder = get_upload_folder(username)
try:
names = sorted([name for name in os.listdir(upload_folder)])
except:
names = []
uploads = []
class Upload: pass
for name in names:
if name.endswith(".meta"):
fname = os.path.join(upload_folder,name)
xmlname = name.rsplit(".",maxsplit=1)[0]
meta = get_meta(fname)
status = meta["status"]
batch_id = ""
status_text = None
if status == STATUS_UPLOADED:
status_text = _("UPLOADED")
elif status == STATUS_LOADING:
status_text = _("STORING")
elif status == STATUS_DONE:
status_text = _("STORED")
if 'batch_id' in meta:
batch_id = meta['batch_id']
elif status == STATUS_FAILED:
status_text = _("FAILED")
elif status == STATUS_ERROR:
status_text = _("ERROR")
elif status == STATUS_REMOVED:
status_text = _("REMOVED")
if not batch_id in batches:
if status_text == _("STORED"):
status_text = _("REMOVED")
batch_id = ""
person_count = 0
else:
status, person_count = batches.pop(batch_id)
if status_text:
upload = Upload()
upload.xmlname = xmlname
upload.status = status_text
upload.batch_id = batch_id
upload.count = person_count
upload.done = (status_text == _("STORED"))
upload.uploaded = (status_text == _("UPLOADED"))
upload.loading = (status_text == _("STORING"))
upload.upload_time = meta["upload_time"]
upload.upload_time_s = util.format_timestamp(upload.upload_time)
upload.user = username
uploads.append(upload)
# 3. Add batches where there is no file
for batch, item in batches.items():
upload = Upload()
upload.batch_id = batch
status, count = item
if status == "started":
upload.status = "?"
elif status == "completed":
upload.status = _("STORED")
upload.count = count
upload.upload_time = 0.0
uploads.append(upload)
return sorted(uploads,key=lambda x: x.upload_time)