def upload_gramps():
""" Load a gramps xml file to temp directory for processing in the server
"""
try:
infile = request.files['filenm']
material = request.form['material']
#logger.debug("Got a {} file '{}'".format(material, infile.filename))
t0 = time.time()
upload_folder = uploads.get_upload_folder(current_user.username)
os.makedirs(upload_folder, exist_ok=True)
pathname = loadfile.upload_file(infile,upload_folder)
shareds.tdiff = time.time()-t0
logname = pathname + ".log"
uploads.set_meta(current_user.username,infile.filename,
status=uploads.STATUS_UPLOADED,
upload_time=time.time())
msg = f"{util.format_timestamp()}: User {current_user.name} ({current_user.username}) uploaded the file {pathname}"
open(logname,"w", encoding='utf-8').write(msg)
email.email_admin(
"Stk: Gramps XML file uploaded",
msg )
syslog.log(type="gramps file uploaded",file=infile.filename)
logger.info(f'-> bp.gramps.routes.upload_gramps/{material} f="{infile.filename}"'
f' e={shareds.tdiff:.3f}sek')
except Exception as e:
return redirect(url_for('gramps.error_page', code=1, text=str(e)))
return redirect(url_for('gramps.list_uploads'))
def email_from_admin(subject, body, receiver):
sender = shareds.app.config.get('ADMIN_EMAIL_FROM')
if sender:
syslog.log(type="sent email from admin",
sender=sender,
receiver=receiver,
subject=subject)
return email(sender, receiver, subject, body)
def move_in_2():
""" Move the accepted Batch to Isotammi database """
owner = request.form['user']
batch_id = request.form['batch']
auditor = current_user.username
logger.info(f' bp.audit.routes.move_in_2 u={owner} b={batch_id}')
merger = Batch_merge()
msg = merger.move_whole_batch(batch_id, owner, auditor)
syslog.log(type="batch to Common data", batch=batch_id, by=owner, msg=msg)
return redirect(url_for('audit.move_in_1', batch_name=batch_id))
def gedcom_delete_old_versions(gedcom):
gedcom_folder = gedcom_utils.get_gedcom_folder()
gedcom_folder = os.path.abspath(gedcom_folder)
gedcom = secure_filename(gedcom)
for name in os.listdir(gedcom_folder):
filename = os.path.join(gedcom_folder, name)
if name.startswith(gedcom+"."):
gedcom_utils.removefile(filename)
logging.info("Deleted:"+filename)
syslog.log(type="deleted old versions for gedcom",gedcom=gedcom)
return redirect(url_for('.gedcom_info',gedcom=gedcom))
def background_load_to_neo4j(username,filename):
''' Imports gramps xml data to database '''
upload_folder = get_upload_folder(username)
pathname = os.path.join(upload_folder,filename)
metaname = pathname+".meta"
logname = pathname+".log"
update_metafile(metaname, progress={})
steps = []
try:
os.makedirs(upload_folder, exist_ok=True)
set_meta(username,filename,status=STATUS_LOADING)
this_thread = threading.current_thread()
this_thread.progress = {}
counts = gramps_loader.analyze_xml(username, filename)
update_metafile(metaname,counts=counts,progress={})
threading.Thread(target=lambda: i_am_alive(metaname,this_thread),name="i_am_alive for " + filename).start()
steps,batch_id = gramps_loader.xml_to_neo4j(pathname,username)
for step in steps:
print(step)
if not batch_id:
raise RuntimeError("Run Failed, missing batch_id")
if os.path.exists(metaname):
set_meta(username,filename,
batch_id=batch_id,
status=STATUS_DONE)
msg = "{}:\nStored the file {} from user {} to neo4j".format(util.format_timestamp(),pathname,username)
msg += "\nBatch id: {}".format(batch_id)
msg += "\nLog file: {}".format(logname)
msg += "\n"
for step in steps:
msg += "\n{}".format(step)
msg += "\n"
open(logname,"w", encoding='utf-8').write(msg)
email.email_admin(
"Stk: Gramps XML file stored",
msg )
syslog.log(type="storing to database complete",file=filename,user=username)
except:
traceback.print_exc()
res = traceback.format_exc()
set_meta(username,filename,status=STATUS_FAILED)
msg = "{}:\nStoring the file {} from user {} to database FAILED".format(util.format_timestamp(),pathname,username)
msg += "\nLog file: {}".format(logname)
msg += "\n" + res
for step in steps:
msg += "\n{}".format(step)
msg += "\n"
open(logname,"w", encoding='utf-8').write(msg)
email.email_admin(
"Stk: Gramps XML file storing FAILED",
msg )
syslog.log(type="gramps store to database failed",file=filename,user=username)
def gedcom_delete(gedcom):
gedcom_folder = gedcom_utils.get_gedcom_folder()
gedcom_folder = os.path.abspath(gedcom_folder)
gedcom = secure_filename(gedcom)
for name in os.listdir(gedcom_folder):
if name.endswith("-history"): continue # do not remove history
if (name == gedcom or
name.startswith(gedcom+".") or
name.startswith(gedcom+"-")
):
filename = os.path.join(gedcom_folder, name)
gedcom_utils.removefile(filename)
logging.info("Deleted:"+filename)
syslog.log(type="deleted a gedcom",gedcom=gedcom)
logger.info(f'-> bp.gedcom.routes.gedcom_delete f="{gedcom}"')
return redirect(url_for('.gedcom_list'))
def email_admin(subject, body, sender=None):
if sender is None:
sender = shareds.app.config.get('ADMIN_EMAIL_FROM')
mail_to = shareds.app.config.get('ADMIN_EMAIL_TO')
if sender and mail_to:
if email(sender, mail_to, subject, body):
syslog.log(type="sent email to admin",
sender=sender,
receiver=mail_to,
subject=subject)
return True
else:
syslog.log(type="FAILED: email to admin",
sender=sender,
receiver=mail_to,
subject=subject)
return False
return False
def initiate_background_load_to_neo4j(userid,filename):
''' Starts gramps xml data import to database '''
#===========================================================================
# subprocess.Popen("PYTHONPATH=app python runload.py "
# + pathname + " "
# + username + " "
# + logname,
# shell=True)
#===========================================================================
def background_load_to_neo4j_thread(app):
with app.app_context():
background_load_to_neo4j(userid,filename)
threading.Thread(target=background_load_to_neo4j_thread,
args=(shareds.app,),
name="neo4j load for " + filename).start()
syslog.log(type="storing to database initiated",file=filename,user=userid)
return False
def batch_delete(batch_id):
from models.gen.batch_audit import Batch
filename = Batch.get_filename(current_user.username,batch_id)
metafile = filename.replace("_clean.",".") + ".meta"
if os.path.exists(metafile):
data = eval(open(metafile).read())
if data.get('batch_id') == batch_id:
del data['batch_id']
data['status'] = uploads.STATUS_REMOVED
open(metafile,"w").write(repr(data))
Batch.delete_batch(current_user.username,batch_id)
logger.info(f'-> bp.gramps.routes.batch_delete f="{batch_id}"')
syslog.log(type="batch_id deleted",batch_id=batch_id)
flash(_("Batch id %(batch_id)s has been deleted", batch_id=batch_id), 'info')
referrer = request.headers.get("Referer")
return redirect(referrer)
def gedcom_upload():
# code from: http://flask.pocoo.org/docs/1.0/patterns/fileuploads/
def allowed_file(filename):
return '.' in filename and \
filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
gedcom_folder = gedcom_utils.get_gedcom_folder()
# check if the post request has the file part
if 'file' not in request.files:
flash(_('Choose a GEDCOM file to upload'), category='flash_warning')
return redirect(url_for('.gedcom_list'))
file = request.files['file']
# if user does not select file, browser also
# submit an empty part without filename
if file.filename == '': # pragma: no cover
flash(_('Choose a GEDCOM file to upload'), category='flash_warning')
return redirect(url_for('.gedcom_list'))
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
os.makedirs(gedcom_folder, exist_ok=True)
fullname = os.path.join(gedcom_folder, filename)
if os.path.exists(fullname):
flash(_('This GEDCOM file already exists'), category='flash_error')
return redirect(url_for('.gedcom_list'))
file.save(fullname)
desc = request.form['desc']
encoding = util.guess_encoding(fullname)
metadata = {
'desc':desc,
'encoding':encoding,
'upload_time':util.format_timestamp(),
'size':os.stat(fullname).st_size,
}
gedcom_utils.save_metadata(filename, metadata)
gedcom_utils.history_init(fullname)
syslog.log(type="uploaded a gedcom",gedcom=file.filename)
logger.info(f'-> bp.gedcom.routes.gedcom_upload n={os.stat(fullname).st_size/1024.}kb')
return redirect(url_for('.gedcom_info',gedcom=filename))
def log_user_logged_out(sender, user, **extra):
syslog.log(type="user logged out")
def log_user_logged_in(sender, user, **extra):
syslog.log(type="user logged in")
session['lang'] = current_user.language
session['lang'] = reqlang
else:
reqlang = session.get('lang')
if not reqlang:
if current_user.is_authenticated:
reqlang = current_user.language
session['lang'] = reqlang
return reqlang
from flask_login import user_logged_in, user_logged_out
from models import syslog
def log_user_logged_in(sender, user, **extra):
syslog.log(type="user logged in")
session['lang'] = current_user.language
def log_user_logged_out(sender, user, **extra):
syslog.log(type="user logged out")
syslog.syslog_init()
syslog.log(type="application initialized")
user_logged_in.connect(log_user_logged_in, shareds.app)
user_logged_out.connect(log_user_logged_out, shareds.app)
import setups
def xml_delete(username,xmlfile):
uploads.delete_files(username,xmlfile)
syslog.log(type="gramps file uploaded",file=xmlfile,user=username)
logger.info(f'-> bp.admin.routes.xml_delete f="{xmlfile}"')
return redirect(url_for('admin.list_uploads', username=username))
def xml_delete(xmlfile):
uploads.delete_files(current_user.username,xmlfile)
logger.info(f'-> bp.gramps.routes.xml_delete f="{os.path.basename(xmlfile)}"')
syslog.log(type="gramps file deleted",file=xmlfile)
return redirect(url_for('gramps.list_uploads'))
