def upload_gramps(): """ Load a gramps xml file to temp directory for processing in the server """ try: infile = request.files['filenm'] material = request.form['material'] #logger.debug("Got a {} file '{}'".format(material, infile.filename)) t0 = time.time() upload_folder = uploads.get_upload_folder(current_user.username) os.makedirs(upload_folder, exist_ok=True) pathname = loadfile.upload_file(infile,upload_folder) shareds.tdiff = time.time()-t0 logname = pathname + ".log" uploads.set_meta(current_user.username,infile.filename, status=uploads.STATUS_UPLOADED, upload_time=time.time()) msg = f"{util.format_timestamp()}: User {current_user.name} ({current_user.username}) uploaded the file {pathname}" open(logname,"w", encoding='utf-8').write(msg) email.email_admin( "Stk: Gramps XML file uploaded", msg ) syslog.log(type="gramps file uploaded",file=infile.filename) logger.info(f'-> bp.gramps.routes.upload_gramps/{material} f="{infile.filename}"' f' e={shareds.tdiff:.3f}sek') except Exception as e: return redirect(url_for('gramps.error_page', code=1, text=str(e))) return redirect(url_for('gramps.list_uploads'))
def email_from_admin(subject, body, receiver): sender = shareds.app.config.get('ADMIN_EMAIL_FROM') if sender: syslog.log(type="sent email from admin", sender=sender, receiver=receiver, subject=subject) return email(sender, receiver, subject, body)
def move_in_2(): """ Move the accepted Batch to Isotammi database """ owner = request.form['user'] batch_id = request.form['batch'] auditor = current_user.username logger.info(f' bp.audit.routes.move_in_2 u={owner} b={batch_id}') merger = Batch_merge() msg = merger.move_whole_batch(batch_id, owner, auditor) syslog.log(type="batch to Common data", batch=batch_id, by=owner, msg=msg) return redirect(url_for('audit.move_in_1', batch_name=batch_id))
def gedcom_delete_old_versions(gedcom): gedcom_folder = gedcom_utils.get_gedcom_folder() gedcom_folder = os.path.abspath(gedcom_folder) gedcom = secure_filename(gedcom) for name in os.listdir(gedcom_folder): filename = os.path.join(gedcom_folder, name) if name.startswith(gedcom+"."): gedcom_utils.removefile(filename) logging.info("Deleted:"+filename) syslog.log(type="deleted old versions for gedcom",gedcom=gedcom) return redirect(url_for('.gedcom_info',gedcom=gedcom))
def background_load_to_neo4j(username,filename): ''' Imports gramps xml data to database ''' upload_folder = get_upload_folder(username) pathname = os.path.join(upload_folder,filename) metaname = pathname+".meta" logname = pathname+".log" update_metafile(metaname, progress={}) steps = [] try: os.makedirs(upload_folder, exist_ok=True) set_meta(username,filename,status=STATUS_LOADING) this_thread = threading.current_thread() this_thread.progress = {} counts = gramps_loader.analyze_xml(username, filename) update_metafile(metaname,counts=counts,progress={}) threading.Thread(target=lambda: i_am_alive(metaname,this_thread),name="i_am_alive for " + filename).start() steps,batch_id = gramps_loader.xml_to_neo4j(pathname,username) for step in steps: print(step) if not batch_id: raise RuntimeError("Run Failed, missing batch_id") if os.path.exists(metaname): set_meta(username,filename, batch_id=batch_id, status=STATUS_DONE) msg = "{}:\nStored the file {} from user {} to neo4j".format(util.format_timestamp(),pathname,username) msg += "\nBatch id: {}".format(batch_id) msg += "\nLog file: {}".format(logname) msg += "\n" for step in steps: msg += "\n{}".format(step) msg += "\n" open(logname,"w", encoding='utf-8').write(msg) email.email_admin( "Stk: Gramps XML file stored", msg ) syslog.log(type="storing to database complete",file=filename,user=username) except: traceback.print_exc() res = traceback.format_exc() set_meta(username,filename,status=STATUS_FAILED) msg = "{}:\nStoring the file {} from user {} to database FAILED".format(util.format_timestamp(),pathname,username) msg += "\nLog file: {}".format(logname) msg += "\n" + res for step in steps: msg += "\n{}".format(step) msg += "\n" open(logname,"w", encoding='utf-8').write(msg) email.email_admin( "Stk: Gramps XML file storing FAILED", msg ) syslog.log(type="gramps store to database failed",file=filename,user=username)
def gedcom_delete(gedcom): gedcom_folder = gedcom_utils.get_gedcom_folder() gedcom_folder = os.path.abspath(gedcom_folder) gedcom = secure_filename(gedcom) for name in os.listdir(gedcom_folder): if name.endswith("-history"): continue # do not remove history if (name == gedcom or name.startswith(gedcom+".") or name.startswith(gedcom+"-") ): filename = os.path.join(gedcom_folder, name) gedcom_utils.removefile(filename) logging.info("Deleted:"+filename) syslog.log(type="deleted a gedcom",gedcom=gedcom) logger.info(f'-> bp.gedcom.routes.gedcom_delete f="{gedcom}"') return redirect(url_for('.gedcom_list'))
def email_admin(subject, body, sender=None): if sender is None: sender = shareds.app.config.get('ADMIN_EMAIL_FROM') mail_to = shareds.app.config.get('ADMIN_EMAIL_TO') if sender and mail_to: if email(sender, mail_to, subject, body): syslog.log(type="sent email to admin", sender=sender, receiver=mail_to, subject=subject) return True else: syslog.log(type="FAILED: email to admin", sender=sender, receiver=mail_to, subject=subject) return False return False
def initiate_background_load_to_neo4j(userid,filename): ''' Starts gramps xml data import to database ''' #=========================================================================== # subprocess.Popen("PYTHONPATH=app python runload.py " # + pathname + " " # + username + " " # + logname, # shell=True) #=========================================================================== def background_load_to_neo4j_thread(app): with app.app_context(): background_load_to_neo4j(userid,filename) threading.Thread(target=background_load_to_neo4j_thread, args=(shareds.app,), name="neo4j load for " + filename).start() syslog.log(type="storing to database initiated",file=filename,user=userid) return False
def batch_delete(batch_id): from models.gen.batch_audit import Batch filename = Batch.get_filename(current_user.username,batch_id) metafile = filename.replace("_clean.",".") + ".meta" if os.path.exists(metafile): data = eval(open(metafile).read()) if data.get('batch_id') == batch_id: del data['batch_id'] data['status'] = uploads.STATUS_REMOVED open(metafile,"w").write(repr(data)) Batch.delete_batch(current_user.username,batch_id) logger.info(f'-> bp.gramps.routes.batch_delete f="{batch_id}"') syslog.log(type="batch_id deleted",batch_id=batch_id) flash(_("Batch id %(batch_id)s has been deleted", batch_id=batch_id), 'info') referrer = request.headers.get("Referer") return redirect(referrer)
def gedcom_upload(): # code from: http://flask.pocoo.org/docs/1.0/patterns/fileuploads/ def allowed_file(filename): return '.' in filename and \ filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS gedcom_folder = gedcom_utils.get_gedcom_folder() # check if the post request has the file part if 'file' not in request.files: flash(_('Choose a GEDCOM file to upload'), category='flash_warning') return redirect(url_for('.gedcom_list')) file = request.files['file'] # if user does not select file, browser also # submit an empty part without filename if file.filename == '': # pragma: no cover flash(_('Choose a GEDCOM file to upload'), category='flash_warning') return redirect(url_for('.gedcom_list')) if file and allowed_file(file.filename): filename = secure_filename(file.filename) os.makedirs(gedcom_folder, exist_ok=True) fullname = os.path.join(gedcom_folder, filename) if os.path.exists(fullname): flash(_('This GEDCOM file already exists'), category='flash_error') return redirect(url_for('.gedcom_list')) file.save(fullname) desc = request.form['desc'] encoding = util.guess_encoding(fullname) metadata = { 'desc':desc, 'encoding':encoding, 'upload_time':util.format_timestamp(), 'size':os.stat(fullname).st_size, } gedcom_utils.save_metadata(filename, metadata) gedcom_utils.history_init(fullname) syslog.log(type="uploaded a gedcom",gedcom=file.filename) logger.info(f'-> bp.gedcom.routes.gedcom_upload n={os.stat(fullname).st_size/1024.}kb') return redirect(url_for('.gedcom_info',gedcom=filename))
def log_user_logged_out(sender, user, **extra): syslog.log(type="user logged out")
def log_user_logged_in(sender, user, **extra): syslog.log(type="user logged in") session['lang'] = current_user.language
session['lang'] = reqlang else: reqlang = session.get('lang') if not reqlang: if current_user.is_authenticated: reqlang = current_user.language session['lang'] = reqlang return reqlang from flask_login import user_logged_in, user_logged_out from models import syslog def log_user_logged_in(sender, user, **extra): syslog.log(type="user logged in") session['lang'] = current_user.language def log_user_logged_out(sender, user, **extra): syslog.log(type="user logged out") syslog.syslog_init() syslog.log(type="application initialized") user_logged_in.connect(log_user_logged_in, shareds.app) user_logged_out.connect(log_user_logged_out, shareds.app) import setups
def xml_delete(username,xmlfile): uploads.delete_files(username,xmlfile) syslog.log(type="gramps file uploaded",file=xmlfile,user=username) logger.info(f'-> bp.admin.routes.xml_delete f="{xmlfile}"') return redirect(url_for('admin.list_uploads', username=username))
def xml_delete(xmlfile): uploads.delete_files(current_user.username,xmlfile) logger.info(f'-> bp.gramps.routes.xml_delete f="{os.path.basename(xmlfile)}"') syslog.log(type="gramps file deleted",file=xmlfile) return redirect(url_for('gramps.list_uploads'))