def get_doc():
id = ""
uuid = ""
if request.method == 'GET':
uuid = request.args.get('uuid')
req = request.get_json()
res = {"error": "Some error on /status"}
if req:
id = req['submission_id']
if 'uuid' in req.keys():
uuid = req['uuid']
wanted_fields = [
"uuid", "process_stage", "process_status", "body", "processed_body",
"file_name", "filepath"
]
# MySQL
db = get_db()
conn = db.connect()
cursor = conn.cursor()
cursor.execute(
"SELECT {} FROM document WHERE uuid = %s;".format(
", ".join(wanted_fields)), (uuid))
result = cursor.fetchall()
if len(result) == 0:
# Record not exist
return jsonify({
"status": "FAILED",
"msg": "No result.",
"result": [],
})
else:
logging.warn(result[0])
if result[0][wanted_fields.index("process_status")] != "FINISHED":
# still waiting for processing
return jsonify({
"status": "OK",
"msg": "Processing.",
"result": [],
})
else:
#body = result[0]
cursor = conn.cursor()
cursor.execute(
"SELECT {} FROM document WHERE uuid = %s;".format(
", ".join(wanted_fields)), (uuid))
result2 = cursor.fetchall()
dict_result = [dict(zip(wanted_fields, row)) for row in result2]
return jsonify({
"status": "OK",
"msg": "OK",
"result": dict_result,
})
def load_user(id):
user = User()
user.set_anonymous(False)
user.set_active(True)
user.set_authenticated(True)
db = get_db()
doc = db.fetch(id)
user.fromJSON(doc.value)
return user
def get_status():
req = request.get_json()
res = {"error": "Some error on /status"}
atype = "all"
page = "0"
limit = "10"
if request.method == 'GET':
atype = request.args.get('type', default="all")
page = request.args.get('page', default="0")
limit = request.args.get('limit', default="10")
wanted_fields = [
"uuid", "process_stage", "process_status", "body", "processed_body",
"file_name", "filepath"
]
afilter = {
"all":
["PROCESSING_CONVERT", "FINISHED", "PROCESSING", "NOT PROCESSED"],
"finished": ["FINISHED", "<FILLER>"],
"unfinished": ["PROCESSING_CONVERT", "PROCESSING", "NOT PROCESSED"],
}
# MySQL
db = get_db()
conn = db.connect()
cursor = conn.cursor()
try:
page = int(page)
limit = int(limit)
except:
page = 0
limit = 10
logging.warn(
f"[WARN] get_status: convert page or limit failed: {page}, {limit}. Use default (page 0, limit 10)"
)
cursor.execute(
"SELECT {} FROM document WHERE process_status in {} LIMIT %s OFFSET %s;"
.format(
", ".join(wanted_fields),
str(tuple(afilter[atype])),
), (limit, page * limit))
result = cursor.fetchall()
dict_result = [dict(zip(wanted_fields, row)) for row in result]
return jsonify({
"status": "OK",
"msg": "OK",
"result": dict_result,
})
def init_db(app):
"""Initialize SQLite database.
"""
logging.debug("Initializing database...")
db = database.get_db()
if getattr(sys, "frozen", False):
root = Path(sys._MEIPASS)
else:
root = Path(".")
with app.open_resource(str(Path(root, "schema.sql"))) as schemafile:
schema = schemafile.read().decode("utf-8")
db.executescript(schema)
db.commit()
database.close_db()
def login():
form = LoginForm()
if form.validate_on_submit():
user = User()
user.email = form.email.data
user.set_id(user.genID())
db = get_db()
doc = db.fetch(user.get_id())
if doc is not None:
user.fromJSON(doc.value)
if (user.check_password(user.password, form.password.data)):
login_user(user)
flash('you were successfully logged in as ' + user.email)
return redirect(url_for('index'))
# else, show error
else:
flash('Login error', 'error')
else:
flash('Login error', 'error')
return render_template('login.html', form=form)
def get_doc():
req = request.get_json()
res = {"error": "Some error on /status"}
id = req['id']
# # MongoDB
# connect_db()
# result = EDocuments.objects.with_id(id)
# body = result.body
# MySQL
db = get_db()
conn = db.connect()
cursor = conn.cursor()
cursor.execute(
"SELECT process_status, body, processed_body FROM document WHERE id = %s;",
(id))
result = cursor.fetchall()
if len(result) == 0:
return jsonify({
"status": "FALED",
"exist": False,
"success": False,
"processed_text": "",
"body": "",
})
else:
body = result[0]
return jsonify({
"status": "OK",
"exist": True,
"success": True if result[0][0] == "FINISHED" else False,
"processed_text": result[0][2],
"body": result[0][1],
})
def register():
form = RegisterForm()
if form.validate_on_submit():
db = get_db()
user = User()
user.email = form.email.data
user.set_bcrypt(form.password.data)
user.set_id(user.genID())
if db.exists(user.get_id()):
flash('Email adress already exists in our database')
else:
if form.isPhysical.data == "physical":
user.identity.createPhysicalPerson()
user.identity.person.firstName = form.firstName.data
user.identity.person.lastName = form.lastName.data
else:
user.identity.createLegalPerson()
user.identity.person.name = form.name.data
db.insert(user.get_id(), user.toJSON())
flash('User created', 'success')
return redirect(url_for('index'))
return render_template('register.html', form=form)
def test(): db = get_db() res = db.test() return str(res)
def queue_convert_doc(res={},
processed_content="",
hash_id="",
user_id="",
filename="",
filepath=""):
"""Add submission to task list in database, and ready to be queue for
further processing.
:param res: dict object which is built to produce Flask.Response() (default {})
:param processed_content: string type data to be input for hashing (default "")
:param hash_id: (default "")
:param user_id: (default "")
:param filename: (default "")
:param filepath: (default "")
:returns: JSON
:rtype: Flask.Response()
"""
cursor = None
cur = None
job_id = ""
insertID = ""
result = {}
# 1. New entry in table `document`, record the status of a) conversion file or b) process_analysis
with current_app.app_context():
try:
# MySQL
db = get_db()
conn = db.connect()
cursor = conn.cursor()
process_stage = "UPLOADED"
if filename == "" and filepath == "":
# Text
cursor.execute(
"INSERT INTO document (body, uuid, process_stage) VALUES (%s, %s, %s);",
(processed_content, hash_id, process_stage))
else:
# File
cursor.execute(
"INSERT INTO document (body, uuid, process_stage, file_name, filepath) VALUES (%s, %s, %s, %s, %s);",
("", hash_id, process_stage, filename, filepath))
conn.commit()
insertID = cursor.lastrowid
except Exception as e:
# Response with failure in case of any exception
res["status"] = "FAILED"
res["msg"] = f"Cannot insert DB.\n{e}"
res["result"] = []
logging.warn(f"Cannot insert DB.\n{e}")
else:
# After successful insertion, process the text and response with the document ID.
result["id"] = insertID
result["uuid"] = hash_id
res["status"] = "OK"
res["msg"] = f"Record inserted."
try:
redis_connection = get_redis_connection()
logging.warn("current_app.config: " +
str(current_app.config["QUEUES"]))
q = Queue(current_app.config["QUEUES"][1],
connection=redis_connection)
job = {}
if filename == "" and filepath == "":
# Text
job = q.enqueue(process_analysis, str(insertID), hash_id)
else:
# File
job = q.enqueue(process_convert, str(insertID), hash_id,
filename, filepath)
job_id = job.get_id()
result["task_id"] = job_id
logging.warn("Processing queued with job ID: " + str(job_id))
logging.warn(
f"Current {current_app.config['QUEUES'][1]} Queue: " +
str(len(q)))
except Exception as e:
logging.warn(f"Queue failed\n{e}")
finally:
# MySQL
if cursor is not None:
cursor.close()
# Default return
res["result"] = [result]
return make_response(jsonify(res))
def process_convert(id, uuid, filename="", filepath=""):
"""Fetch a task in task list, do conversion of the content.
It write output results to text files on local disk.
Update task status in database after task is finished.
:param id: database auto incremental id
:param uuid: unique id of database
:param filename: (default "")
:param filepath: (default "")
:returns: success
:rtype: boolean
"""
cur = None
db = None
with current_app.app_context():
try:
logging.warn("Processing document {}.".format(id))
# MySQL
db = get_db()
conn = db.connect()
cursor = conn.cursor()
cursor.execute(
"SELECT process_status, body FROM document WHERE uuid = %s;",
(uuid))
result = cursor.fetchall()
body = result[0][1]
if True or len(body) > 0:
if False and body["process_status"] == "FINISHED":
logging.warn("document {} already processed.".format(uuid))
success = True
else:
# MySQL
cursor.execute(
"UPDATE document SET process_status = 'PROCESSING_CONVERT' WHERE uuid = %s;",
(uuid))
conn.commit()
result = ""
try:
# result = parser(
# filename=filename,
# filepath=filepath,
# newname=filename+"_converted",
# )
from yourapp.main import main
# return paths of Images files
result = main(
id=id,
uuid=uuid,
filename=filename,
filepath=filepath,
)
except Exception as e:
with open("/home/flask/app/output_files/{}-{}.txt".
format(id, uuid),
mode="a",
encoding="utf8") as f:
f.write("Error in main: {}".format(e))
# MySQL
cursor.execute(
"UPDATE document SET process_status = 'FINISHED', processed_body = %s WHERE uuid = %s;",
(json.dumps(result), uuid))
conn.commit()
success = True
insertID = id
"""
No need text analysis
"""
# try:
# redis_connection = get_redis_connection()
# q = Queue(current_app.config['QUEUES'][0], connection=redis_connection)
# job = q.enqueue(process_analysis, str(insertID), uuid, submission_id)
# job_id = job.get_id()
# logging.warn("text processing queued with job ID: " + str(job_id))
# logging.warn(f"Curr {current_app.config['QUEUES'][0]} Queue: " + str(len(q)))
# except Exception as e:
# logging.warn(f"queue failed, process text in local thread\n{e}")
# # process_success = process_analysis(insertID)
# # if(not process_success["status"]):
# # res["success"] = False
else:
print("unable to get document {} .".format(id))
success = False
except Exception as e:
pass
with open("/home/flask/app/output_files/{}-{}.txt".format(
id, uuid),
mode="a",
encoding="utf8") as f:
f.write("\n")
f.write(str(e))
success = False
finally:
pass
return success
def process_task(id):
cur = None
db = None
with current_app.app_context():
try:
logging.warn("processing document {}.".format(id))
# # MongoDB
# connect_db()
# result = EDocuments.objects.with_id(id)
# body = result.body
# MySQL
db = get_db()
conn = db.connect()
cursor = conn.cursor()
cursor.execute(
"SELECT process_status, body FROM document WHERE id = %s;",
(id))
result = cursor.fetchall()
body = result[0][1]
if len(body) > 0:
if False and body["process_status"] == "FINISHED":
logging.warn("document {} already processed.".format(id))
success = True
else:
# MongoDB
# result.modify(set__process_status = "PROCESSING")
# MySQL
cursor.execute(
"UPDATE document SET process_status = 'PROCESSING' WHERE id = %s;",
(id))
conn.commit()
result = ""
# Import yourapp
from yourapp.main import main
try:
result = main(
uid=id,
body=body,
)
except Exception as e:
with open("/home/flask/app/output_files/{}.txt".format(
id),
mode="a",
encoding="utf8") as f:
f.write("Error in main: {}".format(e))
# MongoDB
# result.modify(set__process_status = "FINISHED", set__processed_body = json.dumps(result))
# MySQL
cursor.execute(
"UPDATE document SET process_status = 'FINISHED', processed_body = %s WHERE id = %s;",
(json.dumps(result), id))
conn.commit()
success = True
else:
print("unable to get document {} .".format(id))
success = False
except Exception as e:
pass
with open("/home/flask/app/output_files/{}.txt".format(id),
mode="a",
encoding="utf8") as f:
f.write("\n")
f.write(str(e))
success = False
finally:
pass
return success
def new_file():
req = request.get_json()
res = {}
res["success"] = False
inputContent = request.form.get('textline')
inputContent = inputContent.strip()
if inputContent == '':
logging.warn('No text in request')
return make_response(
jsonify({
'status': 'FAILED',
'msg': 'No text in in request'
}), 502)
else:
# Generate Hash
processedContent = inputContent
hashId = genHash(processedContent)
cursor = None
cur = None
job_id = ""
with current_app.app_context():
try:
# # MongoDB
# connect_db()
# doc = EDocuments(body = processedContent)
# doc.save()
# insertID = doc.id
# MySQL
db = get_db()
conn = db.connect()
cursor = conn.cursor()
cursor.execute("INSERT INTO document (body) VALUES (%s);",
(processedContent))
conn.commit()
insertID = cursor.lastrowid
except Exception as e:
# response with failure in case of any exception
res["success"] = False
logging.warn("Can't insert DB\n{}".format(e))
else:
# after successful insertion, process the text and response with the document ID.
res["success"] = True
res["id"] = insertID
try:
redis_connection = get_redis_connection()
q = Queue(current_app.config['QUEUES'][0],
connection=redis_connection)
job = q.enqueue(process_task, str(insertID))
job_id = job.get_id()
logging.warn("text processing queued with job ID: " +
str(job_id))
logging.warn("Curr Queue: " + str(len(q)))
except:
logging.warn("queue failed, process text in local thread")
process_success = process_task(insertID)
if (not process_success["status"]):
res["success"] = False
return make_response(
jsonify({
'status': 'OK',
'msg': 'Record inserted.',
'hash': hashId,
'id': insertID,
"task_id": job_id
}))
finally:
# # MongoDB
# close_db()
# MySQL
if cursor is not None:
cursor.close()
return make_response(
jsonify({
'status': 'FAILED',
'msg': 'No file attached in request'
}), 502)
return make_response(
jsonify({
'status': 'FAILED',
'msg': 'POST method is preferred.'
}), 502)