def label():
# Window detail
doc_id = request.form.get("doc_id", type=int)
ref_start = request.form.get("ref_start", type=int)
ref_end = request.form.get("ref_end", type=int)
types = json.loads(request.form.get("types", type=str))
label = request.form.get("label", type=int)
as_rule = request.form.get("rule", type=bool)
_check_permissinns(ref_start, ref_end, doc_id)
interval = Interval(ref_start, ref_end)
with get_cursor() as cursor:
ctl = Controller(cursor, doc_id, g.account["id"])
# Token level annotation
annotation_id = ctl.token_annotation(interval,
AnnotationDecision.SECRET)
# Update label
ctl.set_label(interval, label)
# Improved search for candidates
candidate = ctl.add_candidate_rule(types, annotation_id)
# Set rule label
if as_rule:
ctl.set_rule_label(types, label)
commit()
# Annotate rest of file using new candidate
if candidate:
_call_re_annotate()
return jsonify({"status": "ok"})
def user_remove(user_id: int):
with get_cursor() as cursor:
cursor.execute("DELETE FROM account WHERE id = %s", (user_id, ))
commit()
# Return OK reply
return jsonify({"stutus": "ok"})
def pre_process(document_id: int) -> None:
with get_cursor() as cursor:
cursor.execute("SELECT uid FROM submission WHERE id = %s",
(document_id, ))
uid = cursor.fetchone()["uid"]
# Input recognized file
input_file = get_submission_file(uid, SubmissionStatus.NEW)
recognized_file = get_submission_file(uid, SubmissionStatus.RECOGNIZED)
# Run recognition
num_tokens = recognize_file(input_file, recognized_file)
with get_cursor() as cursor:
cursor.execute(
"UPDATE submission SET status = %s, num_tokens = %s WHERE id = %s",
(SubmissionStatus.RECOGNIZED.value, num_tokens, document_id))
commit()
ctl = controller.Controller(cursor, document_id)
# Run pre-annotation
detect_recognized_name_entries(recognized_file, ctl)
# Re-annotate
apply_rules(recognized_file, ctl)
# Update document status
cursor.execute("UPDATE submission SET status = %s WHERE id = %s",
(SubmissionStatus.PRE_ANNOTATED.value, document_id))
commit()
def test_rule_db(client: FlaskClient) -> None:
"""
Test that 'rule' database exist and has access from Flask
"""
with app.app_context():
condition = ["test_word"]
confidence = random.randint(10, 10000)
# Test insert
with get_cursor() as cursor:
cursor.execute(
"DELETE FROM rule WHERE type = %s and condition = %s",
(RuleType.WORD_TYPE.value, condition))
cursor.execute(
"INSERT INTO rule (type, condition, confidence) VALUES (%s, %s, %s)",
(RuleType.WORD_TYPE.value, condition, confidence))
commit()
# Test select
with get_cursor() as cursor:
cursor.execute(
"SELECT confidence FROM rule WHERE type = %s and condition = %s",
(RuleType.WORD_TYPE.value, condition))
data = cursor.fetchone()
assert data is not None
assert data["confidence"] == confidence
cursor.execute(
"DELETE FROM rule WHERE type = %s and condition = %s",
(RuleType.WORD_TYPE.value, condition))
commit()
def remove(rule_id: int):
with get_cursor() as cursor:
# Remove already made annotation
cursor.execute("DELETE FROM rule WHERE id = %s", (rule_id, ))
commit()
# Return OK reply
return jsonify({"status": "ok"})
def decision():
# Window detail
doc_id = request.form.get("doc_id", type=int)
ref_start = request.form.get("ref_start", type=int)
ref_end = request.form.get("ref_end", type=int)
_check_permissinns(ref_start, ref_end, doc_id)
# Process decision
if request.form["decision"] == "PUBLIC":
decision = AnnotationDecision.PUBLIC
else:
decision = AnnotationDecision.SECRET
# Rule type
rule_type = None
condition = None
if request.form["kind"] == "NE_TYPE":
rule_type = RuleType.NE_TYPE
condition = [request.form["ne_type"]]
elif request.form["kind"] == "WORD_TYPE":
rule_type = RuleType.WORD_TYPE
condition = json.loads(request.form["tokens"])
# Save result to db
interval = Interval(ref_start, ref_end)
rule_confidence = 1 if decision == AnnotationDecision.PUBLIC else -1
with get_cursor() as cursor:
ctl = Controller(cursor, doc_id, g.account["id"])
# Decision connected with rules
if rule_type:
rule = ctl.set_rule(rule_type, condition, rule_confidence)
# Token level annotation
annotation_id = ctl.token_annotation(interval, decision)
# Improved search for candidates
candidate = None
if rule_type != RuleType.NE_TYPE:
if decision == AnnotationDecision.SECRET:
candidate = ctl.add_candidate_rule(
json.loads(request.form["tokens"]), annotation_id)
else:
ctl.drop_candidate_rule(annotation_id)
# (Pre) connect rule to annotation
if rule_type:
ctl.connect(annotation_id, rule)
# Commit changes to database
commit()
if rule_type or candidate:
# Annotate rest using background task
_call_re_annotate(doc_id)
# Send OK reply
return jsonify({"status": "ok"})
def upload():
form = UploadForm()
if form.validate_on_submit():
if not form.file.data and not form.text.data:
flash(_("File or text input required."), category="error")
else:
# Load input
if form.file.data:
csv_input = csv.DictReader(
TextIOWrapper(form.file.data, "UTF8"))
else:
csv_input = csv.DictReader(form.text.data.splitlines())
# Check header format
if any(field not in csv_input.fieldnames
for field in ["type", "condition", "decision"]):
flash(_("Some header columns are missing"), category="warning")
return render_template("rule/import.html", form=form)
if "author" in csv_input.fieldnames:
flash(_("Rule authors were ignored in import"),
category="warning")
# Parse input
line_num = 0
with get_cursor() as cursor:
for row in csv_input:
# Line numbering
line_num += 1
# Try to import to db
try:
# Check row format
if row["type"] is None or row[
"condition"] is None or row["decision"] is None:
raise IndexError
# Import data
cursor.execute(
"INSERT INTO rule (type, condition, confidence) VALUES(%s, %s, %s)"
" ON CONFLICT (type, condition) DO UPDATE SET confidence = EXCLUDED.confidence",
(row["type"], row["condition"].split('='),
row["decision"]))
except (IndexError, DataError):
flash(_("Illegal format on line %(line_num)s.",
line_num=line_num),
category="error")
return render_template("rule/import.html", form=form)
commit()
flash(_("%(num)s rules imported", num=csv_input.line_num),
category="message")
_call_re_annotate()
return redirect(url_for(".index"))
# Prepare output
return render_template("rule/import.html", form=form)
def new():
# Prepare requests
label = request.form["label"]
replacement = request.form["replacement"]
with get_cursor() as cursor:
# Update value
cursor.execute("INSERT INTO label (name, replacement) VALUES (%s, %s)", (label, replacement))
commit()
# Return OK reply
return jsonify({"status": "ok"})
def re_annotate(doc_id: int) -> None:
with get_cursor() as cursor:
# Get submission file
cursor.execute("SELECT uid FROM submission WHERE id = %s", (doc_id, ))
uid = cursor.fetchone()["uid"]
submission_file = get_submission_file(uid, SubmissionStatus.RECOGNIZED)
ctl = controller.Controller(cursor, doc_id)
# Parse file and apply rules
apply_rules(submission_file, ctl)
commit()
def update():
# Prepare requests
row_id = int(request.form["pk"])
new_value = request.form["value"]
with get_cursor() as cursor:
# Update value
if request.form["name"] == "label":
cursor.execute("UPDATE label SET name = %s WHERE id = %s", (new_value, row_id))
else:
cursor.execute("UPDATE label SET replacement = %s WHERE id = %s", (new_value, row_id))
commit()
# Return OK reply
return jsonify({"status": "ok"})
def delete_account():
delete_form = DeleteAccountForm(request.form)
if delete_form.validate_on_submit():
if check_password_hash(g.account["password"],
delete_form.password.data):
with get_cursor() as cursor:
cursor.execute("DELETE FROM account WHERE id = %s",
(g.account["id"], ))
commit()
session.clear()
# Show confirmation UI
flash(_("Account was deleted."), category="message")
return redirect(url_for("index"))
else:
delete_form.password.errors.append(_("Incorrect password."))
return render_template("account/delete.html", form=delete_form)
def re_annotate_all(skip_doc_id: int) -> None:
with get_cursor() as cursor:
# Get submission file
cursor.execute("SELECT id,uid FROM submission WHERE status = %s",
(SubmissionStatus.PRE_ANNOTATED.value, ))
for row in cursor:
id = row["id"]
uid = row["uid"]
if skip_doc_id == id:
continue
submission_file = get_submission_file(uid,
SubmissionStatus.RECOGNIZED)
with get_cursor() as cur:
ctl = controller.Controller(cur, id)
# Parse file and apply rules
apply_rules(submission_file, ctl)
commit()
def change_password():
# Access checks
token = request.args.get("token")
if token: # Token is used for password resetting using email
try:
account_id = read_reset_token(token)
except BadTimeSignature:
flash(_("Link has expired. Create a new one."), category="error")
return redirect(url_for("auth.reset"))
except BadSignature:
raise BadRequest("Invalid token")
elif g.account is None: # Without token the user has to be logged in
return redirect(url_for("auth.login"))
else:
account_id = g.account["id"]
# Web page
form = ChangePasswordForm(request.form)
# Validate form
if form.validate_on_submit():
if token or check_password_hash(g.account["password"],
form.old_password.data):
# Update db
with get_cursor() as cursor:
cursor.execute(
"UPDATE account SET password = %s WHERE id = %s",
(generate_password_hash(
form.new_password.data), account_id))
commit()
# Notify user
flash(_("Password was changed."), category="message")
return redirect(url_for(".index"))
else:
form.old_password.errors.append(_("Incorrect password."))
return render_template("account/password.html",
form=form,
token=token is not None)
def register():
"""Register a new account.
Validates that the email is not already taken. Handles password for security.
"""
form = AccountRegisterForm(request.form)
if request.method == "POST":
# Prepare cursor for db access
cursor = get_cursor()
# Check form data
if not form.validate():
flash(_("Form content is not valid."), category="error")
elif is_email_unique(cursor, form.email.data):
cursor.execute(
"INSERT INTO account (full_name, type, window_size, email, password) "
"VALUES (%s, %s, %s, %s, %s)",
(form.full_name.data, form.type.data, form.window_size.data,
form.email.data, generate_password_hash(form.password.data)),
)
commit()
cursor.close()
# Notify user
flash(_("Registration was successful."), category="message")
# Redirect to correct page
if g.account:
return redirect(url_for("account.index"))
else:
return redirect(url_for("auth.login"))
else:
cursor.close()
form.email.errors.append(_("Value is already taken."))
else:
form.password.data = ''.join(
secrets.choice(string.ascii_letters + string.digits)
for _ in range(8))
return render_template("auth/register.html", form=form)