def switch_control_lists_access(
corpus: Corpus,
users: List[User],
old_control_lists_id: int,
):
"""Switch user access from one control list to another.
:param list users: list of users to switch
:param int old_control_lists_id: ID of old control list
"""
for user in users:
# do not delete access to old control list if another corpus uses it
for user_corpus in Corpus.for_user(user):
if user_corpus.control_lists_id == old_control_lists_id and user_corpus.id != corpus.id:
break
else:
control_lists_user = ControlListsUser.query.filter(
ControlListsUser.control_lists_id == old_control_lists_id,
ControlListsUser.user_id == user.id,
).one_or_none()
# do not delete access to old control list if user is owner
if control_lists_user and not control_lists_user.is_owner:
db.session.delete(control_lists_user)
# add access to new control list
ControlLists.link(corpus, user)
db.session.commit()
def add_control_lists(self):
""" Loads a control list from a folder as a public one
:param folder_name: Name of the folder in app/configurations/langs
"""
ControlLists.add_default_lists()
db.session.commit()
def db_create():
""" Creates a local database
"""
with app.app_context():
db.create_all()
Role.add_default_roles()
User.add_default_users()
ControlLists.add_default_lists()
db.session.commit()
click.echo("Created the database")
def db_recreate():
""" Recreates a local database. You probably should not use this on
production.
"""
with app.app_context():
db.drop_all()
db.create_all()
Role.add_default_roles()
User.add_default_users()
ControlLists.add_default_lists()
db.session.commit()
click.echo("Dropped then recreated the database")
def rename(control_list_id):
""" This routes allows user to send email to list administrators
"""
control_list, is_owner = ControlLists.get_linked_or_404(
control_list_id=control_list_id, user=current_user)
form = Rename(prefix="rename")
control_list_link = url_for('control_lists_bp.get',
control_list_id=control_list_id,
_external=True)
if not is_owner and not current_user.is_admin():
flash("You are not an owner of the list.", category="error")
return redirect(control_list_link)
if request.method == "POST" and form.validate_on_submit():
control_list.name = form.title.data
db.session.add(control_list)
try:
db.session.commit()
flash("The name of the list has been updated.", category="success")
except:
flash(
"There was an error when we tried to rename your control list.",
category="error")
return redirect(control_list_link)
return render_template_with_nav_info('control_lists/rename.html',
form=form,
control_list=control_list)
def contact(control_list_id):
""" This routes allows user to send email to list administrators
"""
control_list, is_owner = ControlLists.get_linked_or_404(
control_list_id=control_list_id, user=current_user)
form = SendMailToAdmin(prefix="mail")
if request.method == "POST" and form.validate_on_submit():
control_list_link = url_for('control_lists_bp.get',
control_list_id=control_list_id,
_external=True)
email.send_email_async(
app=current_app._get_current_object(),
bcc=[u[3] for u in control_list.owners] + [current_user.email],
recipient=[],
subject='[Pyrrha Control List] ' + form.title.data,
template='control_lists/email/contact',
# current_user is a LocalProxy, we want the underlying user
# object
user=current_user._get_current_object(),
message=form.message.data,
control_list_title=control_list.name,
url=control_list_link)
flash('The email has been sent to the control list administrators.',
'success')
return redirect(
url_for('control_lists_bp.contact',
control_list_id=control_list_id))
return render_template_with_nav_info('control_lists/contact.html',
form=form,
control_list=control_list)
def get(control_list_id):
control_list, is_owner = ControlLists.get_linked_or_404(
control_list_id=control_list_id, user=current_user)
return render_template_with_nav_info(
"control_lists/control_list.html",
control_list=control_list,
is_owner=is_owner,
can_edit=is_owner or current_user.is_admin(),
)
def normal_view():
lists = {"public": [], "submitted": [], "private": []}
for cl in ControlLists.get_available(current_user):
lists[cl.str_public].append(cl)
return render_template_with_nav_info('main/corpus_new.html',
lemmatizers=lemmatizers,
public_control_lists=lists,
tsv=request.form.get("tsv", ""))
def edit(cl_id, allowed_type):
""" Find allowed values and allow their edition
:param cl_id: Id of the Control List
:param allowed_type: Type of allowed value (lemma, morph, POS)
"""
if allowed_type not in ["lemma", "POS", "morph"]:
raise NotFound("Unknown type of resource.")
control_list, is_owner = ControlLists.get_linked_or_404(
control_list_id=cl_id, user=current_user)
can_edit = is_owner or current_user.is_admin()
if not can_edit:
return abort(403)
# In case of Post
if request.method == "POST":
allowed_values = request.form.get("allowed_values")
if allowed_type == "lemma":
allowed_values = [
x.replace('\r', '') for x in allowed_values.split("\n")
if len(x.replace('\r', '').strip()) > 0
]
elif allowed_type == "POS":
allowed_values = [
x.replace('\r', '') for x in allowed_values.split(",")
if len(x.replace('\r', '').strip()) > 0
]
else:
allowed_values = list(StringDictReader(allowed_values))
success = control_list.update_allowed_values(allowed_type,
allowed_values)
if success:
flash("Control List Updated", category="success")
else:
flash("An error occured", category="error")
values = control_list.get_allowed_values(allowed_type=allowed_type,
order_by="id")
if allowed_type == "lemma":
format_message = "This should be formatted as a list of lemma separated by new line"
values = "\n".join([d.label for d in values])
elif allowed_type == "POS":
format_message = "This should be formatted as a list of POS separated by comma and no space"
values = ",".join([d.label for d in values])
else:
format_message = "The TSV should at least have the header : label and could have a readable column for human"
values = "\n".join(
["label\treadable"] +
["{}\t{}".format(d.label, d.readable) for d in values])
return render_template_with_nav_info("control_lists/edit.html",
format_message=format_message,
values=values,
allowed_type=allowed_type,
control_list=control_list)
def _get_available():
"""Prepare dictionary of available control lists.
:returns: available control lists
:rtype: dict
"""
lists = {"public": [], "submitted": [], "private": []}
for cl in ControlLists.get_available(current_user):
lists[cl.str_public].append(cl)
return lists
def decorated_view(*args, **kwargs):
control_list, is_owner = ControlLists.get_linked_or_404(
control_list_id=kwargs[control_list_param], user=current_user)
can_edit = is_owner or current_user.is_admin()
if not can_edit:
flash("You are not an owner of the list.", category="error")
return redirect(
url_for(".get",
control_list_id=kwargs[control_list_param]))
return func(*args, control_list=control_list, **kwargs)
def dashboard():
"""admin dashboard page."""
if current_user.is_admin():
corpora = db.session.query(Corpus).all()
control_lists = db.session.query(ControlLists).all()
else:
corpora = Corpus.for_user(current_user)
control_lists = ControlLists.for_user(current_user)
return render_template_with_nav_info('main/dashboard.html',
current_user=current_user,
dashboard_corpora=corpora,
dashboard_control_lists=control_lists)
def test_db_create(self):
""" Test that db is created """
result = self.invoke(["db-create"])
self.assertIn("Created the database", result.output)
with self.app.app_context():
cl = ControlLists(name="Corpus1")
db.session.add(cl)
db.session.flush()
db.session.add(Corpus(name="Corpus1", control_lists_id=cl.id))
db.session.commit()
self.assertEqual(len(Corpus.query.all()), 1,
"There should have been an insert")
def propose_as_public(control_list_id):
""" This routes allows user to send email to application administrators
to propose a list as public for everyone to use
"""
control_list, is_owner = ControlLists.get_linked_or_404(
control_list_id=control_list_id, user=current_user)
if not is_owner:
flash("You are not an owner of the list.", category="error")
return redirect(
url_for("control_lists_bp.get", control_list_id=control_list_id))
elif control_list.public != PublicationStatus.private:
flash("This list is already public or submitted.", category="warning")
return redirect(
url_for("control_lists_bp.get", control_list_id=control_list_id))
form = SendMailToAdmin(prefix="mail")
if form.validate_on_submit():
admins = User.get_admins()
control_list_link = url_for('control_lists_bp.get',
control_list_id=control_list_id,
_external=True)
control_list.public = PublicationStatus.submitted
db.session.add(control_list)
try:
email.send_email_async(
app=current_app._get_current_object(),
bcc=[u.email for u in admins] + [current_user.email],
recipient=[],
subject='[Pyrrha Control List] ' + form.title.data,
template='control_lists/email/contact',
# current_user is a LocalProxy, we want the underlying user
# object
user=current_user._get_current_object(),
message=form.message.data,
control_list_title=control_list.name,
url=control_list_link)
flash('The email has been sent to the administrators.', 'success')
db.session.commit()
except Exception:
db.session.rollback()
flash("There was an error during the messaging step")
return render_template_with_nav_info(
'control_lists/propose_as_public.html',
form=form,
control_list=control_list)
def go_public(control_list_id):
""" This routes makes a list public
"""
control_list, is_owner = ControlLists.get_linked_or_404(
control_list_id=control_list_id, user=current_user)
if not current_user.is_admin():
flash("You do not have the rights for this action.", category="error")
elif control_list.public == PublicationStatus.public:
flash("This list is already public.", category="warning")
else:
control_list.public = PublicationStatus.public
db.session.add(control_list)
try:
db.session.commit()
flash('This list is now public.', 'success')
except Exception:
db.session.rollback()
flash("There was an error during the update.", category="error")
return redirect(
url_for("control_lists_bp.get", control_list_id=control_list_id))
def read_allowed_values(control_list_id, allowed_type):
if allowed_type not in ["POS", "morph"]:
flash("The category you selected is wrong petit coquin !",
category="error")
return redirect(url_for(".get", control_list_id=control_list_id))
control_list, is_owner = ControlLists.get_linked_or_404(
control_list_id=control_list_id, user=current_user)
kwargs = {}
template = "control_lists/read.html"
allowed_values = control_list.get_allowed_values(
allowed_type=allowed_type).all()
return render_template_with_nav_info(template=template,
control_list=control_list,
is_owner=is_owner,
can_edit=is_owner
or current_user.is_admin(),
allowed_type=allowed_type,
allowed_values=allowed_values,
readable=allowed_type == "morph",
**kwargs)
def information_read(control_list_id):
control_list, is_owner = ControlLists.get_linked_or_404(
control_list_id=control_list_id, user=current_user)
return render_template_with_nav_info('control_lists/information_read.html',
control_list=control_list)
from app.models import Corpus, WordToken, Column
from app.models import ControlLists
control_list = ControlLists(id=3, name="Latin")
corpus = Corpus(
name="Priapees",
id=3,
control_lists_id=control_list.id,
)
PriapeeColumns = [
Column(heading="Lemma", corpus_id=3),
Column(heading="POS", corpus_id=3),
Column(heading="Morph", corpus_id=3),
Column(heading="Similar", corpus_id=3),
]
tokens = [
WordToken(corpus=corpus.id, form="Carminis", lemma="carmen1", POS="NOMcom", left_context="Carminis incompti lusus lecture", right_context="procaces ,", label_uniform="carmen1", morph="Case=Gen|Numb=Sing"),
WordToken(corpus=corpus.id, form="incompti", lemma="incomptus", POS="ADJqua", left_context="Carminis incompti lusus lecture", right_context="procaces , conueniens", label_uniform="incomptus", morph="Case=Gen|Numb=Sing|Deg=Pos"),
WordToken(corpus=corpus.id, form="lusus", lemma="lusus", POS="NOMcom", left_context="Carminis incompti lusus lecture", right_context="procaces , conueniens Latio", label_uniform="lusus", morph="Case=Gen|Numb=Sing"),
WordToken(corpus=corpus.id, form="lecture", lemma="lego?", POS="VER", left_context="Carminis incompti lusus lecture", right_context="procaces , conueniens Latio pone", label_uniform="lego?", morph="Case=Voc|Numb=Sing|Mood=Par|Voice=Act"),
WordToken(corpus=corpus.id, form="procaces", lemma="procax", POS="ADJqua", left_context="Carminis incompti lusus lecture", right_context="procaces , conueniens Latio pone supercilium", label_uniform="procax", morph="Case=Acc|Numb=Plur|Deg=Pos"),
WordToken(corpus=corpus.id, form=",", lemma=",", POS="PUNC", left_context="Carminis incompti lusus lecture", right_context="procaces , conueniens Latio pone supercilium .", label_uniform=",", morph="MORPH=empty"),
WordToken(corpus=corpus.id, form="conueniens", lemma="conueniens", POS="ADJqua", left_context="incompti lusus lecture procaces", right_context=", conueniens Latio pone supercilium . non", label_uniform="conueniens", morph="Case=Nom|Numb=Sing|Deg=Pos"),
WordToken(corpus=corpus.id, form="Latio", lemma="latio", POS="NOMcom", left_context="lusus lecture procaces ,", right_context="conueniens Latio pone supercilium . non soror", label_uniform="latio", morph="Case=Nom|Numb=Sing"),
WordToken(corpus=corpus.id, form="pone", lemma="pono", POS="VER", left_context="lecture procaces , conueniens", right_context="Latio pone supercilium . non soror hoc", label_uniform="pono", morph="Numb=Sing|Mood=Imp|Tense=Pres|Voice=Act|Person=2"),
WordToken(corpus=corpus.id, form="supercilium", lemma="supercilium", POS="NOMcom", left_context="procaces , conueniens Latio", right_context="pone supercilium . non soror hoc habitat", label_uniform="supercilium", morph="Case=Acc|Numb=Sing"),
WordToken(corpus=corpus.id, form=".", lemma=".", POS="PUNC", left_context=", conueniens Latio pone", right_context="supercilium . non soror hoc habitat Phoebi", label_uniform=".", morph="MORPH=empty"),
WordToken(corpus=corpus.id, form="non", lemma="non", POS="ADVneg", left_context="conueniens Latio pone supercilium", right_context=". non soror hoc habitat Phoebi ,", label_uniform="non", morph="MORPH=empty"),
WordToken(corpus=corpus.id, form="soror", lemma="soror", POS="NOMcom", left_context="Latio pone supercilium .", right_context="non soror hoc habitat Phoebi , non", label_uniform="soror", morph="Case=Nom|Numb=Sing"),
WordToken(corpus=corpus.id, form="hoc", lemma="hic1", POS="PROdem", left_context="pone supercilium . non", right_context="soror hoc habitat Phoebi , non uesta", label_uniform="hic1", morph="Case=Nom|Numb=Sing"),
WordToken(corpus=corpus.id, form="habitat", lemma="habito", POS="VER", left_context="supercilium . non soror", right_context="hoc habitat Phoebi , non uesta sacello", label_uniform="habito", morph="Numb=Sing|Mood=Ind|Tense=Pres|Voice=Act|Person=3"),
from app.models import Corpus, WordToken, AllowedLemma, AllowedPOS, AllowedMorph, Column
from app.models import ControlLists, ControlListsUser
Floovant = Corpus(
name="Floovant",
id=2,
control_lists_id=2
)
FloovantColumns = [
Column(heading="Lemma", corpus_id=2),
Column(heading="POS", corpus_id=2),
Column(heading="Morph", corpus_id=2),
Column(heading="Similar", corpus_id=2),
]
FCL = ControlLists(id=2, name="Floovant")
FloovantTokens = [
WordToken(corpus=Floovant.id, form="SOIGNORS", lemma="seignor", left_context="", right_context="or escoutez que",
label_uniform="seignor", morph="NOMB.=p|GENRE=m|CAS=n"),
WordToken(corpus=Floovant.id, form="or", lemma="or4", left_context="SOIGNORS", right_context="escoutez que Dés",
label_uniform="or4", morph="DEGRE=-"),
WordToken(corpus=Floovant.id, form="escoutez", lemma="escouter", left_context="SOIGNORS or",
right_context="que Dés vos", label_uniform="escouter", morph="MODE=imp|PERS.=2|NOMB.=p"),
WordToken(corpus=Floovant.id, form="que", lemma="que4", left_context="SOIGNORS or escoutez",
right_context="Dés vos soit", label_uniform="que4", morph="_"),
WordToken(corpus=Floovant.id, form="Dés", lemma="dieu", left_context="or escoutez que",
right_context="vos soit amis", label_uniform="dieu", morph="NOMB.=s|GENRE=m|CAS=n"),
WordToken(corpus=Floovant.id, form="vos", lemma="vos1", left_context="escoutez que Dés",
right_context="soit amis III", label_uniform="vos1", morph="PERS.=2|NOMB.=p|GENRE=m|CAS=r"),
WordToken(corpus=Floovant.id, form="soit", lemma="estre1", left_context="que Dés vos",
right_context="amis III vers", label_uniform="estre1", morph="MODE=sub|TEMPS=pst|PERS.=3|NOMB.=s"),
def lemma_list(control_list_id):
control_list, is_owner = ControlLists.get_linked_or_404(
control_list_id=control_list_id, user=current_user)
can_edit = is_owner or current_user.is_admin()
if request.method == "DELETE" and can_edit:
value = request.args.get("id")
lemma = AllowedLemma.query.get_or_404(value)
try:
AllowedLemma.query.filter(
AllowedLemma.id == lemma.id,
AllowedLemma.control_list == control_list_id).delete()
db.session.commit()
return "", 200
except Exception as E:
db.session.rollback()
return abort(403)
elif request.method == "UPDATE" and request.mimetype == "application/json" and can_edit:
form = request.get_json().get("lemmas", None)
if not form:
return abort(400, jsonify({"message": "No lemma were passed."}))
lemmas = list(set(form.split()))
try:
AllowedLemma.add_batch(lemmas, control_list.id, _commit=True)
return jsonify({"message": "Data saved"})
except ValueError as E:
db.session.rollback()
return make_response(jsonify({"message": str(E)}), 400)
except sqlalchemy.exc.StatementError as E:
db.session.rollback()
error = str(E.orig)
if error.startswith("UNIQUE constraint failed"):
return make_response(
jsonify({
"message":
"One of the lemma you submitted already exist. "
"Remove this lemma and resubmit."
}), 400)
return make_response(
jsonify(
{"message": "Database error. Contact the administrator."}),
400)
except Exception as E:
db.session.rollback()
return make_response(jsonify({"message": "Unknown Error"}), 400)
elif request.method == "GET":
kwargs = {}
page = request.args.get("page", "1")
page = (page.isnumeric()) and int(page) or 1
limit = request.args.get("limit", "1000")
limit = (limit.isnumeric()) and int(limit) or 1
kw = strip_or_none(request.args.get("kw", ""))
template = "control_lists/read_lemma.html"
allowed_values = control_list.get_allowed_values(allowed_type="lemma",
kw=kw).paginate(
page=page,
per_page=limit)
kwargs["kw"] = kw
return render_template_with_nav_info(template=template,
control_list=control_list,
is_owner=is_owner,
allowed_type="lemma",
can_edit=is_owner
or current_user.is_admin(),
allowed_values=allowed_values,
readable=False,
**kwargs)
return abort(405)
def corpus_new():
""" Register a new corpus
"""
lemmatizers = current_app.config.get("LEMMATIZERS", [])
def normal_view():
return render_template_with_nav_info(
'main/corpus_new.html',
lemmatizers=lemmatizers,
public_control_lists=_get_available(),
tsv=request.form.get("tsv", "")
)
def error():
return normal_view(), 400
if request.method == "POST":
if not current_user.is_authenticated:
abort(403)
elif not len(strip_or_none(request.form.get("name", ""))):
flash("You forgot to give a name to your corpus", category="error")
return error()
else:
form_kwargs = {
"name": request.form.get("name"),
"context_left": request.form.get("context_left", None),
"context_right": request.form.get("context_right", None),
"delimiter_token": strip_or_none(request.form.get("sep_token", "")) or None,
"columns": [
Column(heading="Lemma"),
Column(heading="POS"),
Column(heading="Morph"),
Column(heading="Similar"),
]
}
for column in form_kwargs["columns"]:
column.hidden = bool(
request.form.get(f"{column.heading.lower()}Column", "")
)
if (
"lemmaColumn" in request.form
and "posColumn" in request.form
and "morphColumn" in request.form
):
flash(
"You can't disable Lemma and POS and Morph. Keep at least one of them.",
category="error"
)
return error()
if request.form.get("control_list") == "reuse":
tokens = read_input_tokens(request.form.get("tsv"))
try:
control_list = ControlLists.query.get_or_404(request.form.get("control_list_select"))
except Exception:
flash("This control list does not exist", category="error")
return error()
form_kwargs.update({"word_tokens_dict": tokens,
"control_list": control_list})
cl_owner = False
else:
tokens, allowed_lemma, allowed_morph, allowed_POS = create_input_format_convertion(
request.form.get("tsv"),
request.form.get("allowed_lemma", None),
request.form.get("allowed_morph", None),
request.form.get("allowed_POS", None)
)
cl_owner = True
form_kwargs.update({"word_tokens_dict": tokens, "allowed_lemma": allowed_lemma,
"allowed_POS": allowed_POS, "allowed_morph": allowed_morph})
try:
corpus = Corpus.create(**form_kwargs)
db.session.add(CorpusUser(corpus=corpus, user=current_user, is_owner=True))
# Add a link to the control list
ControlLists.link(corpus=corpus, user=current_user, is_owner=cl_owner)
db.session.commit()
flash("New corpus registered", category="success")
return redirect(url_for(".corpus_get", corpus_id=corpus.id))
except (sqlalchemy.exc.StatementError, sqlalchemy.exc.IntegrityError) as e:
db.session.rollback()
flash("The corpus cannot be registered. Check your data", category="error")
if str(e.orig) == "UNIQUE constraint failed: corpus.name":
flash("You have already a corpus going by the name {}".format(request.form.get("name")),
category="error")
return error()
except MissingTokenColumnValue as exc:
db.session.rollback()
flash("At least one line of your corpus is missing a token/form. Check line %s " % exc.line, category="error")
return error()
except NoTokensInput:
db.session.rollback()
flash("You did not input any text.", category="error")
return error()
except ValidationError as exception:
db.session.rollback()
flash(exception, category="error")
return error()
except Exception as e:
db.session.rollback()
flash("The corpus cannot be registered. Check your data", category="error")
return error()
return normal_view()
from app.models import ChangeRecord, WordToken, Corpus, ControlLists
from .base import TestModels
import copy
SimilarityFixtures = [
ControlLists(id=1, name="CL Fixture"),
Corpus(id=1, name="Fixtures !", control_lists_id=1),
WordToken(corpus=1,
form="Cil",
lemma="celui",
left_context="_",
right_context="_",
label_uniform="celui",
morph="smn",
POS="p"), # 1
WordToken(corpus=1,
form="Cil",
lemma="celle",
left_context="_",
right_context="_",
label_uniform="celle",
morph="smn",
POS="n"), # 2
WordToken(corpus=1,
form="Cil",
lemma="cil",
left_context="_",
right_context="_",
label_uniform="cil",
morph="smn",
POS="p"), # 3
