class Work(Base):
"""This object describes a single work or submission of a
:class:`user_models.User` for an :class:`.assignment_models.Assignment`.
"""
__tablename__ = "Work" # type: str
id = db.Column('id', db.Integer, primary_key=True)
assignment_id = db.Column(
'Assignment_id',
db.Integer,
db.ForeignKey('Assignment.id'),
nullable=False,
)
user_id = db.Column(
'User_id',
db.Integer,
db.ForeignKey('User.id', ondelete='CASCADE'),
nullable=False,
)
_grade = db.Column('grade', db.Float, default=None, nullable=True)
comment = orm.deferred(db.Column('comment', db.Unicode, default=None))
comment_author_id = db.Column(
'comment_author_id',
db.Integer,
db.ForeignKey('User.id', ondelete='SET NULL'),
nullable=True,
)
orm.deferred(db.Column('comment', db.Unicode, default=None))
created_at = db.Column(
db.TIMESTAMP(timezone=True),
default=DatetimeWithTimezone.utcnow,
nullable=False,
)
assigned_to = db.Column(
'assigned_to', db.Integer, db.ForeignKey('User.id'), nullable=True
)
selected_items = db.relationship(
lambda: WorkRubricItem, cascade='all, delete-orphan', uselist=True
)
assignment = db.relationship(
lambda: assignment_models.Assignment,
foreign_keys=assignment_id,
lazy='joined',
innerjoin=True,
backref=db.backref('submissions', lazy='select', uselist=True)
)
comment_author = db.relationship(
lambda: user_models.User,
foreign_keys=comment_author_id,
lazy='select'
)
user = db.relationship(
lambda: user_models.User,
foreign_keys=user_id,
lazy='joined',
innerjoin=True
)
assignee = db.relationship(
lambda: user_models.User, foreign_keys=assigned_to, lazy='joined'
)
grade_histories: t.List['GradeHistory']
_deleted = db.Column(
'deleted',
db.Boolean,
default=False,
server_default='false',
nullable=False
)
origin = db.Column(
'work_origin',
db.Enum(WorkOrigin),
nullable=False,
server_default=WorkOrigin.uploaded_files.name,
)
extra_info: ColumnProxy[JSONType] = db.Column(
'extra_info',
JSON,
nullable=True,
default=None,
)
def _get_deleted(self) -> bool:
"""Is this submission deleted.
"""
return self._deleted or not self.assignment.is_visible
@hybrid_expression
def _get_deleted_expr(cls: t.Type['Work']) -> 'DbColumn[bool]':
"""Get a query that checks if this submission is deleted.
"""
# pylint: disable=no-self-argument
return select(
[sql.or_(cls._deleted, ~assignment_models.Assignment.is_visible)]
).where(
cls.assignment_id == assignment_models.Assignment.id,
).label('deleted')
def _set_deleted(self, new_value: bool) -> None:
self._deleted = new_value
deleted = hybrid_property(
_get_deleted, _set_deleted, None, _get_deleted_expr
)
def divide_new_work(self) -> None:
"""Divide a freshly created work.
First we check if an old work of the same author exists, that case the
same grader is assigned. Otherwise we take the grader that misses the
most of work.
:returns: Nothing
"""
self.assigned_to = self.assignment.get_assignee_for_submission(self)
if self.assigned_to is not None:
self.assignment.set_graders_to_not_done(
[self.assigned_to],
send_mail=True,
ignore_errors=True,
)
@signals.WORK_CREATED.connect_immediate
def run_linter(self) -> None:
"""Run all linters for the assignment on this work.
All linters that have been used on the assignment will also run on this
work.
If the linters feature is disabled this function will simply return and
not do anything.
:returns: Nothing
"""
if not features.has_feature(features.Feature.LINTERS):
return
for linter in self.assignment.linters:
instance = LinterInstance(work=self, tester=linter)
db.session.add(instance)
if psef.linters.get_linter_by_name(linter.name).RUN_LINTER:
db.session.flush()
def _inner(name: str, config: str, lint_id: str) -> None:
lint = psef.tasks.lint_instances
psef.helpers.callback_after_this_request(
lambda: lint(name, config, [lint_id])
)
_inner(
name=linter.name,
config=linter.config,
lint_id=instance.id,
)
else:
instance.state = LinterState.done
@classmethod
def get_non_rubric_grade_per_work(
cls, assignment: 'assignment_models.Assignment'
) -> _MyQuery[t.Tuple[int, float]]:
"""Get the non rubric grades of submissions for the given assignment.
:param assignment: The assignment in which you want to get the grades.
:returns: A query that returns tuples (work_id, non_rubric_grade) for
each submission in the given assignment that has a non rubric
grade.
"""
return db.session.query(
cls.id,
# We make sure that it is not ``None`` in the filter
cast_as_non_null(cls._grade),
).filter(
cls.assignment == assignment,
cls._grade.isnot(None),
)
@classmethod
def get_rubric_grade_per_work(
cls, assignment: 'assignment_models.Assignment'
) -> _MyQuery[t.Tuple[int, float]]:
"""Get the rubric grades of submissions for the given assignment.
.. warning::
The returned grade might **not** the final grade for the
submission, as the grade might be overridden.
:param assignment: The assignment in which you want to get the rubric
grades.
:returns: A query that returns tuples (work_id, rubric_grade) for
each submission in the given assignment that has at least one
selected rubric item.
"""
if assignment.max_rubric_points is None:
# The assignment doesn't have a rubric, so simply return an empty
# query result. We filter it with `false` so it will never return
# rows.
return db.session.query(cls.id, sqlalchemy.sql.null()).filter(
sqlalchemy.sql.false()
)
max_rubric_points = assignment.max_rubric_points
min_grade = assignment.min_grade
max_grade = assignment.max_grade
least = sqlalchemy.func.least
greatest = sqlalchemy.func.greatest
return db.session.query(
cls.id,
least(
max_grade,
greatest(
min_grade, (
(sqlalchemy.func.sum(WorkRubricItem.points) * 10) /
max_rubric_points
)
)
)
).join(
# We want an inner join here as we want to filter out any Work that
# doesn't have a work rubric item associated with it.
WorkRubricItem,
WorkRubricItem.work_id == cls.id,
isouter=False
).filter(
cls.assignment == assignment,
cls._grade.is_(None),
).group_by(cls.id)
@property
def grade(self) -> t.Optional[float]:
"""Get the actual current grade for this work.
This is done by not only checking the ``grade`` field but also checking
if rubric could be found.
:returns: The current grade for this work.
"""
if self._grade is None:
if not self.selected_items:
return None
max_rubric_points = self.assignment.max_rubric_points
assert max_rubric_points is not None
selected = sum(item.points for item in self.selected_items)
return helpers.between(
self.assignment.min_grade,
selected / max_rubric_points * 10,
self.assignment.max_grade,
)
return self._grade
@t.overload
def set_grade( # pylint: disable=function-redefined,missing-docstring,unused-argument,no-self-use
self,
new_grade: t.Optional[float],
user: '******',
grade_origin: Literal[GradeOrigin.human] = GradeOrigin.human
) -> GradeHistory:
...
@t.overload
def set_grade( # pylint: disable=function-redefined,missing-docstring,unused-argument,no-self-use
self,
*,
grade_origin: GradeOrigin,
) -> GradeHistory:
...
def set_grade( # pylint: disable=function-redefined
self,
new_grade: t.Union[float, None, helpers.MissingType] = helpers.MISSING,
user: t.Optional['user_models.User'] = None,
grade_origin: GradeOrigin = GradeOrigin.human,
) -> GradeHistory:
"""Set the grade to the new grade.
.. note:: This also passes back the grade to LTI if this is necessary
(see :py:func:`passback_grade`).
.. note:: If ``grade_origin`` is ``human`` the ``user`` is required.
:param new_grade: The new grade to set
:param user: The user setting the new grade.
:param grade_origin: The way this grade was given.
:param never_passback: Never passback the new grade.
:returns: Nothing
"""
assert grade_origin != GradeOrigin.human or user is not None
if new_grade is not helpers.MISSING:
assert isinstance(new_grade, (float, int, type(None)))
self._grade = new_grade
grade = self.grade
history = GradeHistory(
is_rubric=self._grade is None and grade is not None,
grade=-1 if grade is None else grade,
passed_back=False,
work=self,
user=user,
grade_origin=grade_origin,
)
self.grade_histories.append(history)
signals.GRADE_UPDATED.send(self)
return history
@property
def selected_rubric_points(self) -> float:
"""The amount of points that are currently selected in the rubric for
this work.
"""
return sum(item.points for item in self.selected_items)
def select_rubric_items(
self,
items: t.List['WorkRubricItem'],
user: '******',
override: bool = False
) -> None:
""" Selects the given :class:`.RubricItem`.
.. note:: This also passes back the grade to LTI if this is necessary.
.. note:: This also sets the actual grade field to `None`.
.. warning::
You should do all input sanitation before calling this
function. Like checking for duplicate items and correct assignment.
:param item: The item to add.
:param user: The user selecting the item.
:returns: Nothing
"""
# Sanity checks
assert all(
item.rubric_item.rubricrow.assignment_id == self.assignment_id
for item in items
)
row_ids = [item.rubric_item.rubricrow_id for item in items]
assert len(row_ids) == len(set(row_ids))
if override:
self.selected_items = []
for item in items:
self.selected_items.append(item)
self.set_grade(None, user)
def __structlog__(self) -> t.Mapping[str, t.Union[int, str]]:
return {
'type': self.__class__.__name__,
'id': self.id,
}
def __to_json__(self) -> t.MutableMapping[str, t.Any]:
"""Returns the JSON serializable representation of this work.
The representation is based on the permissions (:class:`.Permission`)
of the logged in :class:`.user_models.User`. Namely the assignee,
feedback, and grade attributes are only included if the current user
can see them, otherwise they are set to `None`.
The resulting object will look like this:
.. code:: python
{
'id': int, # Submission id
'user': user_models.User, # User that submitted this work.
'created_at': str, # Submission date in ISO-8601 datetime
# format.
'grade': t.Optional[float], # Grade for this submission, or
# None if the submission hasn't
# been graded yet or if the
# logged in user doesn't have
# permission to see the grade.
'assignee': t.Optional[user_models.User],
# User assigned to grade this
# submission, or None if the logged
# in user doesn't have permission
# to see the assignee.
'grade_overridden': bool, # Does this submission have a
# rubric grade which has been
# overridden.
}
:returns: A dict containing JSON serializable representations of the
attributes of this work.
"""
item = {
'id': self.id,
'user': self.user,
'created_at': self.created_at.isoformat(),
'origin': self.origin.name,
'extra_info': self.extra_info,
'grade': None,
'grade_overridden': False,
'assignee': None,
}
try:
auth.ensure_permission(
CoursePermission.can_see_assignee, self.assignment.course_id
)
except PermissionException:
pass
else:
item['assignee'] = self.assignee
try:
auth.ensure_can_see_grade(self)
except PermissionException:
pass
else:
item['grade'] = self.grade
item['grade_overridden'] = (
self._grade is not None and
self.assignment.max_rubric_points is not None
)
return item
@cg_timers.timed_function(collect_in_request=True)
def __extended_to_json__(self) -> t.Mapping[str, t.Any]:
"""Create a extended JSON serializable representation of this object.
This object will look like this:
.. code:: python
{
'comment': t.Optional[str] # General feedback comment for
# this submission, or None in
# the same cases as the grade.
'comment_author': t.Optional[user_models.User] # The author of the
# comment field
# submission, or None
# if the logged in user
# doesn't have
# permission to see the
# assignee.
**self.__to_json__()
}
:returns: A object as described above.
"""
res: t.Dict[str, object] = {
'comment': None,
'comment_author': None,
'assignment_id': self.assignment.id,
'rubric_result': None,
**self.__to_json__()
}
if self.assignment.rubric_rows:
res['rubric_result'] = self.__rubric_to_json__()
if auth.WorkPermissions(self
).ensure_may_see_general_feedback.as_bool():
res['comment'] = self.comment
if psef.current_user.has_permission(
CoursePermission.can_view_feedback_author,
self.assignment.course_id
):
res['comment_author'] = self.comment_author
return res
def __rubric_to_json__(self) -> t.Mapping[str, t.Any]:
"""Converts a rubric of a work to a object that is JSON serializable.
The resulting object will look like this:
.. code:: python
{
'rubrics': t.List[RubricRow] # A list of all the rubrics for
# this work.
'selected': t.List[RubricItem] # A list of all the selected
# rubric items for this work,
# or an empty list if the logged
# in user doesn't have
# permission to see the rubric.
'points': {
'max': t.Optional[float] # The maximal amount of points
# for this rubric, or `None` if
# logged in user doesn't have
# permission to see the rubric.
'selected': t.Optional[float] # The amount of point that
# is selected for this
# work, or `None` if the
# logged in user doesn't
# have permission to see
# the rubric.
}
}
:returns: A object as described above.
.. todo:: Remove the points object.
"""
res = {
'rubrics': self.assignment.rubric_rows,
'selected': [],
'points': {
'max': None,
'selected': None,
},
}
try:
psef.auth.ensure_can_see_grade(self)
except PermissionException:
pass
else:
res['selected'] = self.selected_items
res['points'] = {
'max': self.assignment.max_rubric_points,
'selected': self.selected_rubric_points,
}
return res
def add_file_tree(
self, tree: 'psef.files.ExtractFileTreeDirectory'
) -> None:
"""Add the given tree to as only files to the current work.
.. warning:: All previous files will be unlinked from this assignment.
:param tree: The file tree as described by
:py:func:`psef.files.rename_directory_structure`
:returns: Nothing
"""
db.session.add(
file_models.File.create_from_extract_directory(
tree, None, {'work': self}
)
)
def get_user_feedback(self) -> t.Iterable[str]:
"""Get all user given feedback for this work.
:returns: An iterator producing human readable representations of the
feedback given by a person.
"""
comments = CommentBase.query.filter(
CommentBase.file.has(work=self),
).order_by(
CommentBase.file_id.asc(),
CommentBase.line.asc(),
)
for com in comments:
path = com.file.get_path()
line = com.line + 1
for idx, reply in enumerate(com.user_visible_replies):
yield f'{path}:{line}:{idx + 1}: {reply.comment}'
def get_linter_feedback(self) -> t.Iterable[str]:
"""Get all linter feedback for this work.
:returns: An iterator that produces the all feedback given on this work
by linters.
"""
linter_comments = LinterComment.query.filter(
LinterComment.file.has(work=self)
).order_by(
LinterComment.file_id.asc(),
LinterComment.line.asc(),
)
for line_comm in linter_comments:
yield (
f'{line_comm.file.get_path()}:{line_comm.line + 1}:1: '
f'({line_comm.linter.tester.name}'
f' {line_comm.linter_code}) {line_comm.comment}'
)
def remove_selected_rubric_item(self, row_id: int) -> None:
"""Deselect selected :class:`.RubricItem` on row.
Deselects the selected rubric item on the given row with _row_id_ (if
there are any selected).
:param row_id: The id of the RubricRow from which to deselect
rubric items
:returns: Nothing
"""
rubricitem = db.session.query(WorkRubricItem).join(
RubricItem, RubricItem.id == WorkRubricItem.rubricitem_id
).filter(
WorkRubricItem.work_id == self.id,
RubricItem.rubricrow_id == row_id
).first()
if rubricitem is not None:
self.selected_items.remove(rubricitem)
def search_file_filters(
self,
pathname: str,
exclude: 'file_models.FileOwner',
) -> t.List[DbColumn[bool]]:
"""Get the filters needed to search for a file in the this directory
with a given name.
:param pathname: The path of the file to search for, this may contain
leading and trailing slashes which do not have any meaning.
:param exclude: The fileowner to exclude from search, like described in
:func:`get_zip`.
:returns: The criteria needed to find the file with the given pathname.
"""
patharr, is_dir = psef.files.split_path(pathname)
parent: t.Optional[t.Any] = None
for idx, pathpart in enumerate(patharr[:-1]):
if parent is not None:
parent = parent.c.id
parent = db.session.query(
t.cast(DbColumn[int], file_models.File.id)
).filter(
file_models.File.name == pathpart,
file_models.File.parent_id == parent,
file_models.File.work_id == self.id,
file_models.File.is_directory,
~file_models.File.self_deleted,
).subquery(f'parent_{idx}')
if parent is not None:
parent = parent.c.id
return [
file_models.File.work_id == self.id,
file_models.File.name == patharr[-1],
file_models.File.parent_id == parent,
file_models.File.fileowner != exclude,
file_models.File.is_directory == is_dir,
~file_models.File.self_deleted,
]
def search_file(
self,
pathname: str,
exclude: 'file_models.FileOwner',
) -> 'file_models.File':
"""Search for a file in the this directory with the given name.
:param pathname: The path of the file to search for, this may contain
leading and trailing slashes which do not have any meaning.
:param exclude: The fileowner to exclude from search, like described in
:func:`get_zip`.
:returns: The found file.
"""
return psef.helpers.filter_single_or_404(
file_models.File,
*self.search_file_filters(pathname, exclude),
)
def get_file_children_mapping(
self, exclude: 'file_models.FileOwner'
) -> t.Mapping[t.Optional[int], t.Sequence['file_models.File']]:
"""Get a mapping that maps a file id to all its children.
This implementation does a single query to the database and runs in
O(n*log(n)), so it will be quite a bit quicker than using the
`children` attribute on files if you are going to need all children or
all files.
The list of children is sorted on filename.
:param exclude: The file owners to exclude
:returns: A mapping from file id to list of all its children for this
submission.
"""
cache: t.Mapping[t.Optional[int], t.
List['file_models.File']] = defaultdict(list)
files = file_models.File.query.filter(
file_models.File.work == self,
file_models.File.fileowner != exclude,
~file_models.File.self_deleted,
).all()
# We sort in Python as this increases consistency between different
# server platforms, Python also has better defaults.
# TODO: Investigate if sorting in the database first and sorting in
# Python after is faster, as sorting in the database should be faster
# overal and sorting an already sorted list in Python is really fast.
files.sort(key=lambda el: el.name.lower())
for f in files:
cache[f.parent_id].append(f)
return cache
@staticmethod
def limit_to_user_submissions(
query: _MyQuery['Work'], user: '******'
) -> _MyQuery['Work']:
"""Limit the given query of submissions to only submission submitted by
the given user.
.. note::
This is not the same as filtering on the author field as this also
checks for groups.
:param query: The query to limit.
:param user: The user to filter for.
:returns: The filtered query.
"""
# This query could be improved, but it seems fast enough. It now gets
# every group of a user. This could be narrowed down probably.
groups_of_user = group_models.Group.contains_users(
[user]
).with_entities(
t.cast(DbColumn[int], group_models.Group.virtual_user_id)
)
return query.filter(
sql.or_(
Work.user_id == user.id,
t.cast(DbColumn[int], Work.user_id).in_(groups_of_user)
)
)
def get_all_authors(self) -> t.List['user_models.User']:
"""Get all the authors of this submission.
:returns: A list of users that were the authors of this submission.
"""
if self.user.group:
return list(self.user.group.members)
else:
return [self.user]
def has_as_author(self, user: '******') -> bool:
"""Check if the given user is (one of) the authors of this submission.
:param user: The user to check for.
:returns: ``True`` if the user is the author of this submission or a
member of the group that is the author of this submission.
"""
return self.user.contains_user(user)
def create_zip(
self,
exclude_owner: 'file_models.FileOwner',
create_leading_directory: bool = True
) -> str:
"""Create zip in `MIRROR_UPLOADS` directory.
:param exclude_owner: Which files to exclude.
:returns: The name of the zip file in the `MIRROR_UPLOADS` dir.
"""
path, name = psef.files.random_file_path(True)
with open(
path,
'w+b',
) as f, tempfile.TemporaryDirectory(
suffix='dir',
) as tmpdir, zipfile.ZipFile(
f,
'w',
compression=zipfile.ZIP_DEFLATED,
) as zipf:
# Restore the files to tmpdir
tree_root = psef.files.restore_directory_structure(
self, tmpdir, exclude_owner
)
if create_leading_directory:
zipf.write(tmpdir, tree_root.name)
leading_len = len(tmpdir)
else:
leading_len = len(tmpdir) + len('/') + len(tree_root.name)
for root, _dirs, files in os.walk(tmpdir):
for file in files:
path = psef.files.safe_join(root, file)
zipf.write(path, path[leading_len:])
return name
@classmethod
def create_from_tree(
cls,
assignment: 'assignment_models.Assignment',
author: 'user_models.User',
tree: psef.extract_tree.ExtractFileTree,
*,
created_at: t.Optional[DatetimeWithTimezone] = None,
) -> 'Work':
"""Create a submission from a file tree.
.. warning::
This function **does not** check if the author has permission to
create a submission, so this is the responsibility of the caller!
:param assignment: The assignment in which the submission should be
created.
:param author: The author of the submission.
:param tree: The tree that are the files of the submission.
:param created_at: At what time was this submission created, defaults
to the current time.
:returns: The created work.
"""
# TODO: Check why we need to pass both user_id and user.
self = cls(assignment=assignment, user_id=author.id, user=author)
if created_at is None:
self.created_at = helpers.get_request_start_time()
else:
self.created_at = created_at
self.divide_new_work()
self.add_file_tree(tree)
db.session.add(self)
db.session.flush()
signals.WORK_CREATED.send(self)
return self
@classmethod
def update_query_for_extended_jsonify(
cls: t.Type['Work'], query: _MyQuery['Work']
) -> _MyQuery['Work']:
"""Update the given query to load all attributes needed for an extended
jsonify eagerly.
:param query: The query to update.
:returns: The updated query, which now loads all attributes needed for
an extended jsonify eagerly.
"""
return query.options(
selectinload(
cls.selected_items,
),
# We want to load all users directly. We do this by loading the
# user, which might be a group. For such groups we also load all
# users. The users in this group will never be a group, so the
# last `selectinload` here might be seen as strange. However,
# during the serialization of a group we access `User.group`, which
# checks if a user is really not a group. To prevent these last
# queries the last `selectinload` is needed here.
selectinload(
cls.user,
).selectinload(
user_models.User.group,
).selectinload(
group_models.Group.members,
).selectinload(
user_models.User.group,
),
undefer(cls.comment),
selectinload(cls.comment_author),
)
class Permission(Base, t.Generic[_T]): # pylint: disable=unsubscriptable-object
"""This class defines **database** permissions.
A permission can be a global- or a course- permission. Global permissions
describe the ability to do something general, e.g. create a course or the
usage of snippets. These permissions are connected to a :class:`.Role`
which is hold be a :class:`.User`. Similarly course permissions are bound
to a :class:`.CourseRole`. These roles are assigned to users only in the
context of a single :class:`.Course`. Thus a user can hold different
permissions in different courses.
.. warning::
Think twice about using this class directly! You probably want a non
database permission (see ``permissions.py``) which are type checked and
WAY faster. If you need to check if a user has a certain permission use
the :meth:`.User.has_permission` of, even better,
:func:`psef.auth.ensure_permission` functions.
:ivar default_value: The default value for this permission.
:ivar course_permission: Indicates if this permission is for course
specific actions. If this is the case a user can have this permission
for a subset of all the courses. If ``course_permission`` is ``False``
this permission is global for the entire site.
"""
__tablename__ = 'Permission'
id = db.Column('id', db.Integer, primary_key=True)
__name = db.Column(
'name', db.Unicode, unique=True, index=True, nullable=False
)
default_value = db.Column(
'default_value', db.Boolean, default=False, nullable=False
)
course_permission = db.Column(
'course_permission', db.Boolean, index=True, nullable=False
)
@classmethod
def get_name_column(cls: t.Type['Permission[_T]']) -> DbColumn[str]:
"""Get the name column in the database for the permissions.
:returns: The name column of permissions.
"""
return t.cast(DbColumn[str], cls.__name)
@classmethod
def get_all_permissions(
cls: t.Type['Permission[_T]'], perm_type: t.Type[_T]
) -> 't.Sequence[Permission[_T]]':
"""Get all database permissions of a certain type.
:param perm_type: The type of permission to get.
:returns: A list of all database permissions of the given type.
"""
assert perm_type in (GlobalPermission, CoursePermission)
return db.session.query(cls).filter_by( # type: ignore
course_permission=perm_type == CoursePermission
).all()
@classmethod
def get_all_permissions_from_list(
cls: t.Type['Permission[_T]'], perms: t.Sequence[_T]
) -> 't.Sequence[Permission[_T]]':
"""Get database permissions corresponding to a list of permissions.
:param perms: The permissions to get the database permission of.
:returns: A list of all requested database permission.
"""
if not perms: # pragma: no cover
return []
assert isinstance(perms[0], (GlobalPermission, CoursePermission))
assert all(isinstance(perm, type(perms[0])) for perm in perms)
return helpers.filter_all_or_404(
cls,
t.cast(DbColumn[str],
Permission.__name).in_([p.name for p in perms]),
Permission.course_permission == isinstance(
perms[0], CoursePermission
),
)
@classmethod
@cache_within_request
def get_permission(
cls: 't.Type[Permission[_T]]', perm: '_T'
) -> 'Permission[_T]':
"""Get a database permission from a permission.
:param perm: The permission to get the database permission of.
:returns: The correct database permission.
"""
return helpers.filter_single_or_404(
cls,
cls.value == perm,
cls.course_permission == isinstance(perm, CoursePermission),
)
def _get_value(self) -> '_T':
"""Get the permission value of the database permission.
:returns: The permission of this database permission.
"""
try:
if self.course_permission:
return t.cast('_T', CoursePermission[self.__name])
else:
return t.cast('_T', GlobalPermission[self.__name])
except KeyError: # pragma: no cover
# We might have old permissions still in the database
return t.cast('_T', UnknownPermission(self.__name))
@hybrid_expression
def _get_value_comp(cls: t.Type['Permission[_T]']) -> PermissionComp[_T]: # pylint: disable=no-self-argument,missing-docstring
return PermissionComp(cls.__name)
value = hybrid_property(_get_value, custom_comparator=_get_value_comp)
class User(NotEqualMixin, Base):
"""This class describes a user of the system.
>>> u1 = User('', '', '', '')
>>> u1.id = 5
>>> u1.id
5
>>> u1.id = 6
Traceback (most recent call last):
...
AssertionError
:ivar ~.User.lti_user_id: The id of this user in a LTI consumer.
:ivar ~.User.name: The name of this user.
:ivar ~.User.role_id: The id of the role this user has.
:ivar ~.User.courses: A mapping between course_id and course-role for all
courses this user is currently enrolled.
:ivar ~.User.email: The e-mail of this user.
:ivar ~.User.virtual: Is this user an actual user of the site, or is it a
virtual user.
:ivar ~.User.password: The password of this user, it is automatically
hashed.
:ivar ~.User.assignment_results: The way this user can do LTI grade
passback.
:ivar ~.User.assignments_assigned: A mapping between assignment_ids and
:py:class:`.AssignmentAssignedGrader` objects.
:ivar reset_email_on_lti: Determines if the email should be reset on the
next LTI launch.
"""
@classmethod
def resolve(cls: t.Type['User'],
possible_user: t.Union['User', LocalProxy]) -> 'User':
"""Unwrap the possible local proxy to a user.
:param possible_user: The user we should unwrap.
:returns: If the given argument was a LocalProxy
`_get_current_object()` is called and the return value is returned,
otherwise the given argument is returned.
:raises AssertionError: If the given argument was not a user after
unwrapping.
"""
return maybe_unwrap_proxy(possible_user, cls, check=True)
__tablename__ = "User"
_id = db.Column('id', db.Integer, primary_key=True)
def __init__(
self,
name: str,
email: str,
password: t.Optional[str],
username: str,
active: Literal[True] = True,
virtual: bool = False,
role: t.Optional[Role] = None,
is_test_student: bool = False,
courses: t.Mapping[int, CourseRole] = None,
) -> None:
super().__init__(
name=name,
email=email,
password=password,
username=username,
active=active,
role=role,
is_test_student=is_test_student,
virtual=virtual,
courses=handle_none(courses, {}),
)
def _get_id(self) -> int:
"""The id of the user
"""
return self._id
def _set_id(self, n_id: int) -> None:
assert not hasattr(self, '_id') or self._id is None
self._id = n_id
id = hybrid_property(_get_id, _set_id)
name = db.Column('name', db.Unicode, nullable=False)
active = db.Column('active', db.Boolean, default=True, nullable=False)
virtual = db.Column('virtual',
db.Boolean,
default=False,
nullable=False,
index=True)
is_test_student = db.Column('is_test_student',
db.Boolean,
default=False,
nullable=False,
index=True)
role_id = db.Column('Role_id', db.Integer, db.ForeignKey('Role.id'))
courses: t.MutableMapping[int, CourseRole] = db.relationship(
'CourseRole',
collection_class=attribute_mapped_collection('course_id'),
secondary=user_course,
backref=db.backref('users', lazy='dynamic'))
_username = db.Column(
'username',
CIText,
unique=True,
nullable=False,
index=True,
)
def get_readable_name(self) -> str:
"""Get the readable name of this user.
:returns: If this is a normal user this method simply returns the name
of the user. If this user is the virtual user of a group a nicely
formatted group name is returned.
"""
if self.group:
return f'group "{self.group.name}"'
else:
return self.name
def _get_username(self) -> str:
"""The username of the user
"""
return self._username
def _set_username(self, username: str) -> None:
assert not hasattr(self, '_username') or self._username is None
self._username = username
username = hybrid_property(_get_username, _set_username)
reset_token = db.Column('reset_token',
db.String(UUID_LENGTH),
nullable=True)
reset_email_on_lti = db.Column(
'reset_email_on_lti',
db.Boolean,
server_default=false(),
default=False,
nullable=False,
)
email = db.Column('email', db.Unicode, unique=False, nullable=False)
password = db.Column(
'password',
PasswordType(schemes=[
'pbkdf2_sha512',
], deprecated=[]),
nullable=True,
)
assignments_assigned: t.MutableMapping[
int, 'AssignmentAssignedGrader'] = db.relationship(
'AssignmentAssignedGrader',
collection_class=attribute_mapped_collection('assignment_id'),
backref=db.backref('user', lazy='select'))
assignment_results: t.MutableMapping[
int, 'AssignmentResult'] = db.relationship(
'AssignmentResult',
collection_class=attribute_mapped_collection('assignment_id'),
backref=db.backref('user', lazy='select'))
group = db.relationship(
lambda: psef.models.Group,
back_populates='virtual_user',
lazy='selectin',
uselist=False,
)
role = db.relationship(lambda: Role, foreign_keys=role_id, lazy='select')
def __structlog__(self) -> t.Mapping[str, t.Union[str, int]]:
return {
'type': self.__class__.__name__,
'id': self.id,
}
def __eq__(self, other: object) -> bool:
if not isinstance(other, User):
return NotImplemented
return self.id == other.id
def __lt__(self, other: 'User') -> bool:
return self.username < other.username
def __hash__(self) -> int:
return hash(self.id)
def is_enrolled(self, course: t.Union[int,
'course_models.Course']) -> bool:
"""Check if a user is enrolled in the given course.
:param course: The course to in which the user might be enrolled. This
can also be a course id for efficiency purposes (so you don't have
to load the entire course object).
:returns: If the user is enrolled in the given course. This is always
``False`` if this user is virtual.
"""
if self.virtual:
return False
course_id = (course.id
if isinstance(course, course_models.Course) else course)
return course_id in self.courses
def enroll_in_course(self, *, course_role: CourseRole) -> None:
"""Enroll this user in a course with the given ``course_role``.
:param course_role: The role the user should get in the new course.
This object already contains the information about the course, so
the user will be enrolled in the course connected to this role.
:returns: Nothing.
:raises AssertionError: If the user is already enrolled in the course.
"""
assert not self.is_enrolled(
course_role.course_id,
), 'User is already enrolled in the given course'
self.courses[course_role.course_id] = course_role
signals.USER_ADDED_TO_COURSE.send(
signals.UserToCourseData(user=self, course_role=course_role))
def contains_user(self, possible_member: 'User') -> bool:
"""Check if given user is part of this user.
A user ``A`` is part of a user ``B`` if either ``A == B`` or
``B.is_group and B.group.has_as_member(A)``
:param possible_member: The user to check for if it is part of
``self``.
:return: A bool indicating if ``self`` contains ``possible_member``.
"""
if self.group is None:
return self == possible_member
else:
return self.group.has_as_member(possible_member)
def get_contained_users(self) -> t.Sequence['User']:
"""Get all contained users of this user.
:returns: If this user is the virtual user of this group a list of
members of the group, otherwise the user itself is wrapped in a
list and returned.
"""
if self.group is None:
return [self]
return self.group.members
@classmethod
def create_new_test_student(cls) -> 'User':
"""Create a new test student.
:return: A newly created test student user named
'TEST_STUDENT' followed by a random string.
"""
return cls(
name='Test Student',
username=f'TEST_STUDENT__{uuid.uuid4()}',
is_test_student=True,
email='',
password=None,
)
@classmethod
def create_virtual_user(cls: t.Type['User'], name: str) -> 'User':
"""Create a virtual user with the given name.
:return: A newly created virtual user with the given name prepended
with 'Virtual - ' and a random username.
"""
return cls(
name=f'Virtual - {name}',
username=f'VIRTUAL_USER__{uuid.uuid4()}',
virtual=True,
email='',
password=None,
)
@t.overload
def has_permission( # pylint: disable=function-redefined,missing-docstring,unused-argument,no-self-use
self, permission: CoursePermission,
course_id: t.Union[int, 'course_models.Course']) -> bool:
... # pylint: disable=pointless-statement
@t.overload
def has_permission( # pylint: disable=function-redefined,missing-docstring,unused-argument,no-self-use
self,
permission: GlobalPermission,
) -> bool:
... # pylint: disable=pointless-statement
def has_permission( # pylint: disable=function-redefined
self,
permission: t.Union[GlobalPermission, CoursePermission],
course_id: t.Union['course_models.Course', int,
None] = None) -> bool:
"""Check whether this user has the specified global or course
:class:`.Permission`.
To check a course permission the course_id has to be set.
:param permission: The permission or permission name
:param course_id: The course or course id
:returns: Whether the role has the permission or not
:raises KeyError: If the permission parameter is a string and no
permission with this name exists.
"""
if not self.active or self.virtual or self.is_test_student:
return False
if course_id is None:
assert isinstance(permission, GlobalPermission)
if self.role is None:
return False
return self.role.has_permission(permission)
else:
assert isinstance(permission, CoursePermission)
if isinstance(course_id, course_models.Course):
course_id = course_id.id
if course_id in self.courses:
return self.courses[course_id].has_permission(permission)
return False
def get_all_permissions_in_courses(
self, ) -> t.Mapping[int, t.Mapping[CoursePermission, bool]]:
"""Get all permissions for all courses the current user is enrolled in
:returns: A mapping from course id to a mapping from
:py:class:`.CoursePermission` to a boolean indicating if the
current user has this permission.
"""
permission_links = db.session.query(
user_course.c.course_id, Permission.get_name_column()).join(
User, User.id == user_course.c.user_id
).filter(user_course.c.user_id == self.id).join(
course_permissions, course_permissions.c.course_role_id ==
user_course.c.course_id).join(
Permission,
course_permissions.c.permission_id == Permission.id,
isouter=True)
lookup: t.Mapping[int, t.Set[str]] = defaultdict(set)
for course_role_id, perm_name in permission_links:
lookup[course_role_id].add(perm_name)
out: t.MutableMapping[int, t.Mapping[CoursePermission, bool]] = {}
for course_id, course_role in self.courses.items():
perms = lookup[course_role.id]
out[course_id] = {
p: (p.name in perms) ^ p.value.default_value
for p in CoursePermission
}
return out
def get_permissions_in_courses(
self,
wanted_perms: t.Sequence[CoursePermission],
) -> t.Mapping[int, t.Mapping[CoursePermission, bool]]:
"""Check for specific :class:`.Permission`s in all courses
(:class:`.course_models.Course`) the user is enrolled in.
Please note that passing an empty ``perms`` object is
supported. However the resulting mapping will be empty.
>>> User('', '', '', '').get_permissions_in_courses([])
{}
:param wanted_perms: The permissions names to check for.
:returns: A mapping where the first keys indicate the course id,
the values at this are a mapping between the given permission names
and a boolean indicating if the current user has this permission
for the course with this course id.
"""
assert not self.virtual
if not wanted_perms:
return {}
perms: t.Sequence[Permission[CoursePermission]]
perms = Permission.get_all_permissions_from_list(wanted_perms)
course_roles = db.session.query(user_course.c.course_id).join(
User, User.id == user_course.c.user_id).filter(
User.id == self.id).subquery('course_roles')
crp = db.session.query(
course_permissions.c.course_role_id,
t.cast(DbColumn[int], Permission.id),
).join(
Permission,
course_permissions.c.permission_id == Permission.id,
).filter(
t.cast(DbColumn[int],
Permission.id).in_([p.id for p in perms])).subquery('crp')
res: t.Sequence[t.Tuple[int, int]]
res = db.session.query(course_roles.c.course_id, crp.c.id).join(
crp,
course_roles.c.course_id == crp.c.course_role_id,
isouter=False,
).all()
lookup: t.Mapping[int, t.Set[int]] = defaultdict(set)
for course_role_id, permission_id in res:
lookup[permission_id].add(course_role_id)
out: t.MutableMapping[int, t.Mapping[CoursePermission, bool]] = {}
for course_id, course_role in self.courses.items():
out[course_id] = {
p.value: (course_role.id in lookup[p.id]) != p.default_value
for p in perms
}
return out
@property
def can_see_hidden(self) -> bool:
"""Can the user see hidden assignments.
"""
return self.has_course_permission_once(
CoursePermission.can_see_hidden_assignments)
def __to_json__(self) -> t.Dict[str, t.Any]:
"""Creates a JSON serializable representation of this object.
This object will look like this:
.. code:: python
{
'id': int, # The id of this user.
'name': str, # The full name of this user.
'username': str, # The username of this user.
'group': t.Optional[Group], # The group that this user
# represents.
}
:returns: An object as described above.
"""
return {
'id': self.id,
'name': self.name,
'username': self.username,
'group': self.group,
'is_test_student': self.is_test_student,
}
def __extended_to_json__(self) -> t.MutableMapping[str, t.Any]:
"""Create a extended JSON serializable representation of this object.
This object will look like this:
.. code:: python
{
'email': str, # The email of this user.
'hidden': bool, # indicating if this user can once
# see hidden assignments.
**self.__to_json__()
}
:returns: A object as described above.
"""
is_self = psef.current_user and psef.current_user.id == self.id
return {
'email': self.email if is_self else '<REDACTED>',
"hidden": self.can_see_hidden,
**self.__to_json__(),
}
def has_course_permission_once(self, perm: CoursePermission) -> bool:
"""Check whether this user has the specified course
:class:`.Permission` in at least one enrolled
:class:`.course_models.Course`.
:param perm: The permission or permission name
:returns: True if the user has the permission once
"""
assert not self.virtual
permission = Permission.get_permission(perm)
assert permission.course_permission
course_roles = db.session.query(user_course.c.course_id).join(
User, User.id == user_course.c.user_id).filter(
User.id == self.id).subquery('course_roles')
crp = db.session.query(course_permissions.c.course_role_id).join(
Permission,
course_permissions.c.permission_id == Permission.id).filter(
Permission.id == permission.id).subquery('crp')
res = db.session.query(course_roles.c.course_id).join(
crp, course_roles.c.course_id == crp.c.course_role_id)
link = db.session.query(res.exists()).scalar()
return link ^ permission.default_value
@t.overload
def get_all_permissions(self) -> t.Mapping[GlobalPermission, bool]: # pylint: disable=function-redefined,missing-docstring,no-self-use
... # pylint: disable=pointless-statement
@t.overload
def get_all_permissions( # pylint: disable=function-redefined,missing-docstring,no-self-use,unused-argument
self,
course_id: t.Union['course_models.Course', int],
) -> t.Mapping[CoursePermission, bool]:
... # pylint: disable=pointless-statement
def get_all_permissions( # pylint: disable=function-redefined
self,
course_id: t.Union['course_models.Course', int, None] = None
) -> t.Union[t.Mapping[CoursePermission, bool], t.Mapping[GlobalPermission,
bool]]:
"""Get all global permissions (:class:`.Permission`) of this user or
all course permissions of the user in a specific
:class:`.course_models.Course`.
:param course_id: The course or course id
:returns: A name boolean mapping where the name is the name of the
permission and the value indicates if this user has this
permission.
"""
assert not self.virtual
if isinstance(course_id, course_models.Course):
course_id = course_id.id
if course_id is None:
if self.role is None:
return {perm: False for perm in GlobalPermission}
else:
return self.role.get_all_permissions()
else:
if course_id in self.courses:
return self.courses[course_id].get_all_permissions()
else:
return {perm: False for perm in CoursePermission}
def get_reset_token(self) -> str:
"""Get a token which a user can use to reset his password.
:returns: A token that can be used in :py:meth:`User.reset_password` to
reset the password of a user.
"""
timed_serializer = URLSafeTimedSerializer(
current_app.config['SECRET_KEY'])
self.reset_token = str(uuid.uuid4())
return str(timed_serializer.dumps(self.username,
salt=self.reset_token))
def reset_password(self, token: str, new_password: str) -> None:
"""Reset a users password by using a token.
.. note:: Don't forget to commit the database.
:param token: A token as generated by :py:meth:`User.get_reset_token`.
:param new_password: The new password to set.
:returns: Nothing.
:raises PermissionException: If something was wrong with the
given token.
"""
assert not self.virtual
timed_serializer = URLSafeTimedSerializer(
current_app.config['SECRET_KEY'])
try:
username = timed_serializer.loads(
token,
max_age=current_app.config['RESET_TOKEN_TIME'],
salt=self.reset_token)
except BadSignature:
logger.warning(
'Invalid password reset token encountered',
token=token,
exc_info=True,
)
raise PermissionException(
'The given token is not valid',
f'The given token {token} is not valid.',
APICodes.INVALID_CREDENTIALS, 403)
# This should never happen but better safe than sorry.
if (username != self.username
or self.reset_token is None): # pragma: no cover
raise PermissionException(
'The given token is not valid for this user',
f'The given token {token} is not valid for user "{self.id}".',
APICodes.INVALID_CREDENTIALS, 403)
self.password = new_password
self.reset_token = None
@property
def is_global_admin_user(self) -> bool:
"""Is this the global administrator of the site.
This can only ever be ``True`` for users flushed to the database.
"""
if self.id is None:
return False # type: ignore[unreachable]
global_admin_username = psef.app.config['ADMIN_USER']
return (bool(global_admin_username)
and (self.username == global_admin_username))
@property
def is_active(self) -> bool:
"""Is the current user an active user.
.. todo::
Remove this property
:returns: If the user is active.
"""
return self.active
@classmethod
def register_new_user(cls, *, username: str, password: str, email: str,
name: str) -> 'User':
"""Register a new user with the given data.
:param username: The username of the new user, if a user already exists
with this username an :class:`.APIException` is raised.
:param password: The password of the new user, if the password is not
strong enough an :class:`.APIException` is raised.
:param email: The email of the new user, if not valid an
:class:`.APIException` is raised.
:name: The name of the new user.
:returns: The created user, already added (but not committed) to the
database.
"""
if not all([username, email, name]):
raise APIException(
'All fields should contain at least one character',
('The lengths of the given password, username and '
'email were not all larger than 1'),
APICodes.INVALID_PARAM,
400,
)
validate.ensure_valid_password(password,
username=username,
email=email,
name=name)
validate.ensure_valid_email(email)
if db.session.query(
cls.query.filter_by(username=username).exists()).scalar():
raise APIException(
'The given username is already in use',
f'The username "{username}" is taken',
APICodes.OBJECT_ALREADY_EXISTS,
400,
)
role = Role.query.filter_by(
name=current_app.config['DEFAULT_ROLE']).one()
self = cls(
username=username,
password=password,
email=email,
name=name,
role=role,
active=True,
)
db.session.add(self)
db.session.flush()
return self
class Job(Base, mixins.TimestampMixin, mixins.IdMixin):
"""This class represents a single job.
A job is something a CodeGrade instance needs a runner for. These jobs are
never deleted, but its state changes during its life.
"""
__tablename__ = 'job'
_state = db.Column(
'state',
db.Enum(JobState),
nullable=False,
# We constantly filter on the state, so it makes to have an index on
# this column.
index=True,
default=JobState.waiting_for_runner)
cg_url = db.Column('cg_url', db.Unicode, nullable=False)
# It is important that this is really unique!
remote_id = db.Column('remote_id',
db.Unicode,
nullable=False,
index=True,
unique=True)
runners = db.relationship(Runner,
back_populates='job',
uselist=True,
lazy='selectin')
job_metadata = db.Column('job_metadata', JSONB, nullable=True, default={})
def update_metadata(self, new_values: t.Dict[str, object]) -> None:
self.job_metadata = {**(self.job_metadata or {}), **new_values}
_wanted_runners = db.Column('wanted_runners',
db.Integer,
default=1,
server_default='1',
nullable=False)
def _get_wanted_runners(self) -> int:
"""The amount of runners this job wants.
"""
return self._wanted_runners
def _set_wanted_runners(self, new_value: int) -> None:
self._wanted_runners = min(
max(new_value, 1),
app.config['MAX_AMOUNT_OF_RUNNERS_PER_JOB'],
)
wanted_runners = hybrid_property(_get_wanted_runners, _set_wanted_runners)
def _get_state(self) -> JobState:
"""Get the state of this job.
"""
return self._state
def _set_state(self, new_state: JobState) -> None:
if new_state < self.state:
raise ValueError('Cannot decrease the state!', self.state,
new_state)
self._state = new_state
state = hybrid_property(_get_state, _set_state)
def __init__(
self,
remote_id: str,
cg_url: str,
state: JobState = JobState.waiting_for_runner,
) -> None:
super().__init__(
remote_id=remote_id,
cg_url=cg_url,
_state=state,
)
def get_active_runners(self) -> t.List[Runner]:
return [
r for r in self.runners
if r.state in RunnerState.get_active_states()
]
def __log__(self) -> t.Dict[str, object]:
return {
'state': self.state and self.state.name,
'cg_url': self.cg_url,
'id': self.id,
}
@staticmethod
def _can_steal_runner(runner: Runner) -> bool:
"""Can this job steal the given runner.
:param runner: The runner we might want to steal.
:returns: If we can steal this runner.
"""
if runner.job is None:
return True
# TODO: We might want to check not only if we can steal this runner,
# but also if we need this runner more than this other job. For example
# if we want 5 runners but have 4, and this other job wants 10 runners
# but has 1, we might not want to steal this runner from that other
# job.
return any(r != runner for r in runner.job.get_active_runners())
def maybe_use_runner(self, runner: Runner) -> bool:
"""Maybe use the given ``runner`` for this job.
This function checks if this job is allowed to use the given
runner. This is the case if the runner is assigned to us, if it is
unassigned or we can steal it from another job.
:param runner: The runner we might want to use.
:returns: ``True`` if we can use the runner, and ``False`` if we
cannot.
"""
if runner in self.runners:
return True
active_runners = self.get_active_runners()
before_active = set(RunnerState.get_before_running_states())
# In this case we have enough running runners for this job.
if sum(r.state not in before_active
for r in active_runners) >= self.wanted_runners:
logger.info('Too many runners assigned to job')
return False
# We want more, but this should only be given if the runner is not
# needed elsewhere.
# Runner is unassigned, so get it.
if runner.job is None:
self.runners.append(runner)
# However, we might assume that this runner can be used for other
# jobs, so we might need to start more runners.
callback_after_this_request(
cg_broker.tasks.maybe_start_more_runners.delay)
return True
# Runner is assigned but we maybe can steal it.
if self._can_steal_runner(runner):
logger.info(
'Stealing runner from job',
new_job_id=self.id,
other_job_id=runner.job.id,
runner=runner,
)
self.runners.append(runner)
unneeded_runner = next(
(r for r in active_runners if r.state in before_active), None)
too_many_active = len(active_runners) + 1 > self.wanted_runners
if too_many_active and unneeded_runner:
# In this case we now have too many runners assigned to us, so
# make one of the runners unassigned. But only do this if we
# have a runner which isn't already running.
unneeded_runner.make_unassigned()
# The runner we stole might be useful for the other job, as it
# might have requested extra runners. So we might want to start
# extra runners.
callback_after_this_request(
cg_broker.tasks.maybe_start_more_runners.delay)
return True
return False
def __to_json__(self) -> t.Dict[str, object]:
return {
'id': self.remote_id,
'state': self.state.name,
'wanted_runners': self.wanted_runners,
}
def add_runners_to_job(self, unassigned_runners: t.List[Runner],
startable: int) -> int:
"""Add runners to the given job.
This adds runners from the ``unassigned_runners`` list to the current
job, or starts new runners as long as ``startable`` is greater than 0.
.. note:: The ``unassigned_runners`` list may be mutated in place.
:param unassigned_runners: Runners that are not assigned yet and can be
used by this job.
:param startable: The amount of runners we may start.
:returns: The amount of new runners started.
"""
needed = max(0, self.wanted_runners - len(self.get_active_runners()))
to_start: t.List[uuid.UUID] = []
created: t.List[Runner] = []
if needed > 0 and unassigned_runners:
# We will assign runner that were previously unassigned, so we
# might need to start some extra runners.
callback_after_this_request(
cg_broker.tasks.start_needed_unassigned_runners.delay)
for _ in range(needed):
if unassigned_runners:
self.runners.append(unassigned_runners.pop())
elif startable > 0:
runner = Runner.create_of_type(app.config['AUTO_TEST_TYPE'])
self.runners.append(runner)
db.session.add(runner)
created.append(runner)
startable -= 1
else:
break
db.session.flush()
for runner in created:
to_start.append(runner.id)
def start_runners() -> None:
for runner_id in to_start:
cg_broker.tasks.start_runner.delay(runner_id.hex)
callback_after_this_request(start_runners)
return len(created)
class AutoTestStepResult(Base, TimestampMixin, IdMixin):
"""This class represents the result of a single AutoTest step.
"""
auto_test_step_id = db.Column('auto_test_step_id',
db.Integer,
db.ForeignKey('AutoTestStep.id',
ondelete='CASCADE'),
nullable=False)
step = db.relationship(
lambda: AutoTestStepBase,
foreign_keys=auto_test_step_id,
lazy='joined',
innerjoin=True,
)
auto_test_result_id = db.Column(
'auto_test_result_id',
db.Integer,
db.ForeignKey('AutoTestResult.id', ondelete='CASCADE'),
)
result = db.relationship(
lambda: psef.models.AutoTestResult,
foreign_keys=auto_test_result_id,
innerjoin=True,
back_populates='step_results',
)
_state = db.Column(
'state',
db.Enum(AutoTestStepResultState),
default=AutoTestStepResultState.not_started,
nullable=False,
)
started_at = db.Column('started_at',
db.TIMESTAMP(timezone=True),
default=None,
nullable=True)
log: ColumnProxy[JSONType] = db.Column('log',
JSON,
nullable=True,
default=None)
_attachment_filename = db.Column('attachment_filename',
db.Unicode,
nullable=True,
default=None)
def _get_has_attachment(self) -> bool:
return self.attachment.is_just
@hybrid_expression
def _get_has_attachment_expr(
cls: t.Type['AutoTestStepResult']) -> DbColumn[bool]:
# pylint: disable=no-self-argument
return cls._attachment_filename.isnot(None)
#: Check if this step has an attachment
has_attachment = hybrid_property(_get_has_attachment,
expr=_get_has_attachment_expr)
@property
def state(self) -> AutoTestStepResultState:
"""The state of this result. Setting this might also change the
``started_at`` property.
"""
return self._state
@state.setter
def state(self, new_state: AutoTestStepResultState) -> None:
if self._state == new_state:
return
self._state = new_state
if new_state == AutoTestStepResultState.running:
self.started_at = DatetimeWithTimezone.utcnow()
else:
self.started_at = None
@property
def achieved_points(self) -> float:
"""Get the amount of achieved points by this step result.
"""
return self.step.get_amount_achieved_points(self)
@property
def attachment(self) -> Maybe[cg_object_storage.File]:
"""Maybe the attachment of this step.
The step might not have an attachment in which case ``Nothing`` is
returned.
"""
if self._attachment_filename is None:
return Nothing
return current_app.file_storage.get(self._attachment_filename)
def schedule_attachment_deletion(self) -> None:
"""Delete the attachment of this result after the current request.
The attachment, if present, will be deleted, if not attachment is
present this function does nothing.
:returns: Nothing.
"""
old_attachment = self.attachment
if old_attachment.is_just:
deleter = old_attachment.value.delete
callback_after_this_request(deleter)
def update_attachment(self, stream: FileStorage) -> None:
"""Update the attachment of this step.
:param stream: Attachment data.
"""
if not self.step.SUPPORTS_ATTACHMENT:
raise APIException('This step type does not support attachment',
(f'The step {self.step.id} does not support'
' attachments but step result {self.id}'
' generated one anyway'),
APICodes.INVALID_STATE, 400)
self.schedule_attachment_deletion()
max_size = current_app.max_single_file_size
with current_app.file_storage.putter() as putter:
new_file = putter.from_stream(stream.stream, max_size=max_size)
if new_file.is_nothing: # pragma: no cover
raise helpers.make_file_too_big_exception(max_size, True)
self._attachment_filename = new_file.value.name
class AsJSON(TypedDict):
"""The step result as JSON.
"""
#: The id of the result of a step
id: int
#: The step this is the result of.
auto_test_step: AutoTestStepBase
#: The state this result is in.
state: AutoTestStepResultState
#: The amount of points achieved by the student in this step.
achieved_points: float
#: The log produced by this result. The format of this log depends on
#: the step result.
log: t.Optional[t.Any]
#: The time this result was started, if ``null`` the result hasn't
#: started yet.
started_at: t.Optional[DatetimeWithTimezone]
#: The id of the attachment produced by this result. If ``null`` no
#: attachment was produced.
attachment_id: t.Optional[str]
def __to_json__(self) -> AsJSON:
try:
auth.ensure_can_view_autotest_step_details(self.step)
except exceptions.PermissionException:
log = self.step.remove_step_details(self.log)
else:
log = self.log
return {
'id': self.id,
'auto_test_step': self.step,
'state': self.state,
'achieved_points': self.achieved_points,
'log': log,
'started_at': self.started_at,
'attachment_id': self._attachment_filename,
}
class Job(Base, mixins.TimestampMixin, mixins.IdMixin):
"""This class represents a single job.
A job is something a CodeGrade instance needs a runner for. These jobs are
never deleted, but its state changes during its life.
"""
__tablename__ = 'job'
_state = db.Column(
'state',
db.Enum(JobState),
nullable=False,
# We constantly filter on the state, so it makes to have an index on
# this column.
index=True,
default=JobState.waiting_for_runner
)
cg_url = db.Column('cg_url', db.Unicode, nullable=False)
# It is important that this is really unique!
remote_id = db.Column(
'remote_id', db.Unicode, nullable=False, index=True, unique=True
)
runners = db.relationship(
Runner, back_populates='job', uselist=True, lazy='selectin'
)
job_metadata = db.Column('job_metadata', JSONB, nullable=True, default={})
def update_metadata(self, new_values: t.Dict[str, object]) -> None:
self.job_metadata = {**(self.job_metadata or {}), **new_values}
_wanted_runners = db.Column(
'wanted_runners',
db.Integer,
default=1,
server_default='1',
nullable=False
)
def _get_wanted_runners(self) -> int:
"""The amount of runners this job wants.
"""
return self._wanted_runners
def _set_wanted_runners(self, new_value: int) -> None:
self._wanted_runners = min(
max(new_value, 1),
app.config['MAX_AMOUNT_OF_RUNNERS_PER_JOB'],
)
wanted_runners = hybrid_property(_get_wanted_runners, _set_wanted_runners)
def _get_state(self) -> JobState:
"""Get the state of this job.
"""
return self._state
def _set_state(self, new_state: JobState) -> None:
if new_state.value < self.state.value:
raise ValueError(
'Cannot decrease the state!', self.state, new_state
)
self._state = new_state
state = hybrid_property(_get_state, _set_state)
def __init__(
self,
remote_id: str,
cg_url: str,
state: JobState = JobState.waiting_for_runner,
) -> None:
super().__init__(
remote_id=remote_id,
cg_url=cg_url,
_state=state,
)
def get_active_runners(self) -> t.List[Runner]:
return [
r for r in self.runners
if r.state in RunnerState.get_active_states()
]
def _use_runner(self, runner: Runner) -> None:
self.runners.append(runner)
active_runners = self.get_active_runners()
before_assigned = set(RunnerState.get_before_assigned_states())
too_many_active = len(active_runners) > self.wanted_runners
if too_many_active:
unneeded_runner = next(
(r for r in active_runners if r.state in before_assigned),
None,
)
if unneeded_runner is not None:
# In this case we now have too many runners assigned to us, so
# make one of the runners unassigned. But only do this if we
# have a runner which isn't already running.
unneeded_runner.make_unassigned()
callback_after_this_request(
cg_broker.tasks.maybe_start_more_runners.delay
)
def __log__(self) -> t.Dict[str, object]:
return {
'state': self.state and self.state.name,
'cg_url': self.cg_url,
'id': self.id,
}
@staticmethod
def _can_steal_runner(runner: Runner) -> bool:
"""Can this job steal the given runner.
:param runner: The runner we might want to steal.
:returns: If we can steal this runner.
"""
if runner.job is None:
return True
# TODO: We might want to check not only if we can steal this runner,
# but also if we need this runner more than this other job. For example
# if we want 5 runners but have 4, and this other job wants 10 runners
# but has 1, we might not want to steal this runner from that other
# job.
return any(r != runner for r in runner.job.get_active_runners())
def _get_needs_more_runners(self) -> bool:
"""Are more runners needed for this job?
This will check how many runners the job currently has running (so this
less or equal to assigned amount) compared to the amount of wanted
runners.
"""
amount_running = sum(1 for r in self.runners if r.see_as_running_job)
return amount_running < self.wanted_runners
@hybrid_expression
def _get_needs_more_runners_expr(cls: t.Type['Job']) -> DbColumn[bool]:
# pylint: disable=no-self-argument
amount_running = Runner.query.filter(
Runner.job_id == cls.id,
Runner.see_as_running_job,
).with_entities(func.count()).as_scalar()
return amount_running < cls.wanted_runners
needs_more_runners = hybrid_property(
fget=_get_needs_more_runners,
expr=_get_needs_more_runners_expr,
)
def maybe_use_runner(self, runner: Runner) -> bool:
"""Maybe use the given ``runner`` for this job.
This function checks if this job is allowed to use the given
runner. This is the case if the runner is assigned to us, if it is
unassigned or we can steal it from another job.
:param runner: The runner we might want to use.
:returns: ``True`` if we can use the runner, and ``False`` if we
cannot.
"""
if runner in self.runners:
return True
elif runner.state not in RunnerState.get_before_assigned_states():
# Never steal a runner that in the assigned (or later) state, as
# the backend probably already thinks it has the right to use this
# runner.
return False
elif not self.needs_more_runners:
logger.info('Too many runners assigned to job')
return False
elif runner.job is None: # Runner is unassigned, so get it.
self._use_runner(runner)
return True
elif self._can_steal_runner(runner):
logger.info(
'Stealing runner from job',
new_job=self,
other_job=runner.job,
runner=runner,
)
self._use_runner(runner)
return True
else:
return False
def __to_json__(self) -> t.Dict[str, object]:
return {
'id': self.remote_id,
'state': self.state.name,
'wanted_runners': self.wanted_runners,
}
def add_runners_to_job(
self, unassigned_runners: t.Sequence[Runner], startable: int
) -> int:
"""Add runners to the given job.
This adds runners from the ``unassigned_runners`` list to the current
job, or starts new runners as long as ``startable`` is greater than 0.
.. note:: The ``unassigned_runners`` list may be mutated in place.
:param unassigned_runners: Runners that are not assigned yet and can be
used by this job.
:param startable: The amount of runners we may start.
:returns: The amount of new runners started.
"""
needed = max(0, self.wanted_runners - len(self.get_active_runners()))
to_start: t.List[uuid.UUID] = []
created: t.List[Runner] = []
unassigned_runners = list(unassigned_runners)
used_unassigned = False
for _ in range(needed):
if unassigned_runners:
used_unassigned = True
self.runners.append(unassigned_runners.pop())
elif startable > 0:
runner = Runner.create_of_type(app.config['AUTO_TEST_TYPE'])
self.runners.append(runner)
db.session.add(runner)
created.append(runner)
startable -= 1
else:
break
db.session.flush()
for runner in created:
to_start.append(runner.id)
def start_runners() -> None:
if used_unassigned:
cg_broker.tasks.start_needed_unassigned_runners.delay()
for runner_id in to_start:
cg_broker.tasks.start_runner.delay(runner_id.hex)
callback_after_this_request(start_runners)
return len(created)
def __structlog__(self) -> t.Dict[str, t.Union[str, int, None]]:
return {
'type': str(type(self)),
'id': self.id,
'remote_id': self.remote_id,
'state': self.state.name,
'cg_url': self.cg_url,
}
class Runner(Base, mixins.TimestampMixin, mixins.UUIDMixin):
"""This class represents a abstract runner.
A runner is used one time to execute a job.
"""
if t.TYPE_CHECKING: # pragma: no cover
__mapper__: t.ClassVar[cgs.types.Mapper['Runner']]
__tablename__ = 'runner'
ipaddr = db.Column(
'ipaddr', db.Unicode, nullable=True, default=None, index=True
)
_runner_type = db.Column('type', db.Enum(RunnerType), nullable=False)
state = db.Column(
'state',
db.Enum(RunnerState),
nullable=False,
# We constantly filter on the state, so it makes to have an index on
# this column.
index=True,
default=RunnerState.not_running
)
started_at = db.Column(
db.TIMESTAMP(timezone=True), default=None, nullable=True
)
# The id used by the provider of this execution unit
instance_id = db.Column('instance_id', db.Unicode, nullable=True)
last_log_emitted = db.Column(
'last_log_emitted', db.Boolean, nullable=False, default=False
)
job_id = db.Column(
'job_id',
db.Integer,
db.ForeignKey('job.id'),
nullable=True,
default=None,
unique=False
)
job = db.relationship(
lambda: Job,
foreign_keys=job_id,
back_populates='runners',
)
public_id = db.Column(
'public_id',
UUIDType,
unique=True,
nullable=False,
default=uuid.uuid4,
server_default=sqlalchemy.func.uuid_generate_v4(),
)
_pass = db.Column(
'pass',
db.Unicode,
unique=False,
nullable=False,
default=utils.make_password,
server_default=sqlalchemy.func.cast(
sqlalchemy.func.uuid_generate_v4(),
db.Unicode,
)
)
def __to_json__(self) -> t.Mapping[str, str]:
return {'id': str(self.public_id)}
def cleanup_runner(self, shutdown_only: bool) -> None:
raise NotImplementedError
def start_runner(self) -> None:
raise NotImplementedError
def is_pass_valid(self, requesting_pass: str) -> bool:
return secrets.compare_digest(self._pass, requesting_pass)
def is_ip_valid(self, requesting_ip: str) -> bool:
"""Check if the requesting ip is valid for this Runner.
"""
if self.ipaddr is None:
return False
return secrets.compare_digest(self.ipaddr, requesting_ip)
def __hash__(self) -> int:
return hash(self.id)
def __eq__(self, other: object) -> bool:
if isinstance(other, Runner):
return self.id == other.id
return NotImplemented
def __ne__(self, other: object) -> bool:
return not self == other
@property
def should_clean(self) -> bool:
"""Is this runner in a state where it still needs cleaning.
This is always true if the runner is not cleaned or currently being
cleaned.
"""
return self.state not in {RunnerState.cleaned, RunnerState.cleaning}
@classmethod
def get_amount_of_startable_runners(cls) -> int:
"""Get the amount of runners that can still be started.
"""
active = len(cls.get_all_active_runners().with_for_update().all())
max_amount = app.config['MAX_AMOUNT_OF_RUNNERS']
return max(max_amount - active, 0)
@classmethod
def can_start_more_runners(cls) -> bool:
"""Is it currently allowed to start more runners.
This checks the amount of runners currently active and makes sure this
is less than the maximum amount of runners.
"""
# We do a all and len here as count() and with_for_update cannot be
# used in combination.
amount = len(cls.get_all_active_runners().with_for_update().all())
max_amount = app.config['MAX_AMOUNT_OF_RUNNERS']
can_start = amount < max_amount
logger.info(
'Checking if we can start more runners',
running_runners=amount,
maximum_amount=max_amount,
can_start_more=can_start
)
if not can_start:
logger.warning('Too many runners active', active_amount=amount)
return can_start
@classmethod
def _create(cls: t.Type[_Y]) -> _Y:
return cls()
@classmethod
def create_of_type(cls, typ: RunnerType) -> 'Runner':
"""Create a runner of the given type.
:param typ: The type of runner you want to create.
:returns: The newly created runner.
"""
return cls.__mapper__.polymorphic_map[typ].class_._create() # pylint: disable=protected-access
@classmethod
def get_all_active_runners(cls) -> types.MyQuery['Runner']:
"""Get a query to get all the currently active runners.
.. warning::
This query does not lock these runners so do so yourself if that is
needed.
"""
return db.session.query(cls).filter(
cls.state.in_(RunnerState.get_active_states())
)
@classmethod
def get_before_active_unassigned_runners(cls) -> types.MyQuery['Runner']:
"""Get all runners runners that are not running yet, and are not
assigned.
"""
return db.session.query(cls).filter(
cls.state.in_(RunnerState.get_before_assigned_states()),
cls.job_id.is_(None)
)
def make_unassigned(self) -> None:
"""Make this runner unassigned.
.. note::
This also starts a job to kill this runner after a certain amount
of time if it is still unassigned.
"""
runner_hex_id = self.id.hex
self.job_id = None
if self.state in RunnerState.get_before_running_states():
if self.state.is_assigned:
self.state = RunnerState.started
eta = DatetimeWithTimezone.utcnow() + timedelta(
minutes=app.config['RUNNER_MAX_TIME_ALIVE']
)
callback_after_this_request(
lambda: cg_broker.tasks.maybe_kill_unneeded_runner.apply_async(
(runner_hex_id, ),
eta=eta,
)
)
else:
self.state = RunnerState.cleaning
callback_after_this_request(
lambda: cg_broker.tasks.kill_runner.delay(runner_hex_id)
)
def kill(self, *, maybe_start_new: bool, shutdown_only: bool) -> None:
"""Kill this runner and maybe start a new one.
:param maybe_start_new: Should we maybe start a new runner after
killing this one.
"""
self.state = RunnerState.cleaning
db.session.commit()
self.cleanup_runner(shutdown_only)
self.state = RunnerState.cleaned
db.session.commit()
if maybe_start_new:
cg_broker.tasks.maybe_start_more_runners.delay()
def __structlog__(self) -> t.Dict[str, t.Union[str, int, None]]:
return {
'type': str(type(self)),
'id': self.id.hex,
'state': self.state.name,
'job_id': self.job_id,
'ipaddr': self.ipaddr,
}
def verify_password(self) -> None:
"""Verify the password in the current request, if this runner type
needs a correct password.
"""
def _get_see_as_running_job(self) -> bool:
if self.state.is_running:
return True
now = DatetimeWithTimezone.utcnow()
grace_period = Setting.get(PossibleSetting.assigned_grace_period)
if (
self.state.is_assigned and
(self.updated_at - now) < timedelta(seconds=grace_period)
):
return True
return False
@hybrid_expression
def _get_see_as_running_job_expr(cls: t.Type['Runner']) -> DbColumn[bool]:
# pylint: disable=no-self-argument
now = DatetimeWithTimezone.utcnow()
grace_period = Setting.get(PossibleSetting.assigned_grace_period)
return expression.or_(
cls.state == RunnerState.running,
expression.and_(
cls.state == RunnerState.assigned,
(cls.updated_at - now) < timedelta(seconds=grace_period),
),
)
see_as_running_job = hybrid_property(
_get_see_as_running_job, expr=_get_see_as_running_job_expr
)
__mapper_args__ = {
'polymorphic_on': _runner_type,
'polymorphic_identity': '__non_existing__',
}
class WorkRubricItem(helpers.NotEqualMixin, Base):
"""The association table between a :class:`.work_models.Work` and a
:class:`.RubricItem`.
"""
def __init__(
self,
*,
rubric_item: 'RubricItem',
work: 'work_models.Work',
multiplier: float = 1.0,
) -> None:
assert rubric_item.id is not None
assert work.id is not None
super().__init__(
rubric_item=rubric_item,
work=work,
rubricitem_id=rubric_item.id,
work_id=work.id,
multiplier=multiplier,
)
__tablename__ = 'work_rubric_item'
__table_args__ = (db.CheckConstraint('multiplier >= 0 AND multiplier <= 1',
name='ck_multiplier_range'), )
work_id = db.Column(
'work_id',
db.Integer,
db.ForeignKey('Work.id', ondelete='CASCADE'),
nullable=False,
primary_key=True,
)
rubricitem_id = db.Column(
'rubricitem_id',
db.Integer,
db.ForeignKey('RubricItem.id', ondelete='CASCADE'),
nullable=False,
primary_key=True,
)
multiplier = db.Column(
'multiplier',
db.Float,
nullable=False,
default=1.0,
server_default='1.0',
)
work = db.relationship(lambda: work_models.Work, foreign_keys=work_id)
rubric_item = db.relationship(
lambda: RubricItem,
foreign_keys=rubricitem_id,
lazy='selectin',
)
def __eq__(self, other: object) -> bool:
"""Check if this rubric item is equal to another one.
>>> w1 = psef.models.Work(id=1)
>>> w2 = psef.models.Work(id=2)
>>> r1 = RubricItem(id=1)
>>> r2 = RubricItem(id=1)
>>> wr1 = WorkRubricItem(rubric_item=r1, work=w1, multiplier=0.3)
>>> wr2 = WorkRubricItem(rubric_item=r1, work=w1, multiplier=0.3)
>>> wr1 == wr2
True
>>> wr1.multiplier = 0.1 + 0.2
>>> wr1 == wr2
True
>>> repr(wr1) == repr(wr2) # Rounding is different
False
>>> wr1 == object()
False
>>> wr1.multiplier = 0.4
>>> wr1 == wr2
False
>>> wr1.multiplier = 0.3
>>> wr1.work_id = 10
>>> wr1 == wr2
False
"""
if not isinstance(other, WorkRubricItem):
return NotImplemented
return (self.work_id == other.work_id
and self.rubricitem_id == other.rubricitem_id
and helpers.FloatHelpers.eq(self.multiplier, other.multiplier))
def __hash__(self) -> int:
# Note that `a == b` implies `hash(a) == hash(b)`, but that `hash(a) ==
# hash(b)` does not imply `a == b`. So this hash might be imperfect, it
# is correct.
# We do not use the multiplier, as this is a float, and thus it has
# rounding issues.
assert self.rubricitem_id is not None
assert self.work_id is not None
return hash((self.rubricitem_id, self.work_id))
def __repr__(self) -> str:
return '<{} rubricitem_id={}, work_id={}, multiplier={}>'.format(
self.__class__.__name__,
self.rubricitem_id,
self.work_id,
self.multiplier,
)
def __to_json__(self) -> t.Mapping[str, object]:
return {
**self.rubric_item.__to_json__(),
'achieved_points': self.points,
'multiplier': self.multiplier,
}
def _get_points(self) -> float:
"""The amount of points achieved by the work.
"""
return self.multiplier * self.rubric_item.points
@hybrid_expression
def _get_points_expr(cls: t.Type['WorkRubricItem']) -> DbColumn[float]:
"""Same as above, but this returns an expression used by
sqlalchemy.
"""
# pylint: disable=no-self-argument
return select([
cls.multiplier * RubricItem.points
]).where(cls.rubricitem_id == RubricItem.id).label('points')
points = hybrid_property(_get_points, expr=_get_points_expr)