class ExpectedStudentFile(AutograderModel):
"""
These objects describe Unix-style shell patterns that files
submitted by students can or should match.
"""
class Meta:
unique_together = ('pattern', 'project')
SERIALIZABLE_FIELDS = (
'pk',
'project',
'pattern',
'min_num_matches',
'max_num_matches',
)
EDITABLE_FIELDS = (
'pattern',
'min_num_matches',
'max_num_matches',
)
project = models.ForeignKey(Project, on_delete=models.CASCADE,
related_name='expected_student_files')
pattern = ag_fields.ShortStringField(
validators=[core_ut.check_filename],
help_text='''A shell-style file pattern suitable for
use with Python's fnmatch.fnmatch()
function (https://docs.python.org/3.5/library/fnmatch.html)
This string must be a legal UNIX filename and may not be
'..' or '.'.
NOTE: Patterns for a given project must not overlap,
otherwise the behavior is undefined.''')
min_num_matches = models.IntegerField(
default=1,
validators=[validators.MinValueValidator(0)],
help_text='''The minimum number of submitted student files that
should match the pattern. Must be non-negative.''')
max_num_matches = models.IntegerField(
default=1,
help_text='''The maximum number of submitted student files that
can match the pattern. Must be >= min_num_matches''')
def clean(self):
if self.max_num_matches < self.min_num_matches:
raise exceptions.ValidationError(
{'max_num_matches': (
'Maximum number of matches must be greater than or '
'equal to minimum number of matches')})
class AGTestCase(AutograderModel):
"""
An AGTestCase consists of a series of commands to be run together.
An AGTestCase must belong to exactly one AGTestSuite.
"""
class Meta:
unique_together = ('name', 'ag_test_suite')
order_with_respect_to = 'ag_test_suite'
name = ag_fields.ShortStringField(
help_text='''The name used to identify this autograder test.
Must be non-empty and non-null.
Must be unique among autograder tests that belong to the same suite.
This field is REQUIRED.''')
ag_test_suite = models.ForeignKey(
AGTestSuite,
related_name='ag_test_cases',
on_delete=models.CASCADE,
help_text='''The suite this autograder test belongs to.
This field is REQUIRED.''')
old_normal_fdbk_config = models.OneToOneField(
AGTestCaseFeedbackConfig,
on_delete=models.PROTECT,
default=make_default_test_fdbk,
related_name='+',
help_text='Feedback settings for a normal Submission.')
old_ultimate_submission_fdbk_config = models.OneToOneField(
AGTestCaseFeedbackConfig,
on_delete=models.PROTECT,
default=make_default_test_fdbk,
related_name='+',
help_text='Feedback settings for an ultimate Submission.')
old_past_limit_submission_fdbk_config = models.OneToOneField(
AGTestCaseFeedbackConfig,
on_delete=models.PROTECT,
default=make_default_test_fdbk,
related_name='+',
help_text=
'Feedback settings for a Submission that is past the daily limit.')
old_staff_viewer_fdbk_config = models.OneToOneField(
AGTestCaseFeedbackConfig,
on_delete=models.PROTECT,
default=make_default_test_fdbk,
related_name='+',
help_text=
'Feedback settings for a staff member viewing a Submission from another group.'
)
normal_fdbk_config = ag_fields.ValidatedJSONField(
NewAGTestCaseFeedbackConfig, default=NewAGTestCaseFeedbackConfig)
ultimate_submission_fdbk_config = ag_fields.ValidatedJSONField(
NewAGTestCaseFeedbackConfig, default=NewAGTestCaseFeedbackConfig)
past_limit_submission_fdbk_config = ag_fields.ValidatedJSONField(
NewAGTestCaseFeedbackConfig, default=NewAGTestCaseFeedbackConfig)
staff_viewer_fdbk_config = ag_fields.ValidatedJSONField(
NewAGTestCaseFeedbackConfig, default=NewAGTestCaseFeedbackConfig)
@transaction.atomic
def validate_and_update(self,
ag_test_suite: Optional[Union[int,
AGTestSuite]] = None,
**kwargs):
"""
:param ag_test_suite:
An AGTestSuite (or its primary key) that this AGTestCase
should be moved to.
It is legal to assign an AGTestCase to a different
AGTestSuite as long as the old and new suites belong to
the same Project.
"""
move_suite = False
if ag_test_suite is not None:
if isinstance(ag_test_suite, int):
move_suite = self.ag_test_suite.pk != ag_test_suite
else:
move_suite = self.ag_test_suite != ag_test_suite
if not move_suite:
super().validate_and_update(**kwargs)
return
if isinstance(ag_test_suite, int):
ag_test_suite = AGTestSuite.objects.get(pk=ag_test_suite)
if ag_test_suite.project != self.ag_test_suite.project:
raise exceptions.ValidationError({
'ag_test_suite':
'AGTestCases can only be moved to AGTestSuites within the same Project.'
})
# Update all the AGTestCaseResult objects that belong to this
# AGTestCase so that they belong to AGTestSuiteResults that
# belong to the destination AGTestSuite.
from .ag_test_suite_result import AGTestSuiteResult
for ag_test_case_result in self.related_ag_test_case_results.select_related(
'ag_test_suite_result__submission').all():
dest_suite_result = AGTestSuiteResult.objects.get_or_create(
ag_test_suite=ag_test_suite,
submission=ag_test_case_result.ag_test_suite_result.submission
)[0]
ag_test_case_result.ag_test_suite_result = dest_suite_result
ag_test_case_result.save()
self.ag_test_suite = ag_test_suite
super().validate_and_update(**kwargs)
@transaction.atomic()
def delete(self, *args, **kwargs):
with connection.cursor() as cursor:
cursor.execute(
'''UPDATE core_submission
SET denormalized_ag_test_results =
denormalized_ag_test_results #- '{%s,ag_test_case_results,%s}'
WHERE core_submission.project_id = %s
''', (self.ag_test_suite_id, self.pk,
self.ag_test_suite.project_id))
return super().delete()
SERIALIZABLE_FIELDS = (
'pk',
'name',
'last_modified',
'ag_test_suite',
'ag_test_commands',
'normal_fdbk_config',
'ultimate_submission_fdbk_config',
'past_limit_submission_fdbk_config',
'staff_viewer_fdbk_config',
)
SERIALIZE_RELATED = ('ag_test_commands', )
EDITABLE_FIELDS = (
'name',
'ag_test_suite',
'normal_fdbk_config',
'ultimate_submission_fdbk_config',
'past_limit_submission_fdbk_config',
'staff_viewer_fdbk_config',
)
class AGCommandBase(AutograderModel):
"""
An abstract base class that contains the core information needed to run
a command during grading.
"""
class Meta:
abstract = True
name = ag_fields.ShortStringField(
blank=True, help_text="An optional name for this command.")
cmd = models.CharField(
max_length=constants.MAX_COMMAND_LENGTH,
help_text='''A string containing the command to be run.
Note: This string will be inserted into ['bash', '-c', <cmd>]
in order to be executed.
Note: This string defaults to the "true" command
(which does nothing and returns 0) so that AGCommands are
default-creatable.''')
time_limit = models.IntegerField(
default=constants.DEFAULT_SUBPROCESS_TIMEOUT,
validators=[
MinValueValidator(1),
MaxValueValidator(constants.MAX_SUBPROCESS_TIMEOUT)
],
help_text='''The time limit in seconds to be placed on the
command.
Must be > 0
Must be <= autograder.shared.global_constants
.MAX_SUBPROCESS_TIMEOUT''')
stack_size_limit = models.IntegerField(
default=constants.DEFAULT_STACK_SIZE_LIMIT,
validators=[
MinValueValidator(1),
MaxValueValidator(constants.MAX_STACK_SIZE_LIMIT)
],
help_text='''
stack_size_limit -- The maximum stack size in bytes.
Must be > 0
Must be <= autograder.shared.global_constants.MAX_STACK_SIZE_LIMIT
NOTE: Setting this value too low may cause the command to crash prematurely.'''
)
virtual_memory_limit = models.BigIntegerField(
default=constants.DEFAULT_VIRTUAL_MEM_LIMIT,
validators=[
MinValueValidator(1),
MaxValueValidator(constants.MAX_VIRTUAL_MEM_LIMIT)
],
help_text='''The maximum amount of virtual memory
(in bytes) the command can use.
Must be > 0
Must be <= autograder.shared.global_constants.MAX_VIRTUAL_MEM_LIMIT
NOTE: Setting this value too low may cause the command to crash prematurely.'''
)
process_spawn_limit = models.IntegerField(
default=constants.DEFAULT_PROCESS_LIMIT,
validators=[
MinValueValidator(0),
MaxValueValidator(constants.MAX_PROCESS_LIMIT)
],
help_text=
'''The maximum number of processes that the command is allowed to spawn.
Must be >= 0
Must be <= autograder.shared.global_constants.MAX_PROCESS_LIMIT
NOTE: This limit applies cumulatively to the processes
spawned by the main program being run. i.e. If a
spawned process spawns it's own child process, both
of those processes will count towards the main
program's process limit.''')
class Project(AutograderModel):
"""
Represents a programming project for which students can
submit solutions and have them evaluated.
Related object fields:
instructor_files -- Resource files to be used in project test
cases. In the API, this field is hidden from non-staff.
expected_student_files -- Patterns that
student-submitted files can or should match.
groups -- The submission groups registered for this
Project.
group_invitations -- The pending submission group
invitations belonging to this Project.
"""
class Meta:
unique_together = ('name', 'course')
name = ag_fields.ShortStringField(
help_text="""The name used to identify this project.
Must be non-empty and non-null.
Must be unique among Projects associated with
a given course.
This field is REQUIRED.""")
course = models.ForeignKey(Course,
related_name='projects',
on_delete=models.CASCADE,
help_text="""The Course this project belongs to.
This field is REQUIRED.""")
visible_to_students = models.BooleanField(
default=False,
help_text="""Whether information about this Project can
be viewed by students.""")
closing_time = models.DateTimeField(
default=None,
null=True,
blank=True,
help_text="""The date and time that this project should stop
accepting submissions.
A value of None indicates that this project should
stay open.
In the API, this field is hidden from non-admins.""")
soft_closing_time = models.DateTimeField(
default=None,
null=True,
blank=True,
help_text="""The date and time that should be displayed as the
due date for this project. Unlike closing_time,
soft_closing_time does not affect whether submissions are
actually accepted.
If not None and closing_time is not None, this value must be
less than (before) closing_time.""")
disallow_student_submissions = models.BooleanField(
default=False,
help_text="""A hard override that indicates that students should
be prevented from submitting even if visible_to_students is
True and it is before closing_time.""")
disallow_group_registration = models.BooleanField(
default=False,
help_text="""A hard override that indicates that students should
not be able to send, accept, or reject group
invitations.""")
guests_can_submit = models.BooleanField(
default=False,
help_text="""By default, only admins, staff, and students
for a given Course can view and submit to its Projects.
When True, submissions will be accepted from guests
with the following caveats:
- Guests must be given a direct link to the project.
- When group work is allowed, guests can
only be in groups with other guests.""")
min_group_size = models.IntegerField(
default=1,
validators=[validators.MinValueValidator(1)],
help_text="""The minimum number of students that can work in a
group on this project.
Must be >= 1.
Must be <= max_group_size.""")
max_group_size = models.IntegerField(
default=1,
validators=[validators.MinValueValidator(1)],
help_text="""The maximum number of students that can work in a
group on this project.
Must be >= 1.
Must be >= min_group_size.""")
submission_limit_per_day = models.IntegerField(
default=None,
null=True,
blank=True,
validators=[validators.MinValueValidator(1)],
help_text="""The number of submissions each group is allowed per
day before either reducing feedback or preventing further
submissions. A value of None indicates no limit.""")
groups_combine_daily_submissions = models.BooleanField(
default=False,
blank=True,
help_text="""If True, group members can "pool" their daily submissions.
For example, if submission_limit_per_day is 3,
a group with 2 members would get 6 submissions per day.""")
allow_submissions_past_limit = models.BooleanField(
default=True,
blank=True,
help_text="""Whether to allow additional submissions after a
group has submitted submission_limit_per_day times.""")
submission_limit_reset_time = models.TimeField(
default=datetime.time,
help_text="""The time that marks the beginning and end of the 24
hour period during which submissions should be counted
towards the daily limit. Defaults to 0:0:0.""")
submission_limit_reset_timezone = TimeZoneField(
default='UTC',
help_text="""The timezone to use when computing how many
submissions a group has made in a 24 hour period.""")
num_bonus_submissions = models.IntegerField(
default=0, validators=[validators.MinValueValidator(0)])
total_submission_limit = models.IntegerField(
default=None,
blank=True,
null=True,
validators=[validators.MinValueValidator(1)],
help_text="""The maximum number of times a Group can submit to
this Project EVER.""")
allow_late_days = models.BooleanField(
default=False,
help_text="""Whether to allow the use of late days for submitting
past the deadline.""")
ultimate_submission_policy = ag_fields.EnumField(
UltimateSubmissionPolicy,
default=UltimateSubmissionPolicy.most_recent,
blank=True,
help_text="""The "ultimate" submission for a group is the one
that will be used for final grading. This field specifies
how the ultimate submission should be determined.""")
hide_ultimate_submission_fdbk = models.BooleanField(
default=True,
blank=True,
help_text="""A hard override that indicates that ultimate
submission feedback should not be shown, even if the
appropriate criteria are met.""")
def save(self, *args, **kwargs):
super().save(*args, **kwargs)
project_root_dir = core_ut.get_project_root_dir(self)
project_files_dir = core_ut.get_project_files_dir(self)
project_submissions_dir = core_ut.get_project_groups_dir(self)
if not os.path.isdir(project_root_dir):
# Since the database is in charge of validating the
# uniqueness of this project, we can assume at this point
# that creating the project directories will succeed.
# If for some reason it fails, this will be considered a
# more severe error, and the OSError thrown by os.makedirs
# will be handled at a higher level.
os.makedirs(project_root_dir)
os.mkdir(project_files_dir)
os.mkdir(project_submissions_dir)
def clean(self):
super().clean()
if self.max_group_size < self.min_group_size:
raise exceptions.ValidationError({
'max_group_size': ('Maximum group size must be greater than '
'or equal to minimum group size')
})
if self.closing_time is not None and self.soft_closing_time is not None:
if self.closing_time < self.soft_closing_time:
raise exceptions.ValidationError({
'soft_closing_time':
('Soft closing time must be before hard closing time')
})
def to_dict(self):
result = super().to_dict()
result['submission_limit_reset_timezone'] = (
self.submission_limit_reset_timezone.tzname(None))
return result
SERIALIZABLE_FIELDS = (
'pk',
'name',
'last_modified',
'course',
'visible_to_students',
'closing_time',
'soft_closing_time',
'disallow_student_submissions',
'disallow_group_registration',
'guests_can_submit',
'min_group_size',
'max_group_size',
'submission_limit_per_day',
'allow_submissions_past_limit',
'groups_combine_daily_submissions',
'submission_limit_reset_time',
'submission_limit_reset_timezone',
'num_bonus_submissions',
'total_submission_limit',
'allow_late_days',
'ultimate_submission_policy',
'hide_ultimate_submission_fdbk',
'instructor_files',
'expected_student_files',
)
SERIALIZE_RELATED = (
'instructor_files',
'expected_student_files',
)
EDITABLE_FIELDS = (
'name',
'visible_to_students',
'closing_time',
'soft_closing_time',
'disallow_student_submissions',
'disallow_group_registration',
'guests_can_submit',
'min_group_size',
'max_group_size',
'submission_limit_per_day',
'allow_submissions_past_limit',
'groups_combine_daily_submissions',
'submission_limit_reset_time',
'submission_limit_reset_timezone',
'num_bonus_submissions',
'total_submission_limit',
'allow_late_days',
'ultimate_submission_policy',
'hide_ultimate_submission_fdbk',
)
class StudentTestSuite(AutograderModel):
"""
A StudentTestSuite defines a way of grading student-submitted
test cases against a set of intentionally buggy implementations
of instructor code.
"""
class Meta:
unique_together = ('name', 'project')
order_with_respect_to = 'project'
STUDENT_TEST_NAME_PLACEHOLDER = r'${student_test_name}'
BUGGY_IMPL_NAME_PLACEHOLDER = r'${buggy_impl_name}'
name = ag_fields.ShortStringField(
help_text="""The name used to identify this StudentTestSuite.
Must be non-empty and non-null.""")
project = models.ForeignKey(
Project,
on_delete=models.CASCADE,
related_name='student_test_suites',
help_text="The Project that this student test suite belongs to.")
instructor_files_needed = models.ManyToManyField(
InstructorFile,
help_text=
"""The project files that will be copied into the sandbox before the suite
is graded.""")
read_only_instructor_files = models.BooleanField(
default=True,
help_text=
"""When True, project files needed for this suite will be read-only when this
suite is graded.""")
student_files_needed = models.ManyToManyField(
ExpectedStudentFile,
help_text=
'''Student-submitted files matching these patterns will be copied into the
sandbox before the suite is graded.''')
buggy_impl_names = ag_fields.StringArrayField(
strip_strings=True,
blank=True,
default=list,
help_text=
"The names of buggy implementations that student tests should be run against."
)
use_setup_command = models.BooleanField(default=False)
setup_command = models.OneToOneField(
AGCommand,
on_delete=models.PROTECT,
related_name='+',
default=make_default_setup_cmd,
help_text="""A command to be run after student and project files have
been added to the sandbox but before any other commands are run.
The AGCommand's 'cmd' field must not be blank. To indicate that no
setup command should be run, set use_setup_command to False."""
)
get_student_test_names_command = models.OneToOneField(
AGCommand,
on_delete=models.PROTECT,
related_name='+',
blank=True,
default=make_default_get_student_test_names_cmd,
help_text=
"""This required command should print out a whitespace-separated
list of detected student names. The output of this command will
be parsed using Python's str.split().
NOTE: This AGCommand's 'cmd' field must not be blank.""")
DEFAULT_STUDENT_TEST_MAX = 25
MAX_STUDENT_TEST_MAX = 50
max_num_student_tests = models.IntegerField(
default=DEFAULT_STUDENT_TEST_MAX,
validators=[
MinValueValidator(0),
MaxValueValidator(MAX_STUDENT_TEST_MAX)
],
help_text=
"""The maximum number of test cases students are allowed to submit.
If more than this many tests are discovered by the
get_student_test_names_command, test names will be discarded
from the end of that list.""")
student_test_validity_check_command = models.OneToOneField(
AGCommand,
on_delete=models.PROTECT,
related_name='+',
blank=True,
default=make_default_validity_check_command,
help_text=
"""This command will be run once for each detected student test case.
An exit status of zero indicates that a student test case is valid,
whereas a nonzero exit status indicates that a student test case
is invalid.
This command must contain the placeholder {} at least once. That
placeholder will be replaced with the name of the student test case
that is to be checked for validity.
NOTE: This AGCommand's 'cmd' field must not be blank.
""".format(STUDENT_TEST_NAME_PLACEHOLDER))
grade_buggy_impl_command = models.OneToOneField(
AGCommand,
on_delete=models.PROTECT,
related_name='+',
blank=True,
default=make_default_grade_buggy_impl_command,
help_text=
"""This command will be run once for every (buggy implementation, valid test)
pair.
A nonzero exit status indicates that the valid student tests exposed the
buggy impl, whereas an exit status of zero indicates that the student
tests did not expose the buggy impl.
This command must contain the placeholders {0} and {1}. The placeholder
{0} will be replaced with the name of a valid student test case.
The placeholder {1} will be replaced with the name of
the buggy impl that the student test is being run against.
NOTE: This AGCommand's 'cmd' field must not be blank.
""".format(STUDENT_TEST_NAME_PLACEHOLDER,
BUGGY_IMPL_NAME_PLACEHOLDER))
points_per_exposed_bug = models.DecimalField(
decimal_places=2,
max_digits=4,
default=0,
validators=[MinValueValidator(0)],
help_text=
"""The number of points to be awarded per buggy implementation exposed by
the student test cases. This field is limited to 4 digits total and a maximum
of 2 decimal places.""")
max_points = models.IntegerField(
null=True,
default=None,
blank=True,
validators=[MinValueValidator(0)],
help_text=
"""An optional ceiling on the number of points to be awarded.""")
deferred = models.BooleanField(
default=False,
help_text=
'''If true, this student test suite can be graded asynchronously.
Deferred suites that have yet to be graded do not prevent members
of a group from submitting again.''')
docker_image_to_use = ag_fields.EnumField(
constants.SupportedImages,
default=constants.SupportedImages.default,
help_text=
"An identifier for the Docker image that the sandbox should be created from."
)
allow_network_access = models.BooleanField(
default=False,
help_text=
'''Specifies whether the sandbox should allow commands run inside of it to
make network calls outside of the sandbox.''')
normal_fdbk_config = models.OneToOneField(
StudentTestSuiteFeedbackConfig,
on_delete=models.PROTECT,
default=make_default_command_fdbk,
related_name='+',
help_text='Feedback settings for a normal Submission.')
ultimate_submission_fdbk_config = models.OneToOneField(
StudentTestSuiteFeedbackConfig,
on_delete=models.PROTECT,
default=make_default_ultimate_submission_command_fdbk,
related_name='+',
help_text='Feedback settings for an ultimate Submission.')
past_limit_submission_fdbk_config = models.OneToOneField(
StudentTestSuiteFeedbackConfig,
on_delete=models.PROTECT,
default=make_default_past_limit_student_suite_fdbk,
related_name='+',
help_text=
'Feedback settings for a Submission that is past the daily limit.')
staff_viewer_fdbk_config = models.OneToOneField(
StudentTestSuiteFeedbackConfig,
on_delete=models.PROTECT,
default=make_max_student_suite_fdbk,
related_name='+',
help_text=
'Feedback settings for a staff member viewing a Submission from another group.'
)
def clean(self):
if self.pk is None:
return
errors = {}
for instructor_file in self.instructor_files_needed.all():
if instructor_file.project != self.project:
errors['instructor_files_needed'] = (
'File {} does not belong to the project "{}".'.format(
instructor_file.name, self.project.name))
for student_file in self.student_files_needed.all():
if student_file.project != self.project:
errors['student_files_needed'] = (
'Student file pattern {} does not belong to the project "{}".'
.format(student_file.pattern, self.project.name))
for cmd_field in [
'setup_command', 'get_student_test_names_command',
'student_test_validity_check_command',
'grade_buggy_impl_command'
]:
cmd = getattr(self, cmd_field) # type: AGCommand
if cmd is None:
continue
if self.STUDENT_TEST_NAME_PLACEHOLDER not in self.student_test_validity_check_command.cmd:
errors['student_test_validity_check_command'] = (
'Validity check command missing placeholder "{}"'.format(
self.STUDENT_TEST_NAME_PLACEHOLDER))
if self.STUDENT_TEST_NAME_PLACEHOLDER not in self.grade_buggy_impl_command.cmd:
errors['grade_buggy_impl_command'] = (
'Grade buggy impl command missing placeholder "{}"'.format(
self.STUDENT_TEST_NAME_PLACEHOLDER))
if self.BUGGY_IMPL_NAME_PLACEHOLDER not in self.grade_buggy_impl_command.cmd:
errors['grade_buggy_impl_command'] = (
'Grade buggy impl command missing placeholder "{}"'.format(
self.BUGGY_IMPL_NAME_PLACEHOLDER))
if errors:
raise exceptions.ValidationError(errors)
SERIALIZABLE_FIELDS = (
'pk',
'name',
'project',
'instructor_files_needed',
'read_only_instructor_files',
'student_files_needed',
'buggy_impl_names',
'use_setup_command',
'setup_command',
'get_student_test_names_command',
'max_num_student_tests',
'student_test_validity_check_command',
'grade_buggy_impl_command',
'points_per_exposed_bug',
'max_points',
'deferred',
'docker_image_to_use',
'allow_network_access',
'normal_fdbk_config',
'ultimate_submission_fdbk_config',
'past_limit_submission_fdbk_config',
'staff_viewer_fdbk_config',
'last_modified',
)
EDITABLE_FIELDS = (
'name',
'instructor_files_needed',
'read_only_instructor_files',
'student_files_needed',
'buggy_impl_names',
'use_setup_command',
'setup_command',
'get_student_test_names_command',
'max_num_student_tests',
'student_test_validity_check_command',
'grade_buggy_impl_command',
'points_per_exposed_bug',
'max_points',
'deferred',
'docker_image_to_use',
'allow_network_access',
'normal_fdbk_config',
'ultimate_submission_fdbk_config',
'past_limit_submission_fdbk_config',
'staff_viewer_fdbk_config',
'last_modified',
)
SERIALIZE_RELATED = (
'instructor_files_needed',
'student_files_needed',
)
TRANSPARENT_TO_ONE_FIELDS = (
'setup_command',
'get_student_test_names_command',
'student_test_validity_check_command',
'grade_buggy_impl_command',
'normal_fdbk_config',
'ultimate_submission_fdbk_config',
'past_limit_submission_fdbk_config',
'staff_viewer_fdbk_config',
)
class AGTestSuite(AutograderModel):
"""
A group of autograder test cases to be run inside the same sandbox.
"""
class Meta:
unique_together = ('name', 'project')
order_with_respect_to = 'project'
name = ag_fields.ShortStringField(
help_text='''The name used to identify this suite.
Must be non-empty and non-null.
Must be unique among suites that belong to the same project.
This field is REQUIRED.''')
project = models.ForeignKey(Project,
related_name='ag_test_suites',
on_delete=models.CASCADE,
help_text='''The project this suite belongs to.
This field is REQUIRED.''')
instructor_files_needed = models.ManyToManyField(
InstructorFile,
help_text=
'''The project files that will be copied into the sandbox before the suite's
tests are run.''')
read_only_instructor_files = models.BooleanField(
default=True,
help_text=
"""When True, project files needed for this suite will be read-only when this
suite is run.""")
student_files_needed = models.ManyToManyField(
ExpectedStudentFile,
help_text=
'''Student-submitted files matching these patterns will be copied into the
sandbox before the suite's tests are run.''')
setup_suite_cmd = models.CharField(
max_length=constants.MAX_COMMAND_LENGTH,
blank=True,
help_text="""A command to be run before this suite's tests are run.
This command is only run once at the beginning of the suite.
This command will be run after the student and project files
have been added to the sandbox.""")
setup_suite_cmd_name = ag_fields.ShortStringField(
blank=True, help_text="""The name of this suite's setup command.""")
docker_image_to_use = ag_fields.EnumField(
constants.SupportedImages,
default=constants.SupportedImages.default,
help_text=
"An identifier for the Docker image that the sandbox should be created from."
)
allow_network_access = models.BooleanField(
default=False,
help_text=
'''Specifies whether the sandbox should allow commands run inside of it to
make network calls outside of the sandbox.''')
deferred = models.BooleanField(
default=False,
help_text=
'''If true, this test suite can be graded asynchronously. Deferred suites that
have yet to be graded do not prevent members of a group from submitting
again.''')
old_normal_fdbk_config = models.OneToOneField(
AGTestSuiteFeedbackConfig,
on_delete=models.PROTECT,
default=make_default_suite_fdbk,
related_name='+',
help_text='Feedback settings for a normal submission.')
old_ultimate_submission_fdbk_config = models.OneToOneField(
AGTestSuiteFeedbackConfig,
on_delete=models.PROTECT,
default=make_default_suite_fdbk,
related_name='+',
help_text='Feedback settings for an ultimate submission.')
old_past_limit_submission_fdbk_config = models.OneToOneField(
AGTestSuiteFeedbackConfig,
on_delete=models.PROTECT,
default=make_default_suite_fdbk,
related_name='+',
help_text=
'Feedback settings for a submission that is past the daily limit.')
old_staff_viewer_fdbk_config = models.OneToOneField(
AGTestSuiteFeedbackConfig,
on_delete=models.PROTECT,
default=make_default_suite_fdbk,
related_name='+',
help_text=
'Feedback settings for a staff member viewing a submission from another group.'
)
normal_fdbk_config = ag_fields.ValidatedJSONField(
NewAGTestSuiteFeedbackConfig, default=NewAGTestSuiteFeedbackConfig)
ultimate_submission_fdbk_config = ag_fields.ValidatedJSONField(
NewAGTestSuiteFeedbackConfig, default=NewAGTestSuiteFeedbackConfig)
past_limit_submission_fdbk_config = ag_fields.ValidatedJSONField(
NewAGTestSuiteFeedbackConfig, default=NewAGTestSuiteFeedbackConfig)
staff_viewer_fdbk_config = ag_fields.ValidatedJSONField(
NewAGTestSuiteFeedbackConfig, default=NewAGTestSuiteFeedbackConfig)
def clean(self):
if self.pk is None:
return
errors = {}
for instructor_file in self.instructor_files_needed.all():
if instructor_file.project != self.project:
errors['instructor_files_needed'] = (
'File {} does not belong to the project "{}".'.format(
instructor_file.name, self.project.name))
for student_file in self.student_files_needed.all():
if student_file.project != self.project:
errors['student_files_needed'] = (
'Student file pattern {} does not belong to the project "{}".'
.format(student_file.pattern, self.project.name))
if errors:
raise exceptions.ValidationError(errors)
def delete(self, *args, **kwargs):
with connection.cursor() as cursor:
cursor.execute(
'''UPDATE core_submission
SET denormalized_ag_test_results = denormalized_ag_test_results #- '{%s}'
WHERE core_submission.project_id = %s
''', (self.pk, self.project_id))
return super().delete()
SERIALIZABLE_FIELDS = (
'pk',
'name',
'project',
'last_modified',
'instructor_files_needed',
'read_only_instructor_files',
'student_files_needed',
'ag_test_cases',
'setup_suite_cmd',
'setup_suite_cmd_name',
'docker_image_to_use',
'allow_network_access',
'deferred',
'normal_fdbk_config',
'ultimate_submission_fdbk_config',
'past_limit_submission_fdbk_config',
'staff_viewer_fdbk_config',
)
SERIALIZE_RELATED = (
'instructor_files_needed',
'student_files_needed',
'ag_test_cases',
)
EDITABLE_FIELDS = ('name', 'instructor_files_needed',
'read_only_instructor_files', 'student_files_needed',
'setup_suite_cmd', 'setup_suite_cmd_name',
'allow_network_access', 'deferred',
'docker_image_to_use', 'normal_fdbk_config',
'ultimate_submission_fdbk_config',
'past_limit_submission_fdbk_config',
'staff_viewer_fdbk_config')
class Submission(ag_model_base.AutograderModel):
"""
This model stores a set of files submitted by a student for grading.
"""
objects = _SubmissionManager()
class Meta:
ordering = ['-pk']
class GradingStatus:
# The submission has been accepted and saved to the database
received = 'received'
# The submission has been queued is waiting to be graded
queued = 'queued'
being_graded = 'being_graded'
# Non-deferred test cases have finished and the group can submit
# again.
waiting_for_deferred = 'waiting_for_deferred'
# All test cases have finished grading.
finished_grading = 'finished_grading'
# A student removed their submission from the queue before it
# started being graded.
removed_from_queue = 'removed_from_queue'
# Something unexpected occurred during the grading process.
error = 'error'
values = [
received,
queued,
being_graded,
waiting_for_deferred,
finished_grading,
removed_from_queue,
error,
]
# These statuses bar users from making another submission
# while the current one is active.
active_statuses = [received, queued, being_graded]
# A submission should only be counted towards the daily limit if
# it has one of these statuses.
count_towards_limit_statuses = [
received, queued, being_graded, waiting_for_deferred,
finished_grading
]
# -------------------------------------------------------------------------
group = models.ForeignKey('core.Group',
related_name='submissions',
on_delete=models.CASCADE,
help_text="""
The SubmissionGroup that this submission belongs to. Note
that this field indirectly links this Submission object to a
Project.
This field is REQUIRED.""")
project = models.ForeignKey(
'core.Project',
related_name='submissions',
# Project will cascade-delete groups, groups will cascade-delete
# submissions.
on_delete=models.DO_NOTHING,
help_text='A shortcut for submission.group.project.',
)
timestamp = models.DateTimeField(default=timezone.now)
submitter = ag_fields.ShortStringField(
blank=True,
help_text="""The name of the user who made this submission""")
@property
def submitted_files(self):
"""
An iterable of the files included in this submission.
"""
return (self.get_file(filename)
for filename in self.submitted_filenames)
submitted_filenames = ag_fields.StringArrayField(
blank=True,
default=list,
help_text="""The names of files that were submitted,
excluding those that were discarded.""")
discarded_files = ag_fields.StringArrayField(
default=list,
blank=True,
help_text=
"""The names of files that were discarded when this Submission was created."""
)
missing_files = pg_fields.JSONField(
default=dict,
blank=True,
help_text="""Stores missing filenames and the additional number
of files needed to satisfy a file pattern requirement.
Stored as key-value pairs of the form:
{pattern: num_additional_needed}""")
status = models.CharField(
max_length=const.MAX_CHAR_FIELD_LEN,
default=GradingStatus.received,
choices=zip(GradingStatus.values, GradingStatus.values),
help_text="""The grading status of this submission see
Submission.GradingStatus for details on allowed values.""")
count_towards_daily_limit = models.BooleanField(
default=True,
help_text="""Indicates whether this submission should count
towards the daily submission limit.""")
is_past_daily_limit = models.BooleanField(
default=False,
help_text="Whether this submission is past the daily submission limit."
)
is_bonus_submission = models.BooleanField(
default=False,
help_text="""When True, indicates that the group that made this
submission should be able to request normal feedback for
this submission's results.
Note: If this field is True, is_past_daily_limit should be
False.""")
count_towards_total_limit = models.BooleanField(
default=True,
help_text=
"Whether this submission should count towards the total submission limit."
)
does_not_count_for = pg_fields.ArrayField(
models.CharField(max_length=constants.MAX_USERNAME_LEN),
default=list,
blank=True,
help_text="""A list of users for whom this submission will NOT
count as their final graded submission. Users are added to
this list if they are out of late days and another group
member (who still has late days remaining) uses their own
late day to submit.""")
error_msg = models.TextField(
blank=True,
help_text=
"""If status is "error", an error message will be stored here.""")
denormalized_ag_test_results = pg_fields.JSONField(
default=dict,
blank=True,
help_text="""Stores denormalized AG test results in order to avoid
expensive joins when getting submission result feedback.
To update this field, use
autograder.core.submission_feedback.update_denormalized_ag_test_results
Data format:
{
"<ag test suite pk>": {
<ag test suite result data>,
"ag_test_case_results": {
"<ag test case pk>": {
<ag test case result data>,
"ag_test_command_results": <ag test command result data>
}
}
}
}
""")
@property
def position_in_queue(self) -> int:
"""
Returns this submission's position in the queue of submissions
to be graded for the associated project.
"""
if self.status != Submission.GradingStatus.queued:
return 0
return Submission.objects.filter(
status=Submission.GradingStatus.queued,
group__project=self.group.project,
pk__lte=self.pk).count()
# -------------------------------------------------------------------------
def get_file(self, filename, mode='rb'):
"""
Returns a Django File object containing the submitted file with
the given name. The file is opened using the specified mode
(mode can be any valid value for the same argument to the Python
open() function).
If the file doesn't exist, ObjectDoesNotExist will be raised.
"""
self._check_file_exists(filename)
return File(open(self._get_submitted_file_dir(filename), mode),
name=os.path.basename(filename))
def _check_file_exists(self, filename):
if filename not in self.submitted_filenames:
raise exceptions.ObjectDoesNotExist()
def _get_submitted_file_dir(self, filename):
return os.path.join(core_ut.get_submission_dir(self), filename)
def get_submitted_file_basenames(self):
return self.submitted_filenames
def save(self, *args, **kwargs):
super().save(*args, **kwargs)
# result_output_dir is a subdir of the submission dir
result_output_dir = core_ut.get_result_output_dir(self)
if not os.path.isdir(result_output_dir):
os.makedirs(result_output_dir, exist_ok=True)
SERIALIZABLE_FIELDS = (
'pk',
'group',
'timestamp',
'submitter',
'submitted_filenames',
'discarded_files',
'missing_files',
'status',
'count_towards_daily_limit',
'is_past_daily_limit',
'is_bonus_submission',
'count_towards_total_limit',
'does_not_count_for',
'position_in_queue',
)
EDITABLE_FIELDS = ('count_towards_daily_limit',
'count_towards_total_limit')