def test_one_ag_suite_deferred_one_student_suite_deferred(self, *args):
suite1 = obj_build.make_ag_test_suite(self.project, deferred=False)
case1 = obj_build.make_ag_test_case(suite1)
cmd1 = obj_build.make_full_ag_test_command(
case1,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=1)
suite2 = obj_build.make_ag_test_suite(self.project, deferred=True)
case2 = obj_build.make_ag_test_case(suite2)
cmd2 = obj_build.make_full_ag_test_command(
case2,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=2)
deferred_student_suite = ag_models.StudentTestSuite.objects.validate_and_create(
name='deferryyyy',
project=self.project,
deferred=True)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(
3,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
deferred_student_suite_result = ag_models.StudentTestSuiteResult.objects.get(
submission=self.submission,
student_test_suite=deferred_student_suite)
def test_all_suites_deferred(self, *args):
suite1 = obj_build.make_ag_test_suite(self.project, deferred=True)
case1 = obj_build.make_ag_test_case(suite1)
cmd1 = obj_build.make_full_ag_test_command(
case1,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=1)
suite2 = obj_build.make_ag_test_suite(self.project, deferred=True)
case2 = obj_build.make_ag_test_case(suite2)
cmd2 = obj_build.make_full_ag_test_command(
case2,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=2)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(
3,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
def setUp(self):
super().setUp()
self.suite1 = obj_build.make_ag_test_suite()
self.project = self.suite1.project
self.suite2 = obj_build.make_ag_test_suite(self.project)
self.client = APIClient()
self.url = reverse('ag_test_suite_order',
kwargs={'project_pk': self.project.pk})
def setUp(self):
super().setUp()
self.ag_test_suite = obj_build.make_ag_test_suite()
self.course = self.ag_test_suite.project.course
self.client = APIClient()
self.url = reverse('ag-test-suite-detail',
kwargs={'pk': self.ag_test_suite.pk})
def test_one_ag_suite_one_non_deferred_student_suite_one_deferred_student_suite(self, *args):
ag_suite = obj_build.make_ag_test_suite(self.project)
ag_case = obj_build.make_ag_test_case(ag_suite)
print_to_stdout_and_stderr = "bash -c 'printf hello; printf whoops >&2'"
ag_cmd = obj_build.make_full_ag_test_command(ag_case, cmd=print_to_stdout_and_stderr)
student_suite = ag_models.StudentTestSuite.objects.validate_and_create(
name='mnkfoae',
project=self.project)
deferred_student_suite = ag_models.StudentTestSuite.objects.validate_and_create(
name='deferryyyy',
project=self.project,
deferred=True)
tasks.grade_submission(self.submission.pk)
cmd_result = ag_models.AGTestCommandResult.objects.get(
ag_test_command=ag_cmd,
ag_test_case_result__ag_test_suite_result__submission=self.submission)
student_suite_result = ag_models.StudentTestSuiteResult.objects.get(
submission=self.submission,
student_test_suite=student_suite)
deferred_student_suite_result = ag_models.StudentTestSuiteResult.objects.get(
submission=self.submission,
student_test_suite=deferred_student_suite)
def setUp(self):
super().setUp()
self.maxDiff = None
self.project = obj_build.make_project(
# Future closing_time
closing_time=timezone.now() + datetime.timedelta(days=1))
self.course = self.project.course
self.ag_test_suite = obj_build.make_ag_test_suite(self.project)
self.ag_test_case = obj_build.make_ag_test_case(self.ag_test_suite)
self.ag_test_cmd = obj_build.make_full_ag_test_command(
self.ag_test_case)
self.student_test_suite = obj_build.make_student_test_suite(
self.project)
self.client = APIClient()
self.base_url = reverse('all-ultimate-submission-results',
kwargs={'project_pk': self.project.pk})
# For use by self._make_group_with_submissions only
self._num_groups = 0
# This is to make sure we use the right group queryset
other_project = obj_build.make_project()
other_group = obj_build.make_group(project=other_project)
other_submission = obj_build.make_finished_submission(other_group)
def test_non_default_docker_image(self, *args):
suite = obj_build.make_ag_test_suite(
self.project, docker_image_to_use=constants.SupportedImages.eecs490)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(
case,
cmd='racket --version',
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
points_for_correct_return_code=3,
expected_return_code=ag_models.ExpectedReturnCode.zero)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
cmd_result = ag_models.AGTestCommandResult.objects.get(
ag_test_command=cmd,
ag_test_case_result__ag_test_suite_result__submission=self.submission)
self.assertEqual(0, cmd_result.return_code)
self.assertEqual(
3,
get_submission_fdbk(
self.submission, ag_models.FeedbackCategory.max).total_points_possible)
self.assertEqual(
3,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
def setUp(self):
super().setUp()
self.maxDiff = None
self.client = APIClient()
self.project = obj_build.make_project(
ultimate_submission_policy=ag_models.UltimateSubmissionPolicy.
most_recent, )
self.url = reverse('project-ultimate-submission-scores',
kwargs={'pk': self.project.pk})
self.admin = obj_build.make_admin_user(self.project.course)
self.ag_test_suite = obj_build.make_ag_test_suite(project=self.project)
self.ag_test_case = obj_build.make_ag_test_case(
ag_test_suite=self.ag_test_suite)
self.ag_test_cmd = obj_build.make_full_ag_test_command(
ag_test_case=self.ag_test_case)
self.student_group = obj_build.make_group(project=self.project,
num_members=2)
self.student_submission = obj_build.make_finished_submission(
self.student_group)
self.student_result = obj_build.make_incorrect_ag_test_command_result(
ag_test_command=self.ag_test_cmd,
submission=self.student_submission)
self.student_submission = update_denormalized_ag_test_results(
self.student_submission.pk)
self.student_result_fdbk = SubmissionResultFeedback(
self.student_submission, ag_models.FeedbackCategory.max,
AGTestPreLoader(self.project))
self.assertEqual(0, self.student_result_fdbk.total_points)
self.assertNotEqual(0, self.student_result_fdbk.total_points_possible)
self.staff_group = obj_build.make_group(
project=self.project, members_role=obj_build.UserRole.admin)
self.staff_submission = obj_build.make_finished_submission(
self.staff_group)
self.staff_result = obj_build.make_correct_ag_test_command_result(
ag_test_command=self.ag_test_cmd, submission=self.staff_submission)
self.staff_submission = update_denormalized_ag_test_results(
self.staff_submission.pk)
self.staff_result_fdbk = SubmissionResultFeedback(
self.staff_submission, ag_models.FeedbackCategory.max,
AGTestPreLoader(self.project))
self.assertNotEqual(0, self.staff_result_fdbk.total_points)
self.assertNotEqual(0, self.staff_result_fdbk.total_points_possible)
# Make sure we use the right queryset
other_project = obj_build.make_project(course=self.project.course)
other_group = obj_build.make_group(project=other_project)
other_submission = obj_build.make_finished_submission(other_group)
def test_group_member_names_in_environment(self, *args):
suite = obj_build.make_ag_test_suite(self.project)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(case, cmd='bash -c "printf $usernames"')
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertEqual(' '.join(self.submission.group.member_names),
open(res.stdout_filename).read())
def test_multiple_suites_cases_and_cmds(self, *args):
suite1 = obj_build.make_ag_test_suite(self.project)
case1 = obj_build.make_ag_test_case(suite1)
case2 = obj_build.make_ag_test_case(suite1)
suite2 = obj_build.make_ag_test_suite(self.project)
case3 = obj_build.make_ag_test_case(suite2)
case4 = obj_build.make_ag_test_case(suite2)
print_to_stdout_and_stderr = "bash -c 'printf hello; printf whoops >&2'"
for case in case1, case2, case3, case4:
for i in range(2):
obj_build.make_full_ag_test_command(
ag_test_case=case,
cmd=print_to_stdout_and_stderr,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
points_for_correct_return_code=4,
points_for_correct_stdout=1,
points_for_correct_stderr=1,
expected_return_code=ag_models.ExpectedReturnCode.zero,
expected_stdout_source=ag_models.ExpectedOutputSource.text,
expected_stdout_text="hello",
expected_stderr_source=ag_models.ExpectedOutputSource.text,
expected_stderr_text="whoops")
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
cmd_results = ag_models.AGTestCommandResult.objects.filter(
ag_test_case_result__ag_test_suite_result__submission=self.submission)
for res in cmd_results:
self.assertEqual(0, res.return_code)
self.assertEqual('hello', open(res.stdout_filename).read())
self.assertEqual('whoops', open(res.stderr_filename).read())
self.assertTrue(res.stdout_correct)
self.assertTrue(res.stderr_correct)
self.assertEqual(
48,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
def test_network_access_allowed_in_suite(self, *args):
suite1 = obj_build.make_ag_test_suite(self.project, allow_network_access=True)
case1 = obj_build.make_ag_test_case(suite1)
cmd = obj_build.make_full_ag_test_command(
case1, cmd='ping -c 2 www.google.com',
expected_return_code=ag_models.ExpectedReturnCode.zero)
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertTrue(res.return_code_correct)
def test_admin_valid_update(self):
other_suite = obj_build.make_ag_test_suite(project=self.ag_test_case.ag_test_suite.project)
patch_data = {
'name': 'ncvai',
'ag_test_suite': other_suite.pk,
'normal_fdbk_config': {
'show_individual_commands': False
}
}
[admin] = obj_build.make_admin_users(self.course, 1)
self.do_patch_object_test(self.ag_test_case, self.client, admin, self.url, patch_data)
def setUp(self):
super().setUp()
self.submission = obj_build.make_submission()
self.project = self.submission.group.project
self.setup_stdout = 'setuppy stdouty'
self.setup_stderr = 'setuppy stderrrry'
self.ag_test_suite = obj_build.make_ag_test_suite(
self.project,
setup_suite_cmd='bash -c "printf \'{}\'; printf \'{}\' >&2"'.
format(self.setup_stdout, self.setup_stderr))
self.ag_test_case = obj_build.make_ag_test_case(self.ag_test_suite)
def test_error_some_ag_test_suites_not_in_project(self):
other_ag_test_suite = obj_build.make_ag_test_suite()
with self.assertRaises(exceptions.ValidationError) as cm:
ag_models.RerunSubmissionsTask.objects.validate_and_create(
creator=self.creator,
project=self.project,
rerun_all_ag_test_suites=False,
ag_test_suite_data={str(other_ag_test_suite.pk): []})
self.assertIn('ag_test_suite_data', cm.exception.message_dict)
def test_submission_removed_from_queue(self, *args):
suite = obj_build.make_ag_test_suite(self.project, deferred=True)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(case)
self.submission.status = ag_models.Submission.GradingStatus.removed_from_queue
self.submission.save()
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(ag_models.Submission.GradingStatus.removed_from_queue,
self.submission.status)
def setUp(self):
self.submission = obj_build.make_submission()
self.project = self.submission.group.project
self.ag_suite = obj_build.make_ag_test_suite(self.project)
# Make sure that ag tests and ag test results don't have
# overlapping pks.
for i in range(20):
self.ag_suite.delete()
self.ag_suite.pk = None
self.ag_suite.save()
def setUp(self):
super().setUp()
self.project = obj_build.make_project()
[self.creator] = obj_build.make_admin_users(self.project.course, 1)
self.ag_test_suite = obj_build.make_ag_test_suite(self.project)
self.ag_test_case = obj_build.make_ag_test_case(self.ag_test_suite)
self.student_test_suite = obj_build.make_student_test_suite(
self.project)
self.submission = obj_build.make_submission(group=obj_build.make_group(
project=self.project))
def test_run_grade_submission_on_already_graded_submission(self, *args):
suite = obj_build.make_ag_test_suite(self.project)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(case, cmd='printf hello')
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertEqual('hello', open(res.stdout_filename).read())
cmd.cmd = 'printf weee'
cmd.save()
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertEqual('weee', open(res.stdout_filename).read())
def test_deferred_ag_test_error(self, *args):
suite = obj_build.make_ag_test_suite(self.project, deferred=True)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(
case,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=3)
with self.assertRaises(tasks.MaxRetriesExceeded):
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(ag_models.Submission.GradingStatus.error,
self.submission.status)
self.assertTrue(self.submission.error_msg.find('MaxRetriesExceeded') != -1)
def prepare_data(self,
project: ag_models.Project,
num_suites=1,
cases_per_suite=1,
cmds_per_suite=1,
num_groups=1,
num_other_submissions=0) -> List[GroupAndSubmissionData]:
cmds = []
for i in range(num_suites):
sys.stdout.write('\rBuilding suite {}'.format(i))
sys.stdout.flush()
suite = obj_build.make_ag_test_suite(project)
for j in range(cases_per_suite):
case = obj_build.make_ag_test_case(suite)
for k in range(cmds_per_suite):
cmd = obj_build.make_full_ag_test_command(case)
cmds.append(cmd)
def setUp(self):
self.submission = obj_build.make_submission()
self.project = self.submission.group.project
self.ag_suite = obj_build.make_ag_test_suite(self.project)
self.ag_suite = ag_models.AGTestSuite.objects.validate_and_create(
name='kajsdhf', project=self.project)
self.case = obj_build.make_ag_test_case(self.ag_suite)
# Make sure that ag tests and ag test results don't have
# overlapping pks.
for i in range(20):
self.case.delete()
self.case.pk = None
self.case.save()
self.suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create(
ag_test_suite=self.ag_suite, submission=self.submission
)
def setUp(self):
super().setUp()
self.maxDiff = None
self.project = obj_build.make_project(
# Future closing_time
closing_time=timezone.now() + datetime.timedelta(days=1))
self.course = self.project.course
self.ag_test_suite = obj_build.make_ag_test_suite(self.project)
self.ag_test_case = obj_build.make_ag_test_case(self.ag_test_suite)
self.ag_test_cmd = obj_build.make_full_ag_test_command(
self.ag_test_case)
self.student_test_suite = obj_build.make_student_test_suite(
self.project)
self.ag_test_preloader = AGTestPreLoader(self.project)
def test_one_suite_one_case_one_cmd(self, *args):
suite = obj_build.make_ag_test_suite(self.project)
case = obj_build.make_ag_test_case(suite)
print_to_stdout_and_stderr = "bash -c 'printf hello; printf whoops >&2'"
cmd = obj_build.make_full_ag_test_command(
case,
cmd=print_to_stdout_and_stderr,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
points_for_correct_return_code=4,
points_for_correct_stdout=1,
points_for_correct_stderr=1,
expected_return_code=ag_models.ExpectedReturnCode.zero,
expected_stdout_source=ag_models.ExpectedOutputSource.text,
expected_stdout_text="hello",
expected_stderr_source=ag_models.ExpectedOutputSource.text,
expected_stderr_text="whoops")
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
cmd_result = ag_models.AGTestCommandResult.objects.get(
ag_test_command=cmd,
ag_test_case_result__ag_test_suite_result__submission=self.submission)
with open(cmd_result.stderr_filename) as f:
output = f.read()
print(output)
self.assertEqual(0, cmd_result.return_code, msg=output)
self.assertEqual('hello', open(cmd_result.stdout_filename).read())
self.assertEqual('whoops', open(cmd_result.stderr_filename).read())
self.assertTrue(cmd_result.stdout_correct)
self.assertTrue(cmd_result.stderr_correct)
self.assertEqual(
6,
get_submission_fdbk(
self.submission, ag_models.FeedbackCategory.max).total_points_possible)
self.assertEqual(
6,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
def do_get_ultimate_submission_test(self,
projects,
group_funcs,
users,
closing_time,
extension=None,
hide_ultimates=False):
for project in projects:
project.validate_and_update(
closing_time=closing_time,
hide_ultimate_submission_fdbk=hide_ultimates)
for group_func in group_funcs:
group = group_func(project)
group.validate_and_update(extended_due_date=extension)
suite = obj_build.make_ag_test_suite(project)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(case)
best_submission = obj_build.make_finished_submission(
group=group)
most_recent_submission = obj_build.make_finished_submission(
group=group)
obj_build.make_correct_ag_test_command_result(
cmd, submission=best_submission)
obj_build.make_incorrect_ag_test_command_result(
cmd, submission=most_recent_submission)
for user in users:
url = self.ultimate_submission_url(group)
project.validate_and_update(
ultimate_submission_policy=ag_models.
UltimateSubmissionPolicy.most_recent)
self.do_get_object_test(self.client, user, url,
most_recent_submission.to_dict())
project.validate_and_update(
ultimate_submission_policy=ag_models.
UltimateSubmissionPolicy.best)
self.do_get_object_test(self.client, user, url,
best_submission.to_dict())
def setUp(self):
# 1. Create an AGTestSuite with 2 test cases (one command per test).
# 2. Each test case should be configured to fail when they are run.
# 3. Grade the AGTestSuite, letting both tests fail.
# 4. Update the test cases so that they will pass when they are rerun.
super().setUp()
self.submission = obj_build.make_submission()
self.project = self.submission.group.project
self.ag_test_suite = obj_build.make_ag_test_suite(self.project)
self.ag_test_case_1 = obj_build.make_ag_test_case(self.ag_test_suite)
self.ag_test_cmd_1 = ag_models.AGTestCommand.objects.validate_and_create(
ag_test_case=self.ag_test_case_1,
name='cmd1',
cmd='false', # Always exits nonzero
expected_return_code=ag_models.ExpectedReturnCode.zero
) # type: ag_models.AGTestCommand
self.ag_test_case_2 = obj_build.make_ag_test_case(self.ag_test_suite)
self.ag_test_cmd_2 = ag_models.AGTestCommand.objects.validate_and_create(
ag_test_case=self.ag_test_case_2,
name='cmd2',
cmd='false',
expected_return_code=ag_models.ExpectedReturnCode.zero
) # type: ag_models.AGTestCommand
# Reverse the order the test cases are run in so that the test cases
# and test case results don't have the same pk's
self.ag_test_suite.set_agtestcase_order(
self.ag_test_suite.get_agtestcase_order()[::-1])
tasks.grade_ag_test_suite_impl(self.ag_test_suite, self.submission)
results = ag_models.AGTestCommandResult.objects.all()
self.assertEqual(2, results.count())
for res in results:
self.assertFalse(res.return_code_correct)
self.ag_test_cmd_1.validate_and_update(cmd='true') # Always exits zero
self.ag_test_cmd_2.validate_and_update(cmd='true')
def test_delete_suite_result_denormed_results_updated(self):
suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create(
ag_test_suite=self.ag_suite, submission=self.submission)
self.assertNotEqual(suite_result.pk, self.ag_suite.pk)
dont_delete_suite = obj_build.make_ag_test_suite(self.project)
dont_delete_suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create(
ag_test_suite=dont_delete_suite, submission=self.submission)
self.assertNotEqual(dont_delete_suite.pk, dont_delete_suite_result.pk)
submission2 = obj_build.make_submission(group=obj_build.make_group(
project=self.project))
suite_result2 = ag_models.AGTestSuiteResult.objects.validate_and_create(
ag_test_suite=self.ag_suite, submission=submission2)
self.assertNotEqual(suite_result2.pk, self.ag_suite.pk)
self.submission = update_denormalized_ag_test_results(
self.submission.pk)
self.assertIn(str(dont_delete_suite.pk),
self.submission.denormalized_ag_test_results)
submission2 = update_denormalized_ag_test_results(submission2.pk)
for submission, suite_res in ((self.submission, suite_result),
(submission2, suite_result2)):
submission.refresh_from_db()
self.assertIn(str(suite_res.ag_test_suite_id),
submission.denormalized_ag_test_results)
self.ag_suite.delete()
for submission, suite_res in ((self.submission, suite_result),
(submission2, suite_result2)):
submission.refresh_from_db()
self.assertNotIn(str(suite_res.ag_test_suite_id),
submission.denormalized_ag_test_results)
self.submission.refresh_from_db()
self.assertIn(str(dont_delete_suite.pk),
self.submission.denormalized_ag_test_results)
def setUp(self):
super().setUp()
self.submission = obj_build.make_submission()
self.project = self.submission.group.project
self.ag_test_suite = obj_build.make_ag_test_suite(self.project)
self.ag_test_case = obj_build.make_ag_test_case(self.ag_test_suite)
self.too_much_output_size = 20000000 # 20 MB
too_much_output_prog = """
import sys
print('a' * {0}, end='')
print('b' * {0}, file=sys.stderr, end='')
""".format(self.too_much_output_size)
self.too_much_output_file = ag_models.InstructorFile.objects.validate_and_create(
project=self.project,
file_obj=SimpleUploadedFile('too_long.py',
too_much_output_prog.encode()
)) # type: ag_models.InstructorFile
self.timeout_cmd = "sleep 10"
self.ag_test_suite.instructor_files_needed.add(
self.too_much_output_file)
self.non_utf_bytes = b'\x80 and some other stuff just because\n'
non_utf_prog = """
import sys
sys.stdout.buffer.write({0})
sys.stdout.flush()
sys.stderr.buffer.write({0})
sys.stderr.flush()
""".format(self.non_utf_bytes)
self.non_utf_file = ag_models.InstructorFile.objects.validate_and_create(
project=self.project,
file_obj=SimpleUploadedFile('non_utf.py', non_utf_prog.encode()))
self.ag_test_suite.instructor_files_needed.add(self.non_utf_file)
def test_deferred_retry_on_error(self, impl_mock, *args):
suite = obj_build.make_ag_test_suite(self.project, deferred=True)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(
case,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=3)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertEqual(0, res.return_code)
self.assertTrue(res.return_code_correct)
self.assertEqual(
3,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
def test_copy_project(self):
# In new project, hide_ultimate_submission_fdbk should be set to True,
# visible_to_students should be set to False,
# and guests_can_submit should be set to False
project = obj_build.make_project(hide_ultimate_submission_fdbk=False,
visible_to_students=True)
instructor_file1 = obj_build.make_instructor_file(project)
instructor_file2 = obj_build.make_instructor_file(project)
student_file1 = obj_build.make_expected_student_file(project)
student_file2 = obj_build.make_expected_student_file(project)
suite1 = obj_build.make_ag_test_suite(
project,
instructor_files_needed=[instructor_file1],
student_files_needed=[student_file1])
case1 = obj_build.make_ag_test_case(suite1)
cmd1 = obj_build.make_full_ag_test_command(
case1,
expected_stderr_source=ag_models.ExpectedOutputSource.
instructor_file,
expected_stderr_instructor_file=instructor_file2)
cmd2 = obj_build.make_full_ag_test_command(
case1,
set_arbitrary_points=False,
expected_stdout_source=ag_models.ExpectedOutputSource.
instructor_file,
expected_stdout_instructor_file=instructor_file1)
suite2 = obj_build.make_ag_test_suite(
project,
instructor_files_needed=[instructor_file1, instructor_file2],
student_files_needed=[student_file2])
case2 = obj_build.make_ag_test_case(suite2)
cmd3 = obj_build.make_full_ag_test_command(
case2,
set_arbitrary_expected_vals=False,
stdin_source=ag_models.StdinSource.instructor_file,
stdin_instructor_file=instructor_file2)
case3 = obj_build.make_ag_test_case(suite2)
suite3 = obj_build.make_ag_test_suite(project)
student_suite1 = obj_build.make_student_test_suite(
project,
instructor_files_needed=[instructor_file1, instructor_file2],
student_files_needed=[student_file1],
setup_command={
'name': 'stave',
'cmd': 'yorp'
})
student_suite2 = obj_build.make_student_test_suite(
project,
instructor_files_needed=[instructor_file1],
student_files_needed=[student_file1, student_file2])
student_suite3 = obj_build.make_student_test_suite(project)
other_course = obj_build.make_course()
new_project = copy_project(project, other_course)
self.assertTrue(new_project.hide_ultimate_submission_fdbk)
self.assertFalse(new_project.visible_to_students)
self.assertEqual(project.name, new_project.name)
self.assertEqual(other_course, new_project.course)
self.assertNotEqual(project.course, other_course)
ignore_fields = [
'pk', 'course', 'last_modified', 'instructor_files',
'expected_student_files'
]
expected_ag_tests = _pop_many(project.to_dict(), ignore_fields)
expected_ag_tests.update({
'visible_to_students': False,
'hide_ultimate_submission_fdbk': True
})
self.assertEqual(expected_ag_tests,
_pop_many(new_project.to_dict(), ignore_fields))
self.assertEqual(project.instructor_files.count(),
new_project.instructor_files.count())
for old_file, new_file in itertools.zip_longest(
sorted(project.instructor_files.all(),
key=lambda obj: obj.name),
sorted(new_project.instructor_files.all(),
key=lambda obj: obj.name)):
self.assertNotEqual(new_file.pk, old_file.pk)
self.assertNotEqual(new_file.abspath, old_file.abspath)
self.assertEqual(old_file.name, new_file.name)
with old_file.open() as old_f, new_file.open() as new_f:
self.assertEqual(old_f.read(), new_f.read())
self.assertEqual(project.expected_student_files.count(),
new_project.expected_student_files.count())
for old_expected_file, new_expected_file in itertools.zip_longest(
project.expected_student_files.order_by('pattern'),
new_project.expected_student_files.order_by('pattern')):
self.assertNotEqual(old_expected_file.pk, new_expected_file.pk)
self.assertEqual(
_pop_many(old_expected_file.to_dict(), ['pk', 'project']),
_pop_many(new_expected_file.to_dict(), ['pk', 'project']))
old_suite_pks = {suite.pk for suite in project.ag_test_suites.all()}
new_suite_pks = {
suite.pk
for suite in new_project.ag_test_suites.all()
}
self.assertTrue(old_suite_pks.isdisjoint(new_suite_pks))
old_case_pks = {
case.pk
for case in ag_models.AGTestCase.objects.filter(
ag_test_suite__project=project)
}
def test_serialize(self):
ag_test_suite = obj_build.make_ag_test_suite()
self.do_basic_serialize_test(ag_test_suite,
ag_serializers.AGTestSuiteSerializer)
