def test_diff_ignore_case_whitespace_changes_and_blank_lines(self, *args):
cmd = obj_build.make_full_ag_test_command(
self.ag_test_case,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
cmd=
"""bash -c 'printf "HELLO world\n\n\n"; printf "lol WUT\n\n" >&2'""",
expected_stdout_source=ag_models.ExpectedOutputSource.text,
expected_stdout_text='hello world\n',
expected_stderr_source=ag_models.ExpectedOutputSource.text,
expected_stderr_text='lol wut\n',
points_for_correct_stdout=4,
points_for_correct_stderr=2,
ignore_case=True,
ignore_whitespace_changes=True,
ignore_blank_lines=True)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertTrue(res.stdout_correct)
self.assertTrue(res.stderr_correct)
self.assertEqual(
6,
get_submission_fdbk(self.submission,
ag_models.FeedbackCategory.max).total_points)
def test_non_default_docker_image(self, *args):
suite = obj_build.make_ag_test_suite(
self.project, docker_image_to_use=constants.SupportedImages.eecs490)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(
case,
cmd='racket --version',
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
points_for_correct_return_code=3,
expected_return_code=ag_models.ExpectedReturnCode.zero)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
cmd_result = ag_models.AGTestCommandResult.objects.get(
ag_test_command=cmd,
ag_test_case_result__ag_test_suite_result__submission=self.submission)
self.assertEqual(0, cmd_result.return_code)
self.assertEqual(
3,
get_submission_fdbk(
self.submission, ag_models.FeedbackCategory.max).total_points_possible)
self.assertEqual(
3,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
def test_points_awarded_and_deducted(self, *args):
cmd = obj_build.make_full_ag_test_command(
self.ag_test_case,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
cmd='printf hello',
expected_return_code=ag_models.ExpectedReturnCode.nonzero,
expected_stdout_source=ag_models.ExpectedOutputSource.text,
expected_stdout_text='hello',
deduction_for_wrong_return_code=-1,
points_for_correct_stdout=3)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertTrue(res.stdout_correct)
self.assertFalse(res.return_code_correct)
self.assertEqual(
2,
get_submission_fdbk(self.submission,
ag_models.FeedbackCategory.max).total_points)
self.assertEqual(
3,
get_submission_fdbk(
self.submission,
ag_models.FeedbackCategory.max).total_points_possible)
def test_setup_command_times_out_no_tests_discovered(self, *args):
self.student_suite.setup_command.validate_and_update(cmd='sleep 10')
with mock.patch('autograder.core.constants.MAX_SUBPROCESS_TIMEOUT', new=1):
tasks.grade_submission(self.submission.pk)
result = ag_models.StudentTestSuiteResult.objects.get(
student_test_suite=self.student_suite)
self.assertTrue(result.setup_result.timed_out)
self.assertEqual([], result.bugs_exposed)
self.assertEqual([], result.student_tests)
self.assertEqual([], result.invalid_tests)
self.assertEqual([], result.timed_out_tests)
with open(result.get_test_names_result.stdout_filename) as f:
self.assertEqual('', f.read())
with open(result.get_test_names_result.stderr_filename) as f:
self.assertEqual('', f.read())
with open(result.validity_check_stdout_filename) as f:
self.assertEqual('', f.read())
with open(result.validity_check_stderr_filename) as f:
self.assertEqual('', f.read())
with open(result.grade_buggy_impls_stdout_filename) as f:
self.assertEqual('', f.read())
with open(result.grade_buggy_impls_stderr_filename) as f:
self.assertEqual('', f.read())
def test_non_unicode_chars_in_test_names(self, *args):
non_unicode = b'test\x80 test2 test3'
escaped_names = non_unicode.decode(errors='backslashreplace').split()
instructor_file = ag_models.InstructorFile.objects.validate_and_create(
file_obj=SimpleUploadedFile('test_names', non_unicode),
project=self.project)
student_suite = ag_models.StudentTestSuite.objects.validate_and_create(
name='qeoriuqewrpqiuerqopwr',
project=self.project,
instructor_files_needed=[instructor_file],
get_student_test_names_command={
'cmd': 'cat {}'.format(instructor_file.name)
}
)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
result = ag_models.StudentTestSuiteResult.objects.get(student_test_suite=student_suite)
self.assertIsNone(result.setup_result)
self.assertEqual(0, result.get_test_names_result.return_code)
self.assertSequenceEqual(escaped_names, result.student_tests)
self.assertSequenceEqual([], result.invalid_tests)
def test_suite_setup_return_code_set(self, *args):
self.ag_test_suite.validate_and_update(
setup_suite_cmd='bash -c "exit 2"')
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestSuiteResult.objects.get(
submission=self.submission)
self.assertEqual(2, res.setup_return_code)
def test_setup_command_fails_no_tests_discovered(self, *args):
self.student_suite.setup_command.validate_and_update(cmd='false')
tasks.grade_submission(self.submission.pk)
result = ag_models.StudentTestSuiteResult.objects.get(
student_test_suite=self.student_suite)
self.assertNotEqual(0, result.setup_result.return_code)
self.assertEqual([], result.bugs_exposed)
self.assertEqual([], result.student_tests)
self.assertEqual([], result.invalid_tests)
self.assertEqual([], result.timed_out_tests)
with open(result.get_test_names_result.stdout_filename) as f:
self.assertEqual('', f.read())
with open(result.get_test_names_result.stderr_filename) as f:
self.assertEqual('', f.read())
with open(result.validity_check_stdout_filename) as f:
self.assertEqual('', f.read())
with open(result.validity_check_stderr_filename) as f:
self.assertEqual('', f.read())
with open(result.grade_buggy_impls_stdout_filename) as f:
self.assertEqual('', f.read())
with open(result.grade_buggy_impls_stderr_filename) as f:
self.assertEqual('', f.read())
def test_one_ag_suite_one_non_deferred_student_suite_one_deferred_student_suite(self, *args):
ag_suite = obj_build.make_ag_test_suite(self.project)
ag_case = obj_build.make_ag_test_case(ag_suite)
print_to_stdout_and_stderr = "bash -c 'printf hello; printf whoops >&2'"
ag_cmd = obj_build.make_full_ag_test_command(ag_case, cmd=print_to_stdout_and_stderr)
student_suite = ag_models.StudentTestSuite.objects.validate_and_create(
name='mnkfoae',
project=self.project)
deferred_student_suite = ag_models.StudentTestSuite.objects.validate_and_create(
name='deferryyyy',
project=self.project,
deferred=True)
tasks.grade_submission(self.submission.pk)
cmd_result = ag_models.AGTestCommandResult.objects.get(
ag_test_command=ag_cmd,
ag_test_case_result__ag_test_suite_result__submission=self.submission)
student_suite_result = ag_models.StudentTestSuiteResult.objects.get(
submission=self.submission,
student_test_suite=student_suite)
deferred_student_suite_result = ag_models.StudentTestSuiteResult.objects.get(
submission=self.submission,
student_test_suite=deferred_student_suite)
def test_all_suites_deferred(self, *args):
suite1 = obj_build.make_ag_test_suite(self.project, deferred=True)
case1 = obj_build.make_ag_test_case(suite1)
cmd1 = obj_build.make_full_ag_test_command(
case1,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=1)
suite2 = obj_build.make_ag_test_suite(self.project, deferred=True)
case2 = obj_build.make_ag_test_case(suite2)
cmd2 = obj_build.make_full_ag_test_command(
case2,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=2)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(
3,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
def test_one_ag_suite_deferred_one_student_suite_deferred(self, *args):
suite1 = obj_build.make_ag_test_suite(self.project, deferred=False)
case1 = obj_build.make_ag_test_case(suite1)
cmd1 = obj_build.make_full_ag_test_command(
case1,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=1)
suite2 = obj_build.make_ag_test_suite(self.project, deferred=True)
case2 = obj_build.make_ag_test_case(suite2)
cmd2 = obj_build.make_full_ag_test_command(
case2,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=2)
deferred_student_suite = ag_models.StudentTestSuite.objects.validate_and_create(
name='deferryyyy',
project=self.project,
deferred=True)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(
3,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
deferred_student_suite_result = ag_models.StudentTestSuiteResult.objects.get(
submission=self.submission,
student_test_suite=deferred_student_suite)
def test_wrong_expected_return_code_nonzero(self, *args):
cmd = obj_build.make_full_ag_test_command(
self.ag_test_case,
cmd='bash -c "exit 0"',
expected_return_code=ag_models.ExpectedReturnCode.nonzero)
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertFalse(res.return_code_correct)
def test_stdin_source_setup_stderr(self, *args):
cmd = obj_build.make_full_ag_test_command(
self.ag_test_case,
cmd='cat',
stdin_source=ag_models.StdinSource.setup_stderr)
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertEqual(self.setup_stderr, open(res.stdout_filename).read())
def test_setup_print_non_unicode_chars(self, *args):
self.ag_test_suite.validate_and_update(setup_suite_cmd='python3 ' +
self.non_utf_file.name)
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestSuiteResult.objects.get(
submission=self.submission)
self.assertEqual(self.non_utf_bytes, res.open_setup_stdout().read())
self.assertEqual(self.non_utf_bytes, res.open_setup_stderr().read())
def test_group_member_names_in_environment(self, *args):
suite = obj_build.make_ag_test_suite(self.project)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(case, cmd='bash -c "printf $usernames"')
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertEqual(' '.join(self.submission.group.member_names),
open(res.stdout_filename).read())
def test_wrong_expected_stderr_text(self, *args):
cmd = obj_build.make_full_ag_test_command(
self.ag_test_case,
cmd='bash -c "printf nopers >&2"',
expected_stderr_source=ag_models.ExpectedOutputSource.text,
expected_stderr_text='hello')
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertFalse(res.stderr_correct)
def test_correct_expected_stdout_text(self, *args):
cmd = obj_build.make_full_ag_test_command(
self.ag_test_case,
cmd='printf "hello"',
expected_stdout_source=ag_models.ExpectedOutputSource.text,
expected_stdout_text='hello')
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertTrue(res.stdout_correct)
def test_network_access_allowed_in_suite(self, *args):
suite1 = obj_build.make_ag_test_suite(self.project, allow_network_access=True)
case1 = obj_build.make_ag_test_case(suite1)
cmd = obj_build.make_full_ag_test_command(
case1, cmd='ping -c 2 www.google.com',
expected_return_code=ag_models.ExpectedReturnCode.zero)
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertTrue(res.return_code_correct)
def test_setup_time_out(self, *args):
self.ag_test_suite.validate_and_update(
setup_suite_cmd=self.timeout_cmd)
with mock.patch('autograder.core.constants.MAX_SUBPROCESS_TIMEOUT',
new=1):
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestSuiteResult.objects.get(
submission=self.submission)
self.assertTrue(res.setup_timed_out)
def test_submission_removed_from_queue(self, *args):
suite = obj_build.make_ag_test_suite(self.project, deferred=True)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(case)
self.submission.status = ag_models.Submission.GradingStatus.removed_from_queue
self.submission.save()
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(ag_models.Submission.GradingStatus.removed_from_queue,
self.submission.status)
def test_program_times_out(self, *args):
cmd = obj_build.make_full_ag_test_command(
self.ag_test_case,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
cmd=self.timeout_cmd,
time_limit=1)
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertTrue(res.timed_out)
def test_project_files_not_read_only(self, *args):
self.ag_suite.validate_and_update(read_only_instructor_files=False)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(
self.retcode_points,
get_submission_fdbk(self.submission,
ag_models.FeedbackCategory.max).total_points)
self.assertEqual(
self.retcode_points,
get_submission_fdbk(
self.submission,
ag_models.FeedbackCategory.max).total_points_possible)
def test_grade_non_deferred(self, *args):
tasks.grade_submission(self.submission.pk)
result = ag_models.StudentTestSuiteResult.objects.get(
student_test_suite=self.student_suite)
self.assertEqual(0, result.setup_result.return_code)
print(result.student_tests)
print(result.invalid_tests)
self.assertCountEqual(self.all_tests, result.student_tests)
self.assertCountEqual(self.invalid_tests, result.invalid_tests)
self.assertCountEqual(self.timeout_tests, result.timed_out_tests)
self.assertCountEqual(self.bugs_exposed, result.bugs_exposed)
self.assertEqual(0, result.get_test_names_result.return_code)
with open(result.get_test_names_result.stdout_filename) as f:
print('get_test_names_result.stdout_filename')
print(f.read())
with open(result.get_test_names_result.stderr_filename) as f:
print('get_test_names_result.stderr_filename')
print(f.read())
with open(result.setup_result.stdout_filename) as f:
print('setup_result stdout')
print(f.read())
with open(result.setup_result.stderr_filename) as f:
print('setup_result stderr')
print(f.read())
with open(result.get_test_names_result.stdout_filename) as f:
print('get_test_names_result stdout')
print(f.read())
with open(result.get_test_names_result.stderr_filename) as f:
print('get_test_names_result stderr')
print(f.read())
with open(result.validity_check_stdout_filename) as f:
print('open_validity_check_stdout')
print(f.read())
with open(result.validity_check_stderr_filename) as f:
print('open_validity_check_stderr')
print(f.read())
with open(result.grade_buggy_impls_stdout_filename) as f:
print('open_grade_buggy_impls_stdout')
print(f.read())
with open(result.grade_buggy_impls_stderr_filename) as f:
print('open_grade_buggy_impls_stderr')
print(f.read())
def test_setup_print_a_lot_of_output(self, *args):
self.ag_test_suite.validate_and_update(setup_suite_cmd='python3 ' +
self.too_much_output_file.name)
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestSuiteResult.objects.get(
submission=self.submission)
self.assertTrue(res.setup_stdout_truncated)
self.assertTrue(res.setup_stderr_truncated)
self.assertEqual(constants.MAX_OUTPUT_LENGTH,
os.path.getsize(res.setup_stdout_filename))
self.assertEqual(constants.MAX_OUTPUT_LENGTH,
os.path.getsize(res.setup_stderr_filename))
def test_program_prints_non_unicode_chars(self, *args):
cmd = obj_build.make_full_ag_test_command(
self.ag_test_case,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
cmd='python3 ' + self.non_utf_file.name)
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertEqual(0, res.return_code)
self.assertEqual(self.non_utf_bytes,
open(res.stdout_filename, 'rb').read())
self.assertEqual(self.non_utf_bytes,
open(res.stderr_filename, 'rb').read())
def test_stdin_source_instructor_file(self, *args):
text = ',vnaejfal;skjdf;lakjsdfklajsl;dkjf;'
instructor_file = ag_models.InstructorFile.objects.validate_and_create(
project=self.project,
file_obj=SimpleUploadedFile('filey.txt', text.encode()))
cmd = obj_build.make_full_ag_test_command(
self.ag_test_case,
cmd='cat',
stdin_source=ag_models.StdinSource.instructor_file,
stdin_instructor_file=instructor_file)
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertEqual(text, open(res.stdout_filename).read())
def test_wrong_expected_stderr_instructor_file(self, *args):
instructor_file = ag_models.InstructorFile.objects.validate_and_create(
project=self.project,
file_obj=SimpleUploadedFile('filey.txt', b'waluigi'))
cmd = obj_build.make_full_ag_test_command(
self.ag_test_case,
cmd='bash -c "printf norp >&2"',
expected_stderr_source=ag_models.ExpectedOutputSource.
instructor_file,
expected_stderr_instructor_file=instructor_file)
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertFalse(res.stderr_correct)
def test_run_grade_submission_on_already_graded_submission(self, *args):
suite = obj_build.make_ag_test_suite(self.project)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(case, cmd='printf hello')
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertEqual('hello', open(res.stdout_filename).read())
cmd.cmd = 'printf weee'
cmd.save()
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertEqual('weee', open(res.stdout_filename).read())
def test_grade_deferred(self, *args):
self.student_suite.validate_and_update(deferred=True)
tasks.grade_submission(self.submission.pk)
result = ag_models.StudentTestSuiteResult.objects.get(
student_test_suite=self.student_suite)
self.assertEqual(0, result.setup_result.return_code)
print(result.student_tests)
print(result.invalid_tests)
self.assertCountEqual(self.all_tests, result.student_tests)
self.assertCountEqual(self.invalid_tests, result.invalid_tests)
self.assertCountEqual(self.timeout_tests, result.timed_out_tests)
self.assertCountEqual(self.bugs_exposed, result.bugs_exposed)
def test_deferred_ag_test_error(self, *args):
suite = obj_build.make_ag_test_suite(self.project, deferred=True)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(
case,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=3)
with self.assertRaises(tasks.MaxRetriesExceeded):
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(ag_models.Submission.GradingStatus.error,
self.submission.status)
self.assertTrue(self.submission.error_msg.find('MaxRetriesExceeded') != -1)
def test_program_prints_a_lot_of_output(self, *args):
cmd = obj_build.make_full_ag_test_command(
self.ag_test_case,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
cmd='python3 ' + self.too_much_output_file.name,
time_limit=30)
tasks.grade_submission(self.submission.pk)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertEqual(0, res.return_code)
self.assertFalse(res.timed_out)
self.assertTrue(res.stdout_truncated)
self.assertTrue(res.stderr_truncated)
self.assertEqual(constants.MAX_OUTPUT_LENGTH,
os.path.getsize(res.stdout_filename))
self.assertEqual(constants.MAX_OUTPUT_LENGTH,
os.path.getsize(res.stderr_filename))