def test_file_io_sources_deserialized_on_update(self):
stdin = obj_build.make_instructor_file(self.project)
stdout = obj_build.make_instructor_file(self.project)
stderr = obj_build.make_instructor_file(self.project)
ag_cmd = ag_models.AGTestCommand.objects.validate_and_create(
name=self.name,
ag_test_case=self.ag_test,
cmd=self.cmd,
stdin_source=ag_models.StdinSource.instructor_file,
stdin_instructor_file=stdin,
expected_stdout_source=ag_models.ExpectedOutputSource.
instructor_file,
expected_stdout_instructor_file=stdout,
expected_stderr_source=ag_models.ExpectedOutputSource.
instructor_file,
expected_stderr_instructor_file=stderr)
self.assertEqual(stdin, ag_cmd.stdin_instructor_file)
self.assertEqual(stdout, ag_cmd.expected_stdout_instructor_file)
self.assertEqual(stderr, ag_cmd.expected_stderr_instructor_file)
ag_cmd.validate_and_update(
stdin_source=ag_models.StdinSource.none,
stdin_instructor_file=None,
expected_stdout_source=ag_models.ExpectedOutputSource.none,
expected_stdout_instructor_file=None,
expected_stderr_source=ag_models.ExpectedOutputSource.none,
expected_stderr_instructor_file=None,
)
self.assertIsNone(ag_cmd.stdin_instructor_file)
self.assertIsNone(ag_cmd.expected_stdout_instructor_file)
self.assertIsNone(ag_cmd.expected_stderr_instructor_file)
ag_cmd.validate_and_update(
stdin_source=ag_models.StdinSource.instructor_file,
stdin_instructor_file=stdin.to_dict(),
expected_stdout_source=ag_models.ExpectedOutputSource.
instructor_file,
expected_stdout_instructor_file=stdout.to_dict(),
expected_stderr_source=ag_models.ExpectedOutputSource.
instructor_file,
expected_stderr_instructor_file=stderr.to_dict())
self.assertEqual(stdin, ag_cmd.stdin_instructor_file)
self.assertEqual(stdout, ag_cmd.expected_stdout_instructor_file)
self.assertEqual(stderr, ag_cmd.expected_stderr_instructor_file)
def test_serialization(self):
student_file = ag_models.ExpectedStudentFile.objects.validate_and_create(
pattern='filey', project=self.project)
suite = ag_models.AGTestSuite.objects.validate_and_create(
name='a;dlskfj',
project=self.project,
instructor_files_needed=[
obj_build.make_instructor_file(self.project)
],
student_files_needed=[student_file],
setup_suite_cmd="echo 'hello world'",
allow_network_access=True,
deferred=True) # type: ag_models.AGTestSuite
ag_test = ag_models.AGTestCase.objects.validate_and_create(
name='asdfkajewfiao', ag_test_suite=suite)
suite_dict = suite.to_dict()
expected_keys = [
'pk',
'name',
'project',
'last_modified',
'instructor_files_needed',
'read_only_instructor_files',
'student_files_needed',
'ag_test_cases',
'setup_suite_cmd',
'setup_suite_cmd_name',
'docker_image_to_use',
'allow_network_access',
'deferred',
'normal_fdbk_config',
'ultimate_submission_fdbk_config',
'past_limit_submission_fdbk_config',
'staff_viewer_fdbk_config',
]
self.assertCountEqual(expected_keys, suite_dict.keys())
self.assertIsInstance(suite_dict['instructor_files_needed'][0], dict)
self.assertIsInstance(suite_dict['student_files_needed'][0], dict)
self.assertSequenceEqual([ag_test.to_dict()],
suite_dict['ag_test_cases'])
self.assertIsInstance(suite_dict['normal_fdbk_config'], dict)
self.assertIsInstance(suite_dict['ultimate_submission_fdbk_config'],
dict)
self.assertIsInstance(suite_dict['past_limit_submission_fdbk_config'],
dict)
self.assertIsInstance(suite_dict['staff_viewer_fdbk_config'], dict)
update_dict = copy.deepcopy(suite_dict)
for non_editable in [
'pk', 'project', 'last_modified', 'ag_test_cases'
]:
update_dict.pop(non_editable)
suite.validate_and_update(**update_dict)
def test_stderr_correctness_show_diff_from_file(self):
instructor_file = obj_build.make_instructor_file(self.project)
self.ag_test_command.expected_stderr_source = (
ag_models.ExpectedOutputSource.instructor_file)
self.ag_test_command.expected_stderr_instructor_file = instructor_file
self.ag_test_command.save()
result = self.make_correct_result()
with instructor_file.open() as f:
expected_stderr = f.read()
result.stderr = expected_stderr
result.save()
diff = _get_expected_diff(expected_stderr, result.stderr_filename)
self.assertEqual(
diff.diff_content,
get_cmd_fdbk(
result,
ag_models.FeedbackCategory.normal).stderr_diff.diff_content)
result.stderr = 'the wrong stderr'
result.save()
diff = _get_expected_diff(expected_stderr, result.stderr_filename)
self.assertEqual(
diff.diff_content,
get_cmd_fdbk(
result,
ag_models.FeedbackCategory.normal).stderr_diff.diff_content)
def test_serialize(self):
project = ag_models.Project.objects.validate_and_create(
name='qeiruqioewiur',
course=self.course) # type: ag_models.Project
instructor_file = obj_build.make_instructor_file(project)
student_file = ag_models.ExpectedStudentFile.objects.validate_and_create(
project=project, pattern='qweiourqpweioru')
project_dict = project.to_dict()
expected_keys = [
'pk',
'name',
'course',
'last_modified',
'visible_to_students',
'closing_time',
'soft_closing_time',
'disallow_student_submissions',
'disallow_group_registration',
'guests_can_submit',
'min_group_size',
'max_group_size',
'submission_limit_per_day',
'allow_submissions_past_limit',
'groups_combine_daily_submissions',
'submission_limit_reset_time',
'submission_limit_reset_timezone',
'num_bonus_submissions',
'total_submission_limit',
'allow_late_days',
'ultimate_submission_policy',
'hide_ultimate_submission_fdbk',
'instructor_files',
'expected_student_files',
]
self.assertCountEqual(expected_keys, project_dict.keys())
self.assertEqual('UTC',
project_dict['submission_limit_reset_timezone'])
self.assertSequenceEqual([instructor_file.to_dict()],
project_dict['instructor_files'])
self.assertSequenceEqual([student_file.to_dict()],
project_dict['expected_student_files'])
update_dict = copy.deepcopy(project_dict)
update_dict.pop('pk')
update_dict.pop('course')
update_dict.pop('last_modified')
update_dict.pop('instructor_files')
update_dict.pop('expected_student_files')
project.validate_and_update(**update_dict)
other_timezone = 'America/Chicago'
project.validate_and_update(
submission_limit_reset_timezone=other_timezone)
project.refresh_from_db()
self.assertEqual(other_timezone,
project.to_dict()['submission_limit_reset_timezone'])
def test_error_instructor_file_needed_that_belongs_to_other_project(self):
other_project = obj_build.make_project(course=self.project.course)
other_instructor_file = obj_build.make_instructor_file(
project=other_project)
with self.assertRaises(exceptions.ValidationError) as cm:
ag_models.StudentTestSuite.objects.validate_and_create(
name=self.name,
project=self.project,
instructor_files_needed=[other_instructor_file.pk])
self.assertIn('instructor_files_needed', cm.exception.message_dict)
def test_error_stdin_instructor_file_belongs_to_other_project(self):
other_project = obj_build.make_project(course=self.project.course)
other_instructor_file = obj_build.make_instructor_file(
project=other_project)
with self.assertRaises(exceptions.ValidationError) as cm:
ag_models.AGTestCommand.objects.validate_and_create(
name=self.name,
ag_test_case=self.ag_test,
cmd='true',
stdin_source=ag_models.StdinSource.instructor_file,
stdin_instructor_file=other_instructor_file)
self.assertIn('stdin_instructor_file', cm.exception.message_dict)
def test_file_io_sources_serialized(self):
stdin = obj_build.make_instructor_file(self.project)
stdout = obj_build.make_instructor_file(self.project)
stderr = obj_build.make_instructor_file(self.project)
ag_cmd = ag_models.AGTestCommand.objects.validate_and_create(
name=self.name,
ag_test_case=self.ag_test,
cmd=self.cmd,
stdin_instructor_file=stdin,
expected_stdout_instructor_file=stdout,
expected_stderr_instructor_file=stderr)
cmd_dict = ag_cmd.to_dict()
self.assertIsInstance(cmd_dict['stdin_instructor_file'], dict)
self.assertEqual(stdin.pk, cmd_dict['stdin_instructor_file']['pk'])
self.assertIsInstance(cmd_dict['expected_stdout_instructor_file'],
dict)
self.assertEqual(stdout.pk,
cmd_dict['expected_stdout_instructor_file']['pk'])
self.assertIsInstance(cmd_dict['expected_stderr_instructor_file'],
dict)
self.assertEqual(stderr.pk,
cmd_dict['expected_stderr_instructor_file']['pk'])
def test_error_instructor_and_student_files_dont_belong_to_same_project(
self):
other_project = obj_build.build_project()
other_instructor_file = obj_build.make_instructor_file(other_project)
other_student_file = ag_models.ExpectedStudentFile.objects.validate_and_create(
pattern='alsdnvaoweijf', project=other_project)
with self.assertRaises(exceptions.ValidationError) as cm:
ag_models.AGTestSuite.objects.validate_and_create(
name='aldksjfnaweij',
project=self.project,
instructor_files_needed=[other_instructor_file],
student_files_needed=[other_student_file])
self.assertIn('instructor_files_needed', cm.exception.message_dict)
self.assertIn('student_files_needed', cm.exception.message_dict)
def test_valid_create_non_defaults(self):
student_file = ag_models.ExpectedStudentFile.objects.validate_and_create(
pattern='filey', project=self.project)
name = 'wee'
project = self.project
instructor_files_needed = [
obj_build.make_instructor_file(self.project)
]
student_files_needed = [student_file]
setup_cmd = "echo 'hello world'"
allow_network_access = True
deferred = True
suite = ag_models.AGTestSuite.objects.validate_and_create(
name=name,
project=project,
instructor_files_needed=instructor_files_needed,
read_only_instructor_files=False,
student_files_needed=student_files_needed,
setup_suite_cmd=setup_cmd,
setup_suite_cmd_name='steve',
allow_network_access=allow_network_access,
deferred=deferred,
docker_image_to_use=constants.SupportedImages.eecs490,
normal_fdbk_config={
'visible': False,
'show_individual_tests': False,
'show_setup_return_code': False,
'show_setup_timed_out': False,
'show_setup_stdout': False,
'show_setup_stderr': False,
})
suite.refresh_from_db()
self.assertEqual(name, suite.name)
self.assertEqual(project, suite.project)
self.assertCountEqual(instructor_files_needed,
suite.instructor_files_needed.all())
self.assertFalse(suite.read_only_instructor_files)
self.assertCountEqual(student_files_needed,
suite.student_files_needed.all())
self.assertEqual(allow_network_access, suite.allow_network_access)
self.assertEqual(deferred, suite.deferred)
self.assertEqual(constants.SupportedImages.eecs490,
suite.docker_image_to_use)
self.assertFalse(suite.normal_fdbk_config.visible)
def test_copy_project(self):
# In new project, hide_ultimate_submission_fdbk should be set to True,
# visible_to_students should be set to False,
# and guests_can_submit should be set to False
project = obj_build.make_project(hide_ultimate_submission_fdbk=False,
visible_to_students=True)
instructor_file1 = obj_build.make_instructor_file(project)
instructor_file2 = obj_build.make_instructor_file(project)
student_file1 = obj_build.make_expected_student_file(project)
student_file2 = obj_build.make_expected_student_file(project)
suite1 = obj_build.make_ag_test_suite(
project,
instructor_files_needed=[instructor_file1],
student_files_needed=[student_file1])
case1 = obj_build.make_ag_test_case(suite1)
cmd1 = obj_build.make_full_ag_test_command(
case1,
expected_stderr_source=ag_models.ExpectedOutputSource.
instructor_file,
expected_stderr_instructor_file=instructor_file2)
cmd2 = obj_build.make_full_ag_test_command(
case1,
set_arbitrary_points=False,
expected_stdout_source=ag_models.ExpectedOutputSource.
instructor_file,
expected_stdout_instructor_file=instructor_file1)
suite2 = obj_build.make_ag_test_suite(
project,
instructor_files_needed=[instructor_file1, instructor_file2],
student_files_needed=[student_file2])
case2 = obj_build.make_ag_test_case(suite2)
cmd3 = obj_build.make_full_ag_test_command(
case2,
set_arbitrary_expected_vals=False,
stdin_source=ag_models.StdinSource.instructor_file,
stdin_instructor_file=instructor_file2)
case3 = obj_build.make_ag_test_case(suite2)
suite3 = obj_build.make_ag_test_suite(project)
student_suite1 = obj_build.make_student_test_suite(
project,
instructor_files_needed=[instructor_file1, instructor_file2],
student_files_needed=[student_file1],
setup_command={
'name': 'stave',
'cmd': 'yorp'
})
student_suite2 = obj_build.make_student_test_suite(
project,
instructor_files_needed=[instructor_file1],
student_files_needed=[student_file1, student_file2])
student_suite3 = obj_build.make_student_test_suite(project)
other_course = obj_build.make_course()
new_project = copy_project(project, other_course)
self.assertTrue(new_project.hide_ultimate_submission_fdbk)
self.assertFalse(new_project.visible_to_students)
self.assertEqual(project.name, new_project.name)
self.assertEqual(other_course, new_project.course)
self.assertNotEqual(project.course, other_course)
ignore_fields = [
'pk', 'course', 'last_modified', 'instructor_files',
'expected_student_files'
]
expected_ag_tests = _pop_many(project.to_dict(), ignore_fields)
expected_ag_tests.update({
'visible_to_students': False,
'hide_ultimate_submission_fdbk': True
})
self.assertEqual(expected_ag_tests,
_pop_many(new_project.to_dict(), ignore_fields))
self.assertEqual(project.instructor_files.count(),
new_project.instructor_files.count())
for old_file, new_file in itertools.zip_longest(
sorted(project.instructor_files.all(),
key=lambda obj: obj.name),
sorted(new_project.instructor_files.all(),
key=lambda obj: obj.name)):
self.assertNotEqual(new_file.pk, old_file.pk)
self.assertNotEqual(new_file.abspath, old_file.abspath)
self.assertEqual(old_file.name, new_file.name)
with old_file.open() as old_f, new_file.open() as new_f:
self.assertEqual(old_f.read(), new_f.read())
self.assertEqual(project.expected_student_files.count(),
new_project.expected_student_files.count())
for old_expected_file, new_expected_file in itertools.zip_longest(
project.expected_student_files.order_by('pattern'),
new_project.expected_student_files.order_by('pattern')):
self.assertNotEqual(old_expected_file.pk, new_expected_file.pk)
self.assertEqual(
_pop_many(old_expected_file.to_dict(), ['pk', 'project']),
_pop_many(new_expected_file.to_dict(), ['pk', 'project']))
old_suite_pks = {suite.pk for suite in project.ag_test_suites.all()}
new_suite_pks = {
suite.pk
for suite in new_project.ag_test_suites.all()
}
self.assertTrue(old_suite_pks.isdisjoint(new_suite_pks))
old_case_pks = {
case.pk
for case in ag_models.AGTestCase.objects.filter(
ag_test_suite__project=project)
}
def test_valid_init_non_defaults(self):
instructor_file1 = obj_build.make_instructor_file(self.project)
instructor_file2 = obj_build.make_instructor_file(self.project)
student_file = obj_build.make_expected_student_file(self.project)
values = {
'name':
'adnlakshfdklajhsdlf',
'project':
self.project,
'instructor_files_needed':
[instructor_file1.to_dict(),
instructor_file2.to_dict()],
'read_only_instructor_files':
False,
'student_files_needed': [student_file.to_dict()],
'buggy_impl_names': ['spam', 'egg', 'sausage', 'waaaaluigi'],
'setup_command': {
'cmd': 'g++ some_stuff.cpp'
},
'get_student_test_names_command': {
'cmd': 'ls test_*.cpp'
},
'max_num_student_tests':
30,
'student_test_validity_check_command': {
'cmd': 'python3 validity_check.py ${student_test_name}'
},
'grade_buggy_impl_command': {
'cmd':
'python3 grade.py ${buggy_impl_name} ${student_test_name}'
},
'points_per_exposed_bug':
42,
'max_points':
462,
'deferred':
True,
'docker_image_to_use':
constants.SupportedImages.eecs280,
'allow_network_access':
True,
'normal_fdbk_config': {
'bugs_exposed_fdbk_level':
(ag_models.BugsExposedFeedbackLevel.num_bugs_exposed.value),
'show_points':
True,
},
'ultimate_submission_fdbk_config': {
'show_setup_stdout': True,
'show_invalid_test_names': True,
},
'past_limit_submission_fdbk_config': {
'visible': False,
'show_setup_stderr': True,
},
'staff_viewer_fdbk_config': {
'show_points': False,
}
}
student_suite = ag_models.StudentTestSuite.objects.validate_and_create(
**values) # type: ag_models.StudentTestSuite
self.assertCountEqual([
instructor_file['pk']
for instructor_file in values['instructor_files_needed']
], [
instructor_file.pk
for instructor_file in student_suite.instructor_files_needed.all()
])
values.pop('instructor_files_needed')
self.assertCountEqual([
student_file['pk']
for student_file in values['student_files_needed']
], [
student_file.pk
for student_file in student_suite.student_files_needed.all()
])
values.pop('student_files_needed')
for key, value in values.items():
if isinstance(value, dict):
self.assert_dict_is_subset(
value,
getattr(student_suite, key).to_dict())
elif isinstance(value, list):
self.assertSequenceEqual(value, getattr(student_suite, key))
else:
self.assertEqual(value, getattr(student_suite, key))
def test_serialization(self):
expected_field_names = [
'pk',
'name',
'project',
'instructor_files_needed',
'read_only_instructor_files',
'student_files_needed',
'buggy_impl_names',
'use_setup_command',
'setup_command',
'get_student_test_names_command',
'max_num_student_tests',
'student_test_validity_check_command',
'grade_buggy_impl_command',
'points_per_exposed_bug',
'max_points',
'deferred',
'docker_image_to_use',
'allow_network_access',
'normal_fdbk_config',
'ultimate_submission_fdbk_config',
'past_limit_submission_fdbk_config',
'staff_viewer_fdbk_config',
'last_modified',
]
instructor_file = obj_build.make_instructor_file(self.project)
student_file = obj_build.make_expected_student_file(self.project)
student_suite = ag_models.StudentTestSuite.objects.validate_and_create(
name=self.name,
project=self.project,
setup_command={'cmd': 'setuppy'},
instructor_files_needed=[instructor_file],
student_files_needed=[student_file
]) # type: ag_models.StudentTestSuite
serialized = student_suite.to_dict()
self.assertCountEqual(expected_field_names, serialized.keys())
self.assertIsInstance(serialized['instructor_files_needed'][0], dict)
self.assertIsInstance(serialized['student_files_needed'][0], dict)
self.assertIsInstance(serialized['setup_command'], dict)
self.assertIsInstance(serialized['get_student_test_names_command'],
dict)
self.assertIsInstance(
serialized['student_test_validity_check_command'], dict)
self.assertIsInstance(serialized['grade_buggy_impl_command'], dict)
self.assertIsInstance(serialized['normal_fdbk_config'], dict)
self.assertIsInstance(serialized['ultimate_submission_fdbk_config'],
dict)
self.assertIsInstance(serialized['past_limit_submission_fdbk_config'],
dict)
self.assertIsInstance(serialized['staff_viewer_fdbk_config'], dict)
update_dict = student_suite.to_dict()
non_editable = ['pk', 'project', 'last_modified']
for field in non_editable:
update_dict.pop(field)
student_suite.validate_and_update(**update_dict)