def test_file_io_sources_deserialized_on_update(self): stdin = obj_build.make_instructor_file(self.project) stdout = obj_build.make_instructor_file(self.project) stderr = obj_build.make_instructor_file(self.project) ag_cmd = ag_models.AGTestCommand.objects.validate_and_create( name=self.name, ag_test_case=self.ag_test, cmd=self.cmd, stdin_source=ag_models.StdinSource.instructor_file, stdin_instructor_file=stdin, expected_stdout_source=ag_models.ExpectedOutputSource. instructor_file, expected_stdout_instructor_file=stdout, expected_stderr_source=ag_models.ExpectedOutputSource. instructor_file, expected_stderr_instructor_file=stderr) self.assertEqual(stdin, ag_cmd.stdin_instructor_file) self.assertEqual(stdout, ag_cmd.expected_stdout_instructor_file) self.assertEqual(stderr, ag_cmd.expected_stderr_instructor_file) ag_cmd.validate_and_update( stdin_source=ag_models.StdinSource.none, stdin_instructor_file=None, expected_stdout_source=ag_models.ExpectedOutputSource.none, expected_stdout_instructor_file=None, expected_stderr_source=ag_models.ExpectedOutputSource.none, expected_stderr_instructor_file=None, ) self.assertIsNone(ag_cmd.stdin_instructor_file) self.assertIsNone(ag_cmd.expected_stdout_instructor_file) self.assertIsNone(ag_cmd.expected_stderr_instructor_file) ag_cmd.validate_and_update( stdin_source=ag_models.StdinSource.instructor_file, stdin_instructor_file=stdin.to_dict(), expected_stdout_source=ag_models.ExpectedOutputSource. instructor_file, expected_stdout_instructor_file=stdout.to_dict(), expected_stderr_source=ag_models.ExpectedOutputSource. instructor_file, expected_stderr_instructor_file=stderr.to_dict()) self.assertEqual(stdin, ag_cmd.stdin_instructor_file) self.assertEqual(stdout, ag_cmd.expected_stdout_instructor_file) self.assertEqual(stderr, ag_cmd.expected_stderr_instructor_file)
def test_serialization(self): student_file = ag_models.ExpectedStudentFile.objects.validate_and_create( pattern='filey', project=self.project) suite = ag_models.AGTestSuite.objects.validate_and_create( name='a;dlskfj', project=self.project, instructor_files_needed=[ obj_build.make_instructor_file(self.project) ], student_files_needed=[student_file], setup_suite_cmd="echo 'hello world'", allow_network_access=True, deferred=True) # type: ag_models.AGTestSuite ag_test = ag_models.AGTestCase.objects.validate_and_create( name='asdfkajewfiao', ag_test_suite=suite) suite_dict = suite.to_dict() expected_keys = [ 'pk', 'name', 'project', 'last_modified', 'instructor_files_needed', 'read_only_instructor_files', 'student_files_needed', 'ag_test_cases', 'setup_suite_cmd', 'setup_suite_cmd_name', 'docker_image_to_use', 'allow_network_access', 'deferred', 'normal_fdbk_config', 'ultimate_submission_fdbk_config', 'past_limit_submission_fdbk_config', 'staff_viewer_fdbk_config', ] self.assertCountEqual(expected_keys, suite_dict.keys()) self.assertIsInstance(suite_dict['instructor_files_needed'][0], dict) self.assertIsInstance(suite_dict['student_files_needed'][0], dict) self.assertSequenceEqual([ag_test.to_dict()], suite_dict['ag_test_cases']) self.assertIsInstance(suite_dict['normal_fdbk_config'], dict) self.assertIsInstance(suite_dict['ultimate_submission_fdbk_config'], dict) self.assertIsInstance(suite_dict['past_limit_submission_fdbk_config'], dict) self.assertIsInstance(suite_dict['staff_viewer_fdbk_config'], dict) update_dict = copy.deepcopy(suite_dict) for non_editable in [ 'pk', 'project', 'last_modified', 'ag_test_cases' ]: update_dict.pop(non_editable) suite.validate_and_update(**update_dict)
def test_stderr_correctness_show_diff_from_file(self): instructor_file = obj_build.make_instructor_file(self.project) self.ag_test_command.expected_stderr_source = ( ag_models.ExpectedOutputSource.instructor_file) self.ag_test_command.expected_stderr_instructor_file = instructor_file self.ag_test_command.save() result = self.make_correct_result() with instructor_file.open() as f: expected_stderr = f.read() result.stderr = expected_stderr result.save() diff = _get_expected_diff(expected_stderr, result.stderr_filename) self.assertEqual( diff.diff_content, get_cmd_fdbk( result, ag_models.FeedbackCategory.normal).stderr_diff.diff_content) result.stderr = 'the wrong stderr' result.save() diff = _get_expected_diff(expected_stderr, result.stderr_filename) self.assertEqual( diff.diff_content, get_cmd_fdbk( result, ag_models.FeedbackCategory.normal).stderr_diff.diff_content)
def test_serialize(self): project = ag_models.Project.objects.validate_and_create( name='qeiruqioewiur', course=self.course) # type: ag_models.Project instructor_file = obj_build.make_instructor_file(project) student_file = ag_models.ExpectedStudentFile.objects.validate_and_create( project=project, pattern='qweiourqpweioru') project_dict = project.to_dict() expected_keys = [ 'pk', 'name', 'course', 'last_modified', 'visible_to_students', 'closing_time', 'soft_closing_time', 'disallow_student_submissions', 'disallow_group_registration', 'guests_can_submit', 'min_group_size', 'max_group_size', 'submission_limit_per_day', 'allow_submissions_past_limit', 'groups_combine_daily_submissions', 'submission_limit_reset_time', 'submission_limit_reset_timezone', 'num_bonus_submissions', 'total_submission_limit', 'allow_late_days', 'ultimate_submission_policy', 'hide_ultimate_submission_fdbk', 'instructor_files', 'expected_student_files', ] self.assertCountEqual(expected_keys, project_dict.keys()) self.assertEqual('UTC', project_dict['submission_limit_reset_timezone']) self.assertSequenceEqual([instructor_file.to_dict()], project_dict['instructor_files']) self.assertSequenceEqual([student_file.to_dict()], project_dict['expected_student_files']) update_dict = copy.deepcopy(project_dict) update_dict.pop('pk') update_dict.pop('course') update_dict.pop('last_modified') update_dict.pop('instructor_files') update_dict.pop('expected_student_files') project.validate_and_update(**update_dict) other_timezone = 'America/Chicago' project.validate_and_update( submission_limit_reset_timezone=other_timezone) project.refresh_from_db() self.assertEqual(other_timezone, project.to_dict()['submission_limit_reset_timezone'])
def test_error_instructor_file_needed_that_belongs_to_other_project(self): other_project = obj_build.make_project(course=self.project.course) other_instructor_file = obj_build.make_instructor_file( project=other_project) with self.assertRaises(exceptions.ValidationError) as cm: ag_models.StudentTestSuite.objects.validate_and_create( name=self.name, project=self.project, instructor_files_needed=[other_instructor_file.pk]) self.assertIn('instructor_files_needed', cm.exception.message_dict)
def test_error_stdin_instructor_file_belongs_to_other_project(self): other_project = obj_build.make_project(course=self.project.course) other_instructor_file = obj_build.make_instructor_file( project=other_project) with self.assertRaises(exceptions.ValidationError) as cm: ag_models.AGTestCommand.objects.validate_and_create( name=self.name, ag_test_case=self.ag_test, cmd='true', stdin_source=ag_models.StdinSource.instructor_file, stdin_instructor_file=other_instructor_file) self.assertIn('stdin_instructor_file', cm.exception.message_dict)
def test_file_io_sources_serialized(self): stdin = obj_build.make_instructor_file(self.project) stdout = obj_build.make_instructor_file(self.project) stderr = obj_build.make_instructor_file(self.project) ag_cmd = ag_models.AGTestCommand.objects.validate_and_create( name=self.name, ag_test_case=self.ag_test, cmd=self.cmd, stdin_instructor_file=stdin, expected_stdout_instructor_file=stdout, expected_stderr_instructor_file=stderr) cmd_dict = ag_cmd.to_dict() self.assertIsInstance(cmd_dict['stdin_instructor_file'], dict) self.assertEqual(stdin.pk, cmd_dict['stdin_instructor_file']['pk']) self.assertIsInstance(cmd_dict['expected_stdout_instructor_file'], dict) self.assertEqual(stdout.pk, cmd_dict['expected_stdout_instructor_file']['pk']) self.assertIsInstance(cmd_dict['expected_stderr_instructor_file'], dict) self.assertEqual(stderr.pk, cmd_dict['expected_stderr_instructor_file']['pk'])
def test_error_instructor_and_student_files_dont_belong_to_same_project( self): other_project = obj_build.build_project() other_instructor_file = obj_build.make_instructor_file(other_project) other_student_file = ag_models.ExpectedStudentFile.objects.validate_and_create( pattern='alsdnvaoweijf', project=other_project) with self.assertRaises(exceptions.ValidationError) as cm: ag_models.AGTestSuite.objects.validate_and_create( name='aldksjfnaweij', project=self.project, instructor_files_needed=[other_instructor_file], student_files_needed=[other_student_file]) self.assertIn('instructor_files_needed', cm.exception.message_dict) self.assertIn('student_files_needed', cm.exception.message_dict)
def test_valid_create_non_defaults(self): student_file = ag_models.ExpectedStudentFile.objects.validate_and_create( pattern='filey', project=self.project) name = 'wee' project = self.project instructor_files_needed = [ obj_build.make_instructor_file(self.project) ] student_files_needed = [student_file] setup_cmd = "echo 'hello world'" allow_network_access = True deferred = True suite = ag_models.AGTestSuite.objects.validate_and_create( name=name, project=project, instructor_files_needed=instructor_files_needed, read_only_instructor_files=False, student_files_needed=student_files_needed, setup_suite_cmd=setup_cmd, setup_suite_cmd_name='steve', allow_network_access=allow_network_access, deferred=deferred, docker_image_to_use=constants.SupportedImages.eecs490, normal_fdbk_config={ 'visible': False, 'show_individual_tests': False, 'show_setup_return_code': False, 'show_setup_timed_out': False, 'show_setup_stdout': False, 'show_setup_stderr': False, }) suite.refresh_from_db() self.assertEqual(name, suite.name) self.assertEqual(project, suite.project) self.assertCountEqual(instructor_files_needed, suite.instructor_files_needed.all()) self.assertFalse(suite.read_only_instructor_files) self.assertCountEqual(student_files_needed, suite.student_files_needed.all()) self.assertEqual(allow_network_access, suite.allow_network_access) self.assertEqual(deferred, suite.deferred) self.assertEqual(constants.SupportedImages.eecs490, suite.docker_image_to_use) self.assertFalse(suite.normal_fdbk_config.visible)
def test_copy_project(self): # In new project, hide_ultimate_submission_fdbk should be set to True, # visible_to_students should be set to False, # and guests_can_submit should be set to False project = obj_build.make_project(hide_ultimate_submission_fdbk=False, visible_to_students=True) instructor_file1 = obj_build.make_instructor_file(project) instructor_file2 = obj_build.make_instructor_file(project) student_file1 = obj_build.make_expected_student_file(project) student_file2 = obj_build.make_expected_student_file(project) suite1 = obj_build.make_ag_test_suite( project, instructor_files_needed=[instructor_file1], student_files_needed=[student_file1]) case1 = obj_build.make_ag_test_case(suite1) cmd1 = obj_build.make_full_ag_test_command( case1, expected_stderr_source=ag_models.ExpectedOutputSource. instructor_file, expected_stderr_instructor_file=instructor_file2) cmd2 = obj_build.make_full_ag_test_command( case1, set_arbitrary_points=False, expected_stdout_source=ag_models.ExpectedOutputSource. instructor_file, expected_stdout_instructor_file=instructor_file1) suite2 = obj_build.make_ag_test_suite( project, instructor_files_needed=[instructor_file1, instructor_file2], student_files_needed=[student_file2]) case2 = obj_build.make_ag_test_case(suite2) cmd3 = obj_build.make_full_ag_test_command( case2, set_arbitrary_expected_vals=False, stdin_source=ag_models.StdinSource.instructor_file, stdin_instructor_file=instructor_file2) case3 = obj_build.make_ag_test_case(suite2) suite3 = obj_build.make_ag_test_suite(project) student_suite1 = obj_build.make_student_test_suite( project, instructor_files_needed=[instructor_file1, instructor_file2], student_files_needed=[student_file1], setup_command={ 'name': 'stave', 'cmd': 'yorp' }) student_suite2 = obj_build.make_student_test_suite( project, instructor_files_needed=[instructor_file1], student_files_needed=[student_file1, student_file2]) student_suite3 = obj_build.make_student_test_suite(project) other_course = obj_build.make_course() new_project = copy_project(project, other_course) self.assertTrue(new_project.hide_ultimate_submission_fdbk) self.assertFalse(new_project.visible_to_students) self.assertEqual(project.name, new_project.name) self.assertEqual(other_course, new_project.course) self.assertNotEqual(project.course, other_course) ignore_fields = [ 'pk', 'course', 'last_modified', 'instructor_files', 'expected_student_files' ] expected_ag_tests = _pop_many(project.to_dict(), ignore_fields) expected_ag_tests.update({ 'visible_to_students': False, 'hide_ultimate_submission_fdbk': True }) self.assertEqual(expected_ag_tests, _pop_many(new_project.to_dict(), ignore_fields)) self.assertEqual(project.instructor_files.count(), new_project.instructor_files.count()) for old_file, new_file in itertools.zip_longest( sorted(project.instructor_files.all(), key=lambda obj: obj.name), sorted(new_project.instructor_files.all(), key=lambda obj: obj.name)): self.assertNotEqual(new_file.pk, old_file.pk) self.assertNotEqual(new_file.abspath, old_file.abspath) self.assertEqual(old_file.name, new_file.name) with old_file.open() as old_f, new_file.open() as new_f: self.assertEqual(old_f.read(), new_f.read()) self.assertEqual(project.expected_student_files.count(), new_project.expected_student_files.count()) for old_expected_file, new_expected_file in itertools.zip_longest( project.expected_student_files.order_by('pattern'), new_project.expected_student_files.order_by('pattern')): self.assertNotEqual(old_expected_file.pk, new_expected_file.pk) self.assertEqual( _pop_many(old_expected_file.to_dict(), ['pk', 'project']), _pop_many(new_expected_file.to_dict(), ['pk', 'project'])) old_suite_pks = {suite.pk for suite in project.ag_test_suites.all()} new_suite_pks = { suite.pk for suite in new_project.ag_test_suites.all() } self.assertTrue(old_suite_pks.isdisjoint(new_suite_pks)) old_case_pks = { case.pk for case in ag_models.AGTestCase.objects.filter( ag_test_suite__project=project) }
def test_valid_init_non_defaults(self): instructor_file1 = obj_build.make_instructor_file(self.project) instructor_file2 = obj_build.make_instructor_file(self.project) student_file = obj_build.make_expected_student_file(self.project) values = { 'name': 'adnlakshfdklajhsdlf', 'project': self.project, 'instructor_files_needed': [instructor_file1.to_dict(), instructor_file2.to_dict()], 'read_only_instructor_files': False, 'student_files_needed': [student_file.to_dict()], 'buggy_impl_names': ['spam', 'egg', 'sausage', 'waaaaluigi'], 'setup_command': { 'cmd': 'g++ some_stuff.cpp' }, 'get_student_test_names_command': { 'cmd': 'ls test_*.cpp' }, 'max_num_student_tests': 30, 'student_test_validity_check_command': { 'cmd': 'python3 validity_check.py ${student_test_name}' }, 'grade_buggy_impl_command': { 'cmd': 'python3 grade.py ${buggy_impl_name} ${student_test_name}' }, 'points_per_exposed_bug': 42, 'max_points': 462, 'deferred': True, 'docker_image_to_use': constants.SupportedImages.eecs280, 'allow_network_access': True, 'normal_fdbk_config': { 'bugs_exposed_fdbk_level': (ag_models.BugsExposedFeedbackLevel.num_bugs_exposed.value), 'show_points': True, }, 'ultimate_submission_fdbk_config': { 'show_setup_stdout': True, 'show_invalid_test_names': True, }, 'past_limit_submission_fdbk_config': { 'visible': False, 'show_setup_stderr': True, }, 'staff_viewer_fdbk_config': { 'show_points': False, } } student_suite = ag_models.StudentTestSuite.objects.validate_and_create( **values) # type: ag_models.StudentTestSuite self.assertCountEqual([ instructor_file['pk'] for instructor_file in values['instructor_files_needed'] ], [ instructor_file.pk for instructor_file in student_suite.instructor_files_needed.all() ]) values.pop('instructor_files_needed') self.assertCountEqual([ student_file['pk'] for student_file in values['student_files_needed'] ], [ student_file.pk for student_file in student_suite.student_files_needed.all() ]) values.pop('student_files_needed') for key, value in values.items(): if isinstance(value, dict): self.assert_dict_is_subset( value, getattr(student_suite, key).to_dict()) elif isinstance(value, list): self.assertSequenceEqual(value, getattr(student_suite, key)) else: self.assertEqual(value, getattr(student_suite, key))
def test_serialization(self): expected_field_names = [ 'pk', 'name', 'project', 'instructor_files_needed', 'read_only_instructor_files', 'student_files_needed', 'buggy_impl_names', 'use_setup_command', 'setup_command', 'get_student_test_names_command', 'max_num_student_tests', 'student_test_validity_check_command', 'grade_buggy_impl_command', 'points_per_exposed_bug', 'max_points', 'deferred', 'docker_image_to_use', 'allow_network_access', 'normal_fdbk_config', 'ultimate_submission_fdbk_config', 'past_limit_submission_fdbk_config', 'staff_viewer_fdbk_config', 'last_modified', ] instructor_file = obj_build.make_instructor_file(self.project) student_file = obj_build.make_expected_student_file(self.project) student_suite = ag_models.StudentTestSuite.objects.validate_and_create( name=self.name, project=self.project, setup_command={'cmd': 'setuppy'}, instructor_files_needed=[instructor_file], student_files_needed=[student_file ]) # type: ag_models.StudentTestSuite serialized = student_suite.to_dict() self.assertCountEqual(expected_field_names, serialized.keys()) self.assertIsInstance(serialized['instructor_files_needed'][0], dict) self.assertIsInstance(serialized['student_files_needed'][0], dict) self.assertIsInstance(serialized['setup_command'], dict) self.assertIsInstance(serialized['get_student_test_names_command'], dict) self.assertIsInstance( serialized['student_test_validity_check_command'], dict) self.assertIsInstance(serialized['grade_buggy_impl_command'], dict) self.assertIsInstance(serialized['normal_fdbk_config'], dict) self.assertIsInstance(serialized['ultimate_submission_fdbk_config'], dict) self.assertIsInstance(serialized['past_limit_submission_fdbk_config'], dict) self.assertIsInstance(serialized['staff_viewer_fdbk_config'], dict) update_dict = student_suite.to_dict() non_editable = ['pk', 'project', 'last_modified'] for field in non_editable: update_dict.pop(field) student_suite.validate_and_update(**update_dict)