def test_course3(): """Test for XML error in course.xml""" errorstore = ErrorStore() course = load_course("testcourses/testcourse3", "course.xml", errorstore) assert_error(errorstore, InvalidXML, 'course.xml', 'attributes construct error, line 1, column 61') assert_caught_all_errors(errorstore)
def test_course2(): """Test for XML loading errors""" errorstore = ErrorStore() course = load_course("testcourses/testcourse2", "coursefile.xml", errorstore) handle_course2_errors(errorstore) assert_caught_all_errors(errorstore)
def test_link_checking(): """Checks for broken internal links""" # Perform all steps on course 10 up to validation steps course, errorstore, url_names = validate("testcourses/testcourse10", 6) handle_general_errors_in_10(errorstore) assert_caught_all_errors(errorstore) # Check the link-finding routines links = find_links(url_names['linktest']) assert set(links) == { '/course/courseware/testing', '/course/courseware/broken_chapter', '/course/courseware/chapter/sequential', '/course/courseware/chapter/sequential/', '/course/courseware/chapter/sequential/vertical', '/course/courseware/chapter/sequential/vertical/', '/course/courseware/chapter/sequential/vertical/1', '/course/courseware/chapter/sequential/vertical/10', '/course/courseware/chapter/sequential/vertical/0', '/course/courseware/chapter/sequential/vertical/html', '/course/courseware/chapter/sequential/vertical/1?last_child', '/course/courseware/chapter/sequential/vertical/?last_child', '/course/courseware/chapter/sequential/vertical?last_child', '/static/testing.png', '/static/testing2.css', '/jump_to_id/oravert', '/jump_to_id/oravert2', '/static/image.png', '/course/discussion/somewhere', '/course/pdfbook/0/chapter/9/11' } # Check the links validator = CheckLinks() validator(course, errorstore, url_names) # Catch everything! handle_link_errors_in_10(errorstore) assert_caught_all_errors(errorstore)
def test_validate_course10(): """This test includes all validation steps. The course is designed to test the validators and slow validators.""" course, errorstore, url_names = validate("testcourses/testcourse10") handle_general_errors_in_10(errorstore) handle_display_name_errors_in_10(errorstore) handle_discussion_id_errors_in_10(errorstore) handle_link_errors_in_10(errorstore) assert_caught_all_errors(errorstore)
def test_validate_course8(): """This test includes individual component validation. Almost everything should pass.""" course, errorstore, url_names = validate("testcourses/testcourse8", 6) assert_error( errorstore, InvalidSetting, 'course/mycourseurl.xml', "The tag <course url_name='mycourseurl'> does not have the required setting 'course_image'." ) assert_caught_all_errors(errorstore)
def test_no_policy(): errorstore = ErrorStore() # Load course (needed before loading policy) course = load_course("testcourses/testcourse1", "course.xml", errorstore) assert_caught_all_errors(errorstore) # Load the (nonexistent) policy files policy, grading_policy = load_policy("testcourses/testcourse1", course, errorstore) handle_course1_errors(errorstore) assert_caught_all_errors(errorstore)
def test_no_url_name(): errorstore = ErrorStore() # Load course (needed before loading policy) course = load_course("testcourses/testcourse4", "course.xml", errorstore) assert_caught_all_errors(errorstore) # Load the (nonexistent) policy files policy, grading_policy = load_policy("testcourses/testcourse4", course, errorstore) assert_error(errorstore, NoRunName, 'course.xml', "The course tag has no url_name.") assert_caught_all_errors(errorstore)
def test_discussion_ids(): # Perform all steps on course 10 up to validation steps course, errorstore, url_names = validate("testcourses/testcourse10", 6) handle_general_errors_in_10(errorstore) assert_caught_all_errors(errorstore) # Check the discussion IDs validator = CheckDiscussionIDs() validator(course, errorstore, url_names) # Catch everything! handle_discussion_id_errors_in_10(errorstore) assert_caught_all_errors(errorstore)
def test_url_names(): errorstore = ErrorStore() # Load course course = load_course("testcourses/testcourse1", "course.xml", errorstore) assert_caught_all_errors(errorstore) # Make a dictionary of url_names url_names = find_url_names(course, errorstore) assert_caught_all_errors(errorstore) expected = ['mycourseurl', 'chapter', 'sequential', 'vertical', 'html'] for i in expected: assert i in url_names
def test_bad_json(): errorstore = ErrorStore() # Load course (needed before loading policy) course = load_course("testcourses/testcourse5", "course.xml", errorstore) assert_caught_all_errors(errorstore) # Load the policy files policy, grading_policy = load_policy("testcourses/testcourse5", course, errorstore) assert_error( errorstore, BadPolicy, 'policies/mycourseurl/policy.json', "The policy file 'policies/mycourseurl/policy.json' has invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" ) assert_error( errorstore, BadPolicy, 'policies/mycourseurl/grading_policy.json', "The policy file 'policies/mycourseurl/grading_policy.json' has invalid JSON: Expecting property name enclosed in double quotes: line 2 column 3 (char 4)" ) assert_caught_all_errors(errorstore)
def test_course7_url(): errorstore = ErrorStore() # Load course course = load_course("testcourses/testcourse7", "course.xml", errorstore) assert_caught_all_errors(errorstore) # Load policy policy, grading_policy = load_policy("testcourses/testcourse7", course, errorstore) assert_caught_all_errors(errorstore) # Make a dictionary of url_names url_names = find_url_names(course, errorstore) # Merge the policy file merge_policy(policy, url_names, errorstore) # Ensure that settings were indeed merged assert (url_names['sequential2'].attributes['setting']) # Validate the grading policy validate_grading_policy(grading_policy, errorstore) # Handle the errors handle_course7_errors(errorstore) assert_caught_all_errors(errorstore)
def test_course6_url(): errorstore = ErrorStore() # Load course course = load_course("testcourses/testcourse6", "course.xml", errorstore) assert_caught_all_errors(errorstore) # Load policy policy, grading_policy = load_policy("testcourses/testcourse6", course, errorstore) assert_caught_all_errors(errorstore) # Make a dictionary of url_names url_names = find_url_names(course, errorstore) # Merge the policy file merge_policy(policy, url_names, errorstore) # Handle the errors handle_course6_errors(errorstore) assert_caught_all_errors(errorstore)
def test_validate_course1(): course, errorstore, url_names = validate("testcourses/testcourse1", 2) handle_course1_errors(errorstore) assert_caught_all_errors(errorstore)
def test_validate_course2(): course, errorstore, url_names = validate( "testcourses/testcourse2/coursefile.xml", 1) handle_course2_errors(errorstore) assert_caught_all_errors(errorstore)
def test_no_course(): errorstore = ErrorStore() course = load_course("testcourses", "nocourse.xml", errorstore) handle_nocourse_errors(errorstore) assert_caught_all_errors(errorstore)
def test_validate_nocourse(): course, errorstore, url_names = validate("testcourses/nocourse.xml", 2) handle_nocourse_errors(errorstore) assert_caught_all_errors(errorstore)
def test_grading_policy(): # Test grading policy entries in isolation errorstore = ErrorStore() # Set up the policy policy = {} # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "'GRADER' entry not found in grading policy") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "'GRADE_CUTOFFS' entry not found in grading policy") assert_caught_all_errors(errorstore) # Set up the policy policy = {'GRADER': {}, 'GRADE_CUTOFFS': []} # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "'GRADER' entry not in grading policy is not a list") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'GRADE_CUTOFFS' entry not in grading policy is not a dictionary") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Lab", "short_label": 15, "weight": 0.15 }, { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 0.15 }, ], 'GRADE_CUTOFFS': { 'Pass': -1 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "'weight' settings do not add up to 1") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'short_label' setting is not a string for entry 1 in the grading policy" ) assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 'hi', "min_count": 'hi', "type": 0, "weight": 'hi' }, { "drop_count": -1, "min_count": 0, "type": "Something", "weight": 1.1 }, {}, ], 'GRADE_CUTOFFS': { 'Pass': 2 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'drop_count' setting is not an integer for entry 1 in the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'min_count' setting is not an integer for entry 1 in the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'type' setting is not a string for entry 1 in the grading policy") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'weight' setting is not a number between 0 and 1 for entry 1 in the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'drop_count' setting is negative for entry 2 in the grading policy") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'min_count' setting is less than 1 for entry 2 in the grading policy") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'weight' setting is not a number between 0 and 1 for entry 2 in the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'drop_count' setting is omitted for entry 3 in the grading policy") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'min_count' setting is omitted for entry 3 in the grading policy") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'type' setting is omitted for entry 3 in the grading policy") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'weight' setting is omitted for entry 3 in the grading policy") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "'weight' settings do not add up to 1") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'Pass' entry is not between 0 and 1 in the GRADE_CUTOFFS part of the grading policy" ) assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 0.6 }, { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 0.4 }, ], 'GRADE_CUTOFFS': { 'A': -1, 'C': 3, 'D': 0.5, 'Pass': '******', 'pass': 1 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'A' entry is not between 0 and 1 in the GRADE_CUTOFFS part of the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'C' entry is not between 0 and 1 in the GRADE_CUTOFFS part of the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'pass' is not allowed in the GRADE_CUTOFFS part of the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'Pass' entry is not a number in the GRADE_CUTOFFS part of the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "Assessment type 'Something' appears multiple times in the grading policy" ) assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'A': 0, 'C': 0.2, 'D': 0.5, 'Pass': 0.6, } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS should have either 'Pass' or letters, not both") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'A': 0, 'C': 0.2, 'D': 0.5 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is missing 'B'") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is entry 'C' is not decreasing") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is entry 'D' is not decreasing") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'A': 0, 'B': 0.1, 'C': 0.2, 'D': 0.5 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is entry 'B' is not decreasing") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is entry 'C' is not decreasing") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is entry 'D' is not decreasing") assert_caught_all_errors(errorstore) # Set up the policy policy = {'GRADER': [], 'GRADE_CUTOFFS': {}} # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADER entry in grading policy should not be empty") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS entry in grading policy should not be empty") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'A': 1, 'C': 0.1 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is missing 'B'") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'B': 1, 'C': 0.1 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is missing 'A'") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'B': 1 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is missing 'A'") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'A': 1 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS should use 'Pass' instead of 'A'") assert_caught_all_errors(errorstore)
def test_validate_course9(): """This test includes individual component validation. This similar to course8, but riddled with errors.""" course, errorstore, url_names = validate("testcourses/testcourse9", 6) assert_error(errorstore, InvalidSetting, 'course/mycourseurl.xml', "Unable to recognize graceperiod format in policy.") assert_error( errorstore, InvalidSetting, 'problem/problem.xml', "The tag <problem url_name='problem'> has an invalid setting 'showanswer=bad'." ) assert_error( errorstore, InvalidSetting, 'chapter/chapter.xml', "The tag <chapter url_name='chapter' display_name='Hi there!'> has an invalid date setting for start: 'Feb 20, 2019, 17:00zzz'." ) assert_error( errorstore, DateOrdering, 'sequential/sequential.xml', "The tag <sequential url_name='sequential' display_name='Hi there!'> has a date out of order: start date cannot be before course start date" ) assert_error( errorstore, DateOrdering, 'vertical/vertical.xml', "The tag <vertical url_name='vertical' display_name='Hi mom!'> has a date out of order: due date must be before course end date" ) assert_error( errorstore, DateOrdering, 'problem/problem.xml', "The tag <problem url_name='problem'> has a date out of order: start date must be before due date" ) assert_error( errorstore, MissingURLName, 'vertical/vertical.xml', "The tag <problem display_name='no url_name'> has no url_name.") assert_error( errorstore, InvalidSetting, 'vertical/vertical.xml', "The tag <problem display_name='no url_name'> has an invalid setting 'showanswer=hah!'." ) assert_error( errorstore, DateOrdering, 'vertical/vertical.xml', "The tag <problem display_name='no url_name'> has a date out of order: due date must be before course end date" ) assert_error( errorstore, Obsolete, 'vertical/vertical.xml', "The tag <discussion url_name='discussion'> should be included inline rather than through the discussion directory." ) assert_error( errorstore, InvalidSetting, 'discussion/discussion.xml', "The tag <discussion url_name='discussion'> does not have the required setting 'discussion_id'." ) assert_error( errorstore, InvalidSetting, 'discussion/discussion.xml', "The tag <discussion url_name='discussion'> does not have the required setting 'discussion_category'." ) assert_error( errorstore, InvalidSetting, 'discussion/discussion.xml', "The tag <discussion url_name='discussion'> does not have the required setting 'discussion_target'." ) assert_error( errorstore, Obsolete, 'lti/lti.xml', "The tag <lti url_name='lti'> should be converted to the newer lti_consumer Xblock." ) assert_error( errorstore, InvalidSetting, 'lti/lti.xml', "The tag <lti url_name='lti'> does not have the required setting 'launch_url'." ) assert_error( errorstore, LTIError, 'lti/lti.xml', "Course policy does not include an 'lti_passports' entry for 'nothere', required for <lti url_name='lti'>." ) assert_error( errorstore, LTIError, 'lti/lti.xml', "Course policy does not include the 'lti' advanced module, required for <lti url_name='lti'>." ) assert_error( errorstore, InvalidSetting, 'vertical/vertical.xml', "The tag <lti_consumer url_name='meep'> does not have the required setting 'launch_url'." ) assert_error( errorstore, LTIError, 'vertical/vertical.xml', "Course policy does not include an 'lti_passports' entry for 'nothere', required for <lti_consumer url_name='meep'>." ) assert_error( errorstore, LTIError, 'vertical/vertical.xml', "Course policy does not include the 'lti_consumer' advanced module, required for <lti_consumer url_name='meep'>." ) assert_error( errorstore, MissingFile, 'course/mycourseurl.xml', "The <course url_name='mycourseurl'> tag contains a reference to a missing static file: course_image_small.jpg" ) assert_error( errorstore, InvalidSetting, 'problem/problem.xml', "The tag <problem url_name='problem'> has a negative problem weight.") assert_error( errorstore, InvalidSetting, 'course/mycourseurl.xml', "The tag <course url_name='mycourseurl'> should have a positive number of attempts." ) assert_error( errorstore, InvalidSetting, 'problem/problem.xml', "The tag <problem url_name='problem'> should have a positive number of attempts." ) assert_error( errorstore, InvalidSetting, 'vertical/vertical.xml', "The tag <drag-and-drop-v2 url_name='studio_mess2' display_name='This is my title'> has an error in the data JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)." ) assert_error( errorstore, InvalidSetting, 'vertical/vertical.xml', "The tag <drag-and-drop-v2 url_name='studio_mess3' display_name='This is my title'> data JSON is not a dictionary." ) assert_error( errorstore, InvalidSetting, 'sequential/examseq.xml', "The tag <sequential url_name='examseq' display_name='Exam'> is a timed exam, but the course policy does not have 'enable_timed_exams=true'." ) assert_error( errorstore, DateOrdering, 'vertical/oravert.xml', "The tag <openassessment url_name='paper-draft'> has a date out of order: assessment 1 due date must be before course end date" ) assert_caught_all_errors(errorstore) # Ensure that we got the scripts from the problem file assert set( url_names['problem'].scripts) == {'python', 'perl', 'javascript'} # Also make sure we detected the response types assert set(url_names['problem'].response_types) == { 'customresponse', 'multiplechoiceresponse' } # as well as the input types assert set(url_names['problem'].input_types) == {'choicegroup', 'textline'} # We also found the solution assert url_names['problem'].has_solution # Make sure our exam sequential was detected assert url_names['examseq'].is_exam
def test_validate_course7(): course, errorstore, url_names = validate( "testcourses/testcourse7/course.xml", 5) handle_course7_errors(errorstore) assert_caught_all_errors(errorstore)