def test_course2(): """Test for XML loading errors""" errorstore = ErrorStore() course = load_course("testcourses/testcourse2", "coursefile.xml", errorstore) handle_course2_errors(errorstore) assert_caught_all_errors(errorstore)
def test_course3(): """Test for XML error in course.xml""" errorstore = ErrorStore() course = load_course("testcourses/testcourse3", "course.xml", errorstore) assert_error(errorstore, InvalidXML, 'course.xml', 'attributes construct error, line 1, column 61') assert_caught_all_errors(errorstore)
def test_course1(): """Make sure that things load properly when everything is ok""" errorstore = ErrorStore() course = load_course("testcourses/testcourse1", "course.xml", errorstore) assert errorstore.errors == [] assert course.attributes == { 'url_name': 'mycourseurl', 'org': 'myorg', 'course': 'mycourse' } [chapter] = course.children assert chapter.attributes == { 'url_name': 'chapter', 'display_name': 'chapter name' } [sequential] = chapter.children assert sequential.attributes == { 'url_name': 'sequential', 'display_name': 'display name' } [vertical] = sequential.children assert vertical.attributes == { 'url_name': 'vertical', 'display_name': 'vertical name' } [html] = vertical.children assert html.attributes == {'url_name': 'html', 'display_name': 'html name'} assert '<p>Test course</p>' in etree.tostring(html.content, pretty_print=True).decode()
def test_course7_url(): errorstore = ErrorStore() # Load course course = load_course("testcourses/testcourse7", "course.xml", errorstore) assert_caught_all_errors(errorstore) # Load policy policy, grading_policy = load_policy("testcourses/testcourse7", course, errorstore) assert_caught_all_errors(errorstore) # Make a dictionary of url_names url_names = find_url_names(course, errorstore) # Merge the policy file merge_policy(policy, url_names, errorstore) # Ensure that settings were indeed merged assert (url_names['sequential2'].attributes['setting']) # Validate the grading policy validate_grading_policy(grading_policy, errorstore) # Handle the errors handle_course7_errors(errorstore) assert_caught_all_errors(errorstore)
def test_no_policy(): errorstore = ErrorStore() # Load course (needed before loading policy) course = load_course("testcourses/testcourse1", "course.xml", errorstore) assert_caught_all_errors(errorstore) # Load the (nonexistent) policy files policy, grading_policy = load_policy("testcourses/testcourse1", course, errorstore) handle_course1_errors(errorstore) assert_caught_all_errors(errorstore)
def test_no_url_name(): errorstore = ErrorStore() # Load course (needed before loading policy) course = load_course("testcourses/testcourse4", "course.xml", errorstore) assert_caught_all_errors(errorstore) # Load the (nonexistent) policy files policy, grading_policy = load_policy("testcourses/testcourse4", course, errorstore) assert_error(errorstore, NoRunName, 'course.xml', "The course tag has no url_name.") assert_caught_all_errors(errorstore)
def test_url_names(): errorstore = ErrorStore() # Load course course = load_course("testcourses/testcourse1", "course.xml", errorstore) assert_caught_all_errors(errorstore) # Make a dictionary of url_names url_names = find_url_names(course, errorstore) assert_caught_all_errors(errorstore) expected = ['mycourseurl', 'chapter', 'sequential', 'vertical', 'html'] for i in expected: assert i in url_names
def test_bad_json(): errorstore = ErrorStore() # Load course (needed before loading policy) course = load_course("testcourses/testcourse5", "course.xml", errorstore) assert_caught_all_errors(errorstore) # Load the policy files policy, grading_policy = load_policy("testcourses/testcourse5", course, errorstore) assert_error( errorstore, BadPolicy, 'policies/mycourseurl/policy.json', "The policy file 'policies/mycourseurl/policy.json' has invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" ) assert_error( errorstore, BadPolicy, 'policies/mycourseurl/grading_policy.json', "The policy file 'policies/mycourseurl/grading_policy.json' has invalid JSON: Expecting property name enclosed in double quotes: line 2 column 3 (char 4)" ) assert_caught_all_errors(errorstore)
def test_course6_url(): errorstore = ErrorStore() # Load course course = load_course("testcourses/testcourse6", "course.xml", errorstore) assert_caught_all_errors(errorstore) # Load policy policy, grading_policy = load_policy("testcourses/testcourse6", course, errorstore) assert_caught_all_errors(errorstore) # Make a dictionary of url_names url_names = find_url_names(course, errorstore) # Merge the policy file merge_policy(policy, url_names, errorstore) # Handle the errors handle_course6_errors(errorstore) assert_caught_all_errors(errorstore)
def test_no_course(): errorstore = ErrorStore() course = load_course("testcourses", "nocourse.xml", errorstore) handle_nocourse_errors(errorstore) assert_caught_all_errors(errorstore)
def validate(filename, steps=8, ignore=None): """ Validate an OLX course by performing the given number of steps: * 1: Load the course * 2: Load the policy and grading policy * 3: Validate url_names * 4: Merge policy data with course, ensuring that all references are valid * 5: Validate the grading policy * 6: Have every object validate itself * 7: Parse the course for global errors * 8: Parse the course for global errors that may be time-consuming to detect :param filename: Location of course xml file or directory :param steps: Number of validation steps to take (1 = first only, 8 = all) :param ignore: List of errors to ignore :return: course object, errorstore object, url_names dictionary (or None if steps < 3) """ # Create an error store if ignore is None: ignore = [] errorstore = ErrorStore(ignore) # Validation Step #1: Load the course if os.path.isdir(filename): directory = os.path.join(filename) file = "course.xml" else: directory, file = os.path.split(filename) course = load_course(directory, file, errorstore) if not course: return None, errorstore, None if steps > 1: # Validation Step #2: Load the policy files policy, grading_policy = load_policy(directory, course, errorstore) url_names = None if steps > 2: # Validation Step #3: Construct a dictionary of url_names url_names = find_url_names(course, errorstore) if steps > 3: # Validation Step #4: Merge policy data into object attributes merge_policy(policy, url_names, errorstore) if steps > 4: # Validation Step #5: Validate grading policy validate_grading_policy(grading_policy, errorstore) if steps > 5: # Validation Step #6: Have every object validate itself for edxobj in traverse(course): edxobj.validate(course, errorstore) if steps > 6: # Validation Step #7: Parse the course for global errors for validator in GlobalValidator.validators(): validator(course, errorstore, url_names) if steps > 7: # Validation Step #8: Parse the course for global errors that are time-consuming to detect for validator in SlowValidator.validators(): validator(course, errorstore, url_names) return course, errorstore, url_names
def test_grading_policy(): # Test grading policy entries in isolation errorstore = ErrorStore() # Set up the policy policy = {} # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "'GRADER' entry not found in grading policy") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "'GRADE_CUTOFFS' entry not found in grading policy") assert_caught_all_errors(errorstore) # Set up the policy policy = {'GRADER': {}, 'GRADE_CUTOFFS': []} # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "'GRADER' entry not in grading policy is not a list") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'GRADE_CUTOFFS' entry not in grading policy is not a dictionary") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Lab", "short_label": 15, "weight": 0.15 }, { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 0.15 }, ], 'GRADE_CUTOFFS': { 'Pass': -1 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "'weight' settings do not add up to 1") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'short_label' setting is not a string for entry 1 in the grading policy" ) assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 'hi', "min_count": 'hi', "type": 0, "weight": 'hi' }, { "drop_count": -1, "min_count": 0, "type": "Something", "weight": 1.1 }, {}, ], 'GRADE_CUTOFFS': { 'Pass': 2 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'drop_count' setting is not an integer for entry 1 in the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'min_count' setting is not an integer for entry 1 in the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'type' setting is not a string for entry 1 in the grading policy") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'weight' setting is not a number between 0 and 1 for entry 1 in the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'drop_count' setting is negative for entry 2 in the grading policy") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'min_count' setting is less than 1 for entry 2 in the grading policy") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'weight' setting is not a number between 0 and 1 for entry 2 in the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'drop_count' setting is omitted for entry 3 in the grading policy") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'min_count' setting is omitted for entry 3 in the grading policy") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'type' setting is omitted for entry 3 in the grading policy") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'weight' setting is omitted for entry 3 in the grading policy") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "'weight' settings do not add up to 1") assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'Pass' entry is not between 0 and 1 in the GRADE_CUTOFFS part of the grading policy" ) assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 0.6 }, { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 0.4 }, ], 'GRADE_CUTOFFS': { 'A': -1, 'C': 3, 'D': 0.5, 'Pass': '******', 'pass': 1 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'A' entry is not between 0 and 1 in the GRADE_CUTOFFS part of the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'C' entry is not between 0 and 1 in the GRADE_CUTOFFS part of the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'pass' is not allowed in the GRADE_CUTOFFS part of the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "'Pass' entry is not a number in the GRADE_CUTOFFS part of the grading policy" ) assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "Assessment type 'Something' appears multiple times in the grading policy" ) assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'A': 0, 'C': 0.2, 'D': 0.5, 'Pass': 0.6, } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error( errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS should have either 'Pass' or letters, not both") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'A': 0, 'C': 0.2, 'D': 0.5 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is missing 'B'") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is entry 'C' is not decreasing") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is entry 'D' is not decreasing") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'A': 0, 'B': 0.1, 'C': 0.2, 'D': 0.5 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is entry 'B' is not decreasing") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is entry 'C' is not decreasing") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is entry 'D' is not decreasing") assert_caught_all_errors(errorstore) # Set up the policy policy = {'GRADER': [], 'GRADE_CUTOFFS': {}} # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADER entry in grading policy should not be empty") assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS entry in grading policy should not be empty") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'A': 1, 'C': 0.1 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is missing 'B'") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'B': 1, 'C': 0.1 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is missing 'A'") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'B': 1 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS is missing 'A'") assert_caught_all_errors(errorstore) # Set up the policy policy = { 'GRADER': [ { "drop_count": 2, "min_count": 12, "type": "Something", "weight": 1 }, ], 'GRADE_CUTOFFS': { 'A': 1 } } # Validate it validate_grading_policy(policy, errorstore) # Handle errors assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json', "GRADE_CUTOFFS should use 'Pass' instead of 'A'") assert_caught_all_errors(errorstore)