def test_course3():
"""Test for XML error in course.xml"""
errorstore = ErrorStore()
course = load_course("testcourses/testcourse3", "course.xml", errorstore)
assert_error(errorstore, InvalidXML, 'course.xml',
'attributes construct error, line 1, column 61')
assert_caught_all_errors(errorstore)
def test_course2():
"""Test for XML loading errors"""
errorstore = ErrorStore()
course = load_course("testcourses/testcourse2", "coursefile.xml",
errorstore)
handle_course2_errors(errorstore)
assert_caught_all_errors(errorstore)
def test_link_checking():
"""Checks for broken internal links"""
# Perform all steps on course 10 up to validation steps
course, errorstore, url_names = validate("testcourses/testcourse10", 6)
handle_general_errors_in_10(errorstore)
assert_caught_all_errors(errorstore)
# Check the link-finding routines
links = find_links(url_names['linktest'])
assert set(links) == {
'/course/courseware/testing', '/course/courseware/broken_chapter',
'/course/courseware/chapter/sequential',
'/course/courseware/chapter/sequential/',
'/course/courseware/chapter/sequential/vertical',
'/course/courseware/chapter/sequential/vertical/',
'/course/courseware/chapter/sequential/vertical/1',
'/course/courseware/chapter/sequential/vertical/10',
'/course/courseware/chapter/sequential/vertical/0',
'/course/courseware/chapter/sequential/vertical/html',
'/course/courseware/chapter/sequential/vertical/1?last_child',
'/course/courseware/chapter/sequential/vertical/?last_child',
'/course/courseware/chapter/sequential/vertical?last_child',
'/static/testing.png', '/static/testing2.css', '/jump_to_id/oravert',
'/jump_to_id/oravert2', '/static/image.png',
'/course/discussion/somewhere', '/course/pdfbook/0/chapter/9/11'
}
# Check the links
validator = CheckLinks()
validator(course, errorstore, url_names)
# Catch everything!
handle_link_errors_in_10(errorstore)
assert_caught_all_errors(errorstore)
def test_validate_course10():
"""This test includes all validation steps. The course is designed to test the validators and slow validators."""
course, errorstore, url_names = validate("testcourses/testcourse10")
handle_general_errors_in_10(errorstore)
handle_display_name_errors_in_10(errorstore)
handle_discussion_id_errors_in_10(errorstore)
handle_link_errors_in_10(errorstore)
assert_caught_all_errors(errorstore)
def test_validate_course8():
"""This test includes individual component validation. Almost everything should pass."""
course, errorstore, url_names = validate("testcourses/testcourse8", 6)
assert_error(
errorstore, InvalidSetting, 'course/mycourseurl.xml',
"The tag <course url_name='mycourseurl'> does not have the required setting 'course_image'."
)
assert_caught_all_errors(errorstore)
def test_no_policy():
errorstore = ErrorStore()
# Load course (needed before loading policy)
course = load_course("testcourses/testcourse1", "course.xml", errorstore)
assert_caught_all_errors(errorstore)
# Load the (nonexistent) policy files
policy, grading_policy = load_policy("testcourses/testcourse1", course,
errorstore)
handle_course1_errors(errorstore)
assert_caught_all_errors(errorstore)
def test_no_url_name():
errorstore = ErrorStore()
# Load course (needed before loading policy)
course = load_course("testcourses/testcourse4", "course.xml", errorstore)
assert_caught_all_errors(errorstore)
# Load the (nonexistent) policy files
policy, grading_policy = load_policy("testcourses/testcourse4", course,
errorstore)
assert_error(errorstore, NoRunName, 'course.xml',
"The course tag has no url_name.")
assert_caught_all_errors(errorstore)
def test_discussion_ids():
# Perform all steps on course 10 up to validation steps
course, errorstore, url_names = validate("testcourses/testcourse10", 6)
handle_general_errors_in_10(errorstore)
assert_caught_all_errors(errorstore)
# Check the discussion IDs
validator = CheckDiscussionIDs()
validator(course, errorstore, url_names)
# Catch everything!
handle_discussion_id_errors_in_10(errorstore)
assert_caught_all_errors(errorstore)
def test_url_names():
errorstore = ErrorStore()
# Load course
course = load_course("testcourses/testcourse1", "course.xml", errorstore)
assert_caught_all_errors(errorstore)
# Make a dictionary of url_names
url_names = find_url_names(course, errorstore)
assert_caught_all_errors(errorstore)
expected = ['mycourseurl', 'chapter', 'sequential', 'vertical', 'html']
for i in expected:
assert i in url_names
def test_bad_json():
errorstore = ErrorStore()
# Load course (needed before loading policy)
course = load_course("testcourses/testcourse5", "course.xml", errorstore)
assert_caught_all_errors(errorstore)
# Load the policy files
policy, grading_policy = load_policy("testcourses/testcourse5", course,
errorstore)
assert_error(
errorstore, BadPolicy, 'policies/mycourseurl/policy.json',
"The policy file 'policies/mycourseurl/policy.json' has invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)"
)
assert_error(
errorstore, BadPolicy, 'policies/mycourseurl/grading_policy.json',
"The policy file 'policies/mycourseurl/grading_policy.json' has invalid JSON: Expecting property name enclosed in double quotes: line 2 column 3 (char 4)"
)
assert_caught_all_errors(errorstore)
def test_course7_url():
errorstore = ErrorStore()
# Load course
course = load_course("testcourses/testcourse7", "course.xml", errorstore)
assert_caught_all_errors(errorstore)
# Load policy
policy, grading_policy = load_policy("testcourses/testcourse7", course,
errorstore)
assert_caught_all_errors(errorstore)
# Make a dictionary of url_names
url_names = find_url_names(course, errorstore)
# Merge the policy file
merge_policy(policy, url_names, errorstore)
# Ensure that settings were indeed merged
assert (url_names['sequential2'].attributes['setting'])
# Validate the grading policy
validate_grading_policy(grading_policy, errorstore)
# Handle the errors
handle_course7_errors(errorstore)
assert_caught_all_errors(errorstore)
def test_course6_url():
errorstore = ErrorStore()
# Load course
course = load_course("testcourses/testcourse6", "course.xml", errorstore)
assert_caught_all_errors(errorstore)
# Load policy
policy, grading_policy = load_policy("testcourses/testcourse6", course,
errorstore)
assert_caught_all_errors(errorstore)
# Make a dictionary of url_names
url_names = find_url_names(course, errorstore)
# Merge the policy file
merge_policy(policy, url_names, errorstore)
# Handle the errors
handle_course6_errors(errorstore)
assert_caught_all_errors(errorstore)
def test_validate_course1():
course, errorstore, url_names = validate("testcourses/testcourse1", 2)
handle_course1_errors(errorstore)
assert_caught_all_errors(errorstore)
def test_validate_course2():
course, errorstore, url_names = validate(
"testcourses/testcourse2/coursefile.xml", 1)
handle_course2_errors(errorstore)
assert_caught_all_errors(errorstore)
def test_no_course():
errorstore = ErrorStore()
course = load_course("testcourses", "nocourse.xml", errorstore)
handle_nocourse_errors(errorstore)
assert_caught_all_errors(errorstore)
def test_validate_nocourse():
course, errorstore, url_names = validate("testcourses/nocourse.xml", 2)
handle_nocourse_errors(errorstore)
assert_caught_all_errors(errorstore)
def test_grading_policy():
# Test grading policy entries in isolation
errorstore = ErrorStore()
# Set up the policy
policy = {}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"'GRADER' entry not found in grading policy")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"'GRADE_CUTOFFS' entry not found in grading policy")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {'GRADER': {}, 'GRADE_CUTOFFS': []}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"'GRADER' entry not in grading policy is not a list")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'GRADE_CUTOFFS' entry not in grading policy is not a dictionary")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Lab",
"short_label": 15,
"weight": 0.15
},
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 0.15
},
],
'GRADE_CUTOFFS': {
'Pass': -1
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"'weight' settings do not add up to 1")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'short_label' setting is not a string for entry 1 in the grading policy"
)
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 'hi',
"min_count": 'hi',
"type": 0,
"weight": 'hi'
},
{
"drop_count": -1,
"min_count": 0,
"type": "Something",
"weight": 1.1
},
{},
],
'GRADE_CUTOFFS': {
'Pass': 2
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'drop_count' setting is not an integer for entry 1 in the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'min_count' setting is not an integer for entry 1 in the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'type' setting is not a string for entry 1 in the grading policy")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'weight' setting is not a number between 0 and 1 for entry 1 in the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'drop_count' setting is negative for entry 2 in the grading policy")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'min_count' setting is less than 1 for entry 2 in the grading policy")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'weight' setting is not a number between 0 and 1 for entry 2 in the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'drop_count' setting is omitted for entry 3 in the grading policy")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'min_count' setting is omitted for entry 3 in the grading policy")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'type' setting is omitted for entry 3 in the grading policy")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'weight' setting is omitted for entry 3 in the grading policy")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"'weight' settings do not add up to 1")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'Pass' entry is not between 0 and 1 in the GRADE_CUTOFFS part of the grading policy"
)
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 0.6
},
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 0.4
},
],
'GRADE_CUTOFFS': {
'A': -1,
'C': 3,
'D': 0.5,
'Pass': '******',
'pass': 1
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'A' entry is not between 0 and 1 in the GRADE_CUTOFFS part of the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'C' entry is not between 0 and 1 in the GRADE_CUTOFFS part of the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'pass' is not allowed in the GRADE_CUTOFFS part of the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'Pass' entry is not a number in the GRADE_CUTOFFS part of the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"Assessment type 'Something' appears multiple times in the grading policy"
)
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'A': 0,
'C': 0.2,
'D': 0.5,
'Pass': 0.6,
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS should have either 'Pass' or letters, not both")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'A': 0,
'C': 0.2,
'D': 0.5
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is missing 'B'")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is entry 'C' is not decreasing")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is entry 'D' is not decreasing")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'A': 0,
'B': 0.1,
'C': 0.2,
'D': 0.5
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is entry 'B' is not decreasing")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is entry 'C' is not decreasing")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is entry 'D' is not decreasing")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {'GRADER': [], 'GRADE_CUTOFFS': {}}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADER entry in grading policy should not be empty")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS entry in grading policy should not be empty")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'A': 1,
'C': 0.1
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is missing 'B'")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'B': 1,
'C': 0.1
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is missing 'A'")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'B': 1
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is missing 'A'")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'A': 1
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS should use 'Pass' instead of 'A'")
assert_caught_all_errors(errorstore)
def test_validate_course9():
"""This test includes individual component validation. This similar to course8, but riddled with errors."""
course, errorstore, url_names = validate("testcourses/testcourse9", 6)
assert_error(errorstore, InvalidSetting, 'course/mycourseurl.xml',
"Unable to recognize graceperiod format in policy.")
assert_error(
errorstore, InvalidSetting, 'problem/problem.xml',
"The tag <problem url_name='problem'> has an invalid setting 'showanswer=bad'."
)
assert_error(
errorstore, InvalidSetting, 'chapter/chapter.xml',
"The tag <chapter url_name='chapter' display_name='Hi there!'> has an invalid date setting for start: 'Feb 20, 2019, 17:00zzz'."
)
assert_error(
errorstore, DateOrdering, 'sequential/sequential.xml',
"The tag <sequential url_name='sequential' display_name='Hi there!'> has a date out of order: start date cannot be before course start date"
)
assert_error(
errorstore, DateOrdering, 'vertical/vertical.xml',
"The tag <vertical url_name='vertical' display_name='Hi mom!'> has a date out of order: due date must be before course end date"
)
assert_error(
errorstore, DateOrdering, 'problem/problem.xml',
"The tag <problem url_name='problem'> has a date out of order: start date must be before due date"
)
assert_error(
errorstore, MissingURLName, 'vertical/vertical.xml',
"The tag <problem display_name='no url_name'> has no url_name.")
assert_error(
errorstore, InvalidSetting, 'vertical/vertical.xml',
"The tag <problem display_name='no url_name'> has an invalid setting 'showanswer=hah!'."
)
assert_error(
errorstore, DateOrdering, 'vertical/vertical.xml',
"The tag <problem display_name='no url_name'> has a date out of order: due date must be before course end date"
)
assert_error(
errorstore, Obsolete, 'vertical/vertical.xml',
"The tag <discussion url_name='discussion'> should be included inline rather than through the discussion directory."
)
assert_error(
errorstore, InvalidSetting, 'discussion/discussion.xml',
"The tag <discussion url_name='discussion'> does not have the required setting 'discussion_id'."
)
assert_error(
errorstore, InvalidSetting, 'discussion/discussion.xml',
"The tag <discussion url_name='discussion'> does not have the required setting 'discussion_category'."
)
assert_error(
errorstore, InvalidSetting, 'discussion/discussion.xml',
"The tag <discussion url_name='discussion'> does not have the required setting 'discussion_target'."
)
assert_error(
errorstore, Obsolete, 'lti/lti.xml',
"The tag <lti url_name='lti'> should be converted to the newer lti_consumer Xblock."
)
assert_error(
errorstore, InvalidSetting, 'lti/lti.xml',
"The tag <lti url_name='lti'> does not have the required setting 'launch_url'."
)
assert_error(
errorstore, LTIError, 'lti/lti.xml',
"Course policy does not include an 'lti_passports' entry for 'nothere', required for <lti url_name='lti'>."
)
assert_error(
errorstore, LTIError, 'lti/lti.xml',
"Course policy does not include the 'lti' advanced module, required for <lti url_name='lti'>."
)
assert_error(
errorstore, InvalidSetting, 'vertical/vertical.xml',
"The tag <lti_consumer url_name='meep'> does not have the required setting 'launch_url'."
)
assert_error(
errorstore, LTIError, 'vertical/vertical.xml',
"Course policy does not include an 'lti_passports' entry for 'nothere', required for <lti_consumer url_name='meep'>."
)
assert_error(
errorstore, LTIError, 'vertical/vertical.xml',
"Course policy does not include the 'lti_consumer' advanced module, required for <lti_consumer url_name='meep'>."
)
assert_error(
errorstore, MissingFile, 'course/mycourseurl.xml',
"The <course url_name='mycourseurl'> tag contains a reference to a missing static file: course_image_small.jpg"
)
assert_error(
errorstore, InvalidSetting, 'problem/problem.xml',
"The tag <problem url_name='problem'> has a negative problem weight.")
assert_error(
errorstore, InvalidSetting, 'course/mycourseurl.xml',
"The tag <course url_name='mycourseurl'> should have a positive number of attempts."
)
assert_error(
errorstore, InvalidSetting, 'problem/problem.xml',
"The tag <problem url_name='problem'> should have a positive number of attempts."
)
assert_error(
errorstore, InvalidSetting, 'vertical/vertical.xml',
"The tag <drag-and-drop-v2 url_name='studio_mess2' display_name='This is my title'> has an error in the data JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)."
)
assert_error(
errorstore, InvalidSetting, 'vertical/vertical.xml',
"The tag <drag-and-drop-v2 url_name='studio_mess3' display_name='This is my title'> data JSON is not a dictionary."
)
assert_error(
errorstore, InvalidSetting, 'sequential/examseq.xml',
"The tag <sequential url_name='examseq' display_name='Exam'> is a timed exam, but the course policy does not have 'enable_timed_exams=true'."
)
assert_error(
errorstore, DateOrdering, 'vertical/oravert.xml',
"The tag <openassessment url_name='paper-draft'> has a date out of order: assessment 1 due date must be before course end date"
)
assert_caught_all_errors(errorstore)
# Ensure that we got the scripts from the problem file
assert set(
url_names['problem'].scripts) == {'python', 'perl', 'javascript'}
# Also make sure we detected the response types
assert set(url_names['problem'].response_types) == {
'customresponse', 'multiplechoiceresponse'
}
# as well as the input types
assert set(url_names['problem'].input_types) == {'choicegroup', 'textline'}
# We also found the solution
assert url_names['problem'].has_solution
# Make sure our exam sequential was detected
assert url_names['examseq'].is_exam
def test_validate_course7():
course, errorstore, url_names = validate(
"testcourses/testcourse7/course.xml", 5)
handle_course7_errors(errorstore)
assert_caught_all_errors(errorstore)
