def test_course2():
"""Test for XML loading errors"""
errorstore = ErrorStore()
course = load_course("testcourses/testcourse2", "coursefile.xml",
errorstore)
handle_course2_errors(errorstore)
assert_caught_all_errors(errorstore)
def test_course3():
"""Test for XML error in course.xml"""
errorstore = ErrorStore()
course = load_course("testcourses/testcourse3", "course.xml", errorstore)
assert_error(errorstore, InvalidXML, 'course.xml',
'attributes construct error, line 1, column 61')
assert_caught_all_errors(errorstore)
def test_course1():
"""Make sure that things load properly when everything is ok"""
errorstore = ErrorStore()
course = load_course("testcourses/testcourse1", "course.xml", errorstore)
assert errorstore.errors == []
assert course.attributes == {
'url_name': 'mycourseurl',
'org': 'myorg',
'course': 'mycourse'
}
[chapter] = course.children
assert chapter.attributes == {
'url_name': 'chapter',
'display_name': 'chapter name'
}
[sequential] = chapter.children
assert sequential.attributes == {
'url_name': 'sequential',
'display_name': 'display name'
}
[vertical] = sequential.children
assert vertical.attributes == {
'url_name': 'vertical',
'display_name': 'vertical name'
}
[html] = vertical.children
assert html.attributes == {'url_name': 'html', 'display_name': 'html name'}
assert '<p>Test course</p>' in etree.tostring(html.content,
pretty_print=True).decode()
def test_course7_url():
errorstore = ErrorStore()
# Load course
course = load_course("testcourses/testcourse7", "course.xml", errorstore)
assert_caught_all_errors(errorstore)
# Load policy
policy, grading_policy = load_policy("testcourses/testcourse7", course,
errorstore)
assert_caught_all_errors(errorstore)
# Make a dictionary of url_names
url_names = find_url_names(course, errorstore)
# Merge the policy file
merge_policy(policy, url_names, errorstore)
# Ensure that settings were indeed merged
assert (url_names['sequential2'].attributes['setting'])
# Validate the grading policy
validate_grading_policy(grading_policy, errorstore)
# Handle the errors
handle_course7_errors(errorstore)
assert_caught_all_errors(errorstore)
def test_no_policy():
errorstore = ErrorStore()
# Load course (needed before loading policy)
course = load_course("testcourses/testcourse1", "course.xml", errorstore)
assert_caught_all_errors(errorstore)
# Load the (nonexistent) policy files
policy, grading_policy = load_policy("testcourses/testcourse1", course,
errorstore)
handle_course1_errors(errorstore)
assert_caught_all_errors(errorstore)
def test_no_url_name():
errorstore = ErrorStore()
# Load course (needed before loading policy)
course = load_course("testcourses/testcourse4", "course.xml", errorstore)
assert_caught_all_errors(errorstore)
# Load the (nonexistent) policy files
policy, grading_policy = load_policy("testcourses/testcourse4", course,
errorstore)
assert_error(errorstore, NoRunName, 'course.xml',
"The course tag has no url_name.")
assert_caught_all_errors(errorstore)
def test_url_names():
errorstore = ErrorStore()
# Load course
course = load_course("testcourses/testcourse1", "course.xml", errorstore)
assert_caught_all_errors(errorstore)
# Make a dictionary of url_names
url_names = find_url_names(course, errorstore)
assert_caught_all_errors(errorstore)
expected = ['mycourseurl', 'chapter', 'sequential', 'vertical', 'html']
for i in expected:
assert i in url_names
def test_bad_json():
errorstore = ErrorStore()
# Load course (needed before loading policy)
course = load_course("testcourses/testcourse5", "course.xml", errorstore)
assert_caught_all_errors(errorstore)
# Load the policy files
policy, grading_policy = load_policy("testcourses/testcourse5", course,
errorstore)
assert_error(
errorstore, BadPolicy, 'policies/mycourseurl/policy.json',
"The policy file 'policies/mycourseurl/policy.json' has invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)"
)
assert_error(
errorstore, BadPolicy, 'policies/mycourseurl/grading_policy.json',
"The policy file 'policies/mycourseurl/grading_policy.json' has invalid JSON: Expecting property name enclosed in double quotes: line 2 column 3 (char 4)"
)
assert_caught_all_errors(errorstore)
def test_course6_url():
errorstore = ErrorStore()
# Load course
course = load_course("testcourses/testcourse6", "course.xml", errorstore)
assert_caught_all_errors(errorstore)
# Load policy
policy, grading_policy = load_policy("testcourses/testcourse6", course,
errorstore)
assert_caught_all_errors(errorstore)
# Make a dictionary of url_names
url_names = find_url_names(course, errorstore)
# Merge the policy file
merge_policy(policy, url_names, errorstore)
# Handle the errors
handle_course6_errors(errorstore)
assert_caught_all_errors(errorstore)
def test_no_course():
errorstore = ErrorStore()
course = load_course("testcourses", "nocourse.xml", errorstore)
handle_nocourse_errors(errorstore)
assert_caught_all_errors(errorstore)
def validate(filename, steps=8, ignore=None):
"""
Validate an OLX course by performing the given number of steps:
* 1: Load the course
* 2: Load the policy and grading policy
* 3: Validate url_names
* 4: Merge policy data with course, ensuring that all references are valid
* 5: Validate the grading policy
* 6: Have every object validate itself
* 7: Parse the course for global errors
* 8: Parse the course for global errors that may be time-consuming to detect
:param filename: Location of course xml file or directory
:param steps: Number of validation steps to take (1 = first only, 8 = all)
:param ignore: List of errors to ignore
:return: course object, errorstore object, url_names dictionary (or None if steps < 3)
"""
# Create an error store
if ignore is None:
ignore = []
errorstore = ErrorStore(ignore)
# Validation Step #1: Load the course
if os.path.isdir(filename):
directory = os.path.join(filename)
file = "course.xml"
else:
directory, file = os.path.split(filename)
course = load_course(directory, file, errorstore)
if not course:
return None, errorstore, None
if steps > 1:
# Validation Step #2: Load the policy files
policy, grading_policy = load_policy(directory, course, errorstore)
url_names = None
if steps > 2:
# Validation Step #3: Construct a dictionary of url_names
url_names = find_url_names(course, errorstore)
if steps > 3:
# Validation Step #4: Merge policy data into object attributes
merge_policy(policy, url_names, errorstore)
if steps > 4:
# Validation Step #5: Validate grading policy
validate_grading_policy(grading_policy, errorstore)
if steps > 5:
# Validation Step #6: Have every object validate itself
for edxobj in traverse(course):
edxobj.validate(course, errorstore)
if steps > 6:
# Validation Step #7: Parse the course for global errors
for validator in GlobalValidator.validators():
validator(course, errorstore, url_names)
if steps > 7:
# Validation Step #8: Parse the course for global errors that are time-consuming to detect
for validator in SlowValidator.validators():
validator(course, errorstore, url_names)
return course, errorstore, url_names
def test_grading_policy():
# Test grading policy entries in isolation
errorstore = ErrorStore()
# Set up the policy
policy = {}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"'GRADER' entry not found in grading policy")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"'GRADE_CUTOFFS' entry not found in grading policy")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {'GRADER': {}, 'GRADE_CUTOFFS': []}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"'GRADER' entry not in grading policy is not a list")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'GRADE_CUTOFFS' entry not in grading policy is not a dictionary")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Lab",
"short_label": 15,
"weight": 0.15
},
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 0.15
},
],
'GRADE_CUTOFFS': {
'Pass': -1
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"'weight' settings do not add up to 1")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'short_label' setting is not a string for entry 1 in the grading policy"
)
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 'hi',
"min_count": 'hi',
"type": 0,
"weight": 'hi'
},
{
"drop_count": -1,
"min_count": 0,
"type": "Something",
"weight": 1.1
},
{},
],
'GRADE_CUTOFFS': {
'Pass': 2
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'drop_count' setting is not an integer for entry 1 in the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'min_count' setting is not an integer for entry 1 in the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'type' setting is not a string for entry 1 in the grading policy")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'weight' setting is not a number between 0 and 1 for entry 1 in the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'drop_count' setting is negative for entry 2 in the grading policy")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'min_count' setting is less than 1 for entry 2 in the grading policy")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'weight' setting is not a number between 0 and 1 for entry 2 in the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'drop_count' setting is omitted for entry 3 in the grading policy")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'min_count' setting is omitted for entry 3 in the grading policy")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'type' setting is omitted for entry 3 in the grading policy")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'weight' setting is omitted for entry 3 in the grading policy")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"'weight' settings do not add up to 1")
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'Pass' entry is not between 0 and 1 in the GRADE_CUTOFFS part of the grading policy"
)
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 0.6
},
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 0.4
},
],
'GRADE_CUTOFFS': {
'A': -1,
'C': 3,
'D': 0.5,
'Pass': '******',
'pass': 1
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'A' entry is not between 0 and 1 in the GRADE_CUTOFFS part of the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'C' entry is not between 0 and 1 in the GRADE_CUTOFFS part of the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'pass' is not allowed in the GRADE_CUTOFFS part of the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"'Pass' entry is not a number in the GRADE_CUTOFFS part of the grading policy"
)
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"Assessment type 'Something' appears multiple times in the grading policy"
)
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'A': 0,
'C': 0.2,
'D': 0.5,
'Pass': 0.6,
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(
errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS should have either 'Pass' or letters, not both")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'A': 0,
'C': 0.2,
'D': 0.5
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is missing 'B'")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is entry 'C' is not decreasing")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is entry 'D' is not decreasing")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'A': 0,
'B': 0.1,
'C': 0.2,
'D': 0.5
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is entry 'B' is not decreasing")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is entry 'C' is not decreasing")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is entry 'D' is not decreasing")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {'GRADER': [], 'GRADE_CUTOFFS': {}}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADER entry in grading policy should not be empty")
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS entry in grading policy should not be empty")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'A': 1,
'C': 0.1
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is missing 'B'")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'B': 1,
'C': 0.1
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is missing 'A'")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'B': 1
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS is missing 'A'")
assert_caught_all_errors(errorstore)
# Set up the policy
policy = {
'GRADER': [
{
"drop_count": 2,
"min_count": 12,
"type": "Something",
"weight": 1
},
],
'GRADE_CUTOFFS': {
'A': 1
}
}
# Validate it
validate_grading_policy(policy, errorstore)
# Handle errors
assert_error(errorstore, GradingPolicyIssue, 'grading_policy.json',
"GRADE_CUTOFFS should use 'Pass' instead of 'A'")
assert_caught_all_errors(errorstore)
