def evaluate(generateSolutions,
testRoot,
moduleDict,
exceptionMap=ERROR_HINT_MAP,
edxOutput=False,
muteOutput=False,
gsOutput=False,
printTestCase=False,
questionToGrade=None,
display=None):
# imports of testbench code. note that the testClasses import must follow
# the import of student code due to dependencies
import testParser
import testClasses
for module in moduleDict:
setattr(sys.modules[__name__], module, moduleDict[module])
questions = []
questionDicts = {}
test_subdirs = getTestSubdirs(testParser, testRoot, questionToGrade)
for q in test_subdirs:
subdir_path = os.path.join(testRoot, q)
if not os.path.isdir(subdir_path) or q[0] == '.':
continue
# create a question object
questionDict = testParser.TestParser(
os.path.join(subdir_path, 'CONFIG')).parse()
questionClass = getattr(testClasses, questionDict['class'])
question = questionClass(questionDict, display)
questionDicts[q] = questionDict
# load test cases into question
tests = filter(lambda t: re.match('[^#~.].*\.test\Z', t),
os.listdir(subdir_path))
tests = map(lambda t: re.match('(.*)\.test\Z', t).group(1), tests)
for t in sorted(tests):
test_file = os.path.join(subdir_path, '%s.test' % t)
solution_file = os.path.join(subdir_path, '%s.solution' % t)
test_out_file = os.path.join(subdir_path, '%s.test_output' % t)
testDict = testParser.TestParser(test_file).parse()
if testDict.get("disabled", "false").lower() == "true":
continue
testDict['test_out_file'] = test_out_file
testClass = getattr(projectTestClasses, testDict['class'])
testCase = testClass(question, testDict)
def makefun(testCase, solution_file):
if generateSolutions:
# write solution file to disk
return lambda grades: testCase.writeSolution(
moduleDict, solution_file)
else:
# read in solution dictionary and pass as an argument
testDict = testParser.TestParser(test_file).parse()
solutionDict = testParser.TestParser(solution_file).parse()
if printTestCase:
return lambda grades: printTest(
testDict, solutionDict) or testCase.execute(
grades, moduleDict, solutionDict)
else:
return lambda grades: testCase.execute(
grades, moduleDict, solutionDict)
question.addTestCase(testCase, makefun(testCase, solution_file))
# Note extra function is necessary for scoping reasons
def makefun(question):
return lambda grades: question.execute(grades)
setattr(sys.modules[__name__], q, makefun(question))
questions.append((q, question.getMaxPoints()))
grades = grading.Grades(projectParams.PROJECT_NAME,
questions,
gsOutput=gsOutput,
edxOutput=edxOutput,
muteOutput=muteOutput)
if questionToGrade == None:
for q in questionDicts:
for prereq in questionDicts[q].get('depends', '').split():
grades.addPrereq(q, prereq)
grades.grade(sys.modules[__name__], bonusPic=projectParams.BONUS_PIC)
return grades.points
def evaluate(
generateSolutions,
testRoot,
moduleDict,
exceptionMap=ERROR_HINT_MAP,
edxOutput=False,
muteOutput=False,
printTestCase=False,
questionToGrade=None,
):
# imports of testbench code. note that the testClasses import must follow
# the import of student code due to dependencies
import testParser
import testClasses
for module in moduleDict:
setattr(sys.modules[__name__], module, moduleDict[module])
problemDict = testParser.TestParser(os.path.join(testRoot, "CONFIG")).parse()
# iterate through and run tests
if "order" in problemDict:
test_subdirs = problemDict["order"].split()
else:
test_subdirs = sorted(os.listdir(testRoot))
questions = []
questionDicts = {}
for q in test_subdirs:
subdir_path = os.path.join(testRoot, q)
if not os.path.isdir(subdir_path) or q[0] == ".":
continue
if questionToGrade != None and q != questionToGrade:
continue
# create a question object
questionDict = testParser.TestParser(
os.path.join(subdir_path, "CONFIG")
).parse()
questionClass = getattr(testClasses, questionDict["class"])
question = questionClass(questionDict)
questionDicts[q] = questionDict
# load test cases into question
tests = [t for t in os.listdir(subdir_path) if re.match("[^#~.].*\.test\Z", t)]
tests = [re.match("(.*)\.test\Z", t).group(1) for t in tests]
for t in sorted(tests):
test_file = os.path.join(subdir_path, "%s.test" % t)
solution_file = os.path.join(subdir_path, "%s.solution" % t)
test_out_file = os.path.join(subdir_path, "%s.test_output" % t)
testDict = testParser.TestParser(test_file).parse()
if testDict.get("disabled", "false").lower() == "true":
continue
testDict["test_out_file"] = test_out_file
testClass = getattr(projectTestClasses, testDict["class"])
testCase = testClass(question, testDict)
def makefun(testCase, solution_file):
if generateSolutions:
# write solution file to disk
return lambda grades: testCase.writeSolution(
moduleDict, solution_file
)
else:
# read in solution dictionary and pass as an argument
testDict = testParser.TestParser(test_file).parse()
solutionDict = testParser.TestParser(solution_file).parse()
if printTestCase:
return lambda grades: printTest(
testDict, solutionDict
) or testCase.execute(grades, moduleDict, solutionDict)
else:
return lambda grades: testCase.execute(
grades, moduleDict, solutionDict
)
question.addTestCase(testCase, makefun(testCase, solution_file))
# Note extra function is necessary for scoping reasons
def makefun(question):
return lambda grades: question.execute(grades)
setattr(sys.modules[__name__], q, makefun(question))
questions.append((q, question.getMaxPoints()))
grades = grading.Grades(
projectParams.PROJECT_NAME,
questions,
edxOutput=edxOutput,
muteOutput=muteOutput,
)
if questionToGrade == None:
for q in questionDicts:
for prereq in questionDicts[q].get("depends", "").split():
grades.addPrereq(q, prereq)
grades.grade(sys.modules[__name__])
return grades.points
def evaluate(generateSolutions,
testRoot,
moduleDict,
exceptionMap=ERROR_HINT_MAP,
edxOutput=False,
muteOutput=False):
# imports of testbench code. note that the testClasses import must follow
# the import of student code due to dependencies
import testParser
import testClasses
for module in moduleDict:
setattr(sys.modules[__name__], module, moduleDict[module])
# iterate through and run tests
test_subdirs = os.listdir(testRoot)
questions = []
for i in test_subdirs:
subdir_path = os.path.join(testRoot, i)
if not os.path.isdir(subdir_path) or i[0] == '.':
continue
# create a question object
questionDict = testParser.TestParser(
os.path.join(subdir_path, 'CONFIG')).parse()
questionClass = getattr(testClasses, questionDict['class'])
question = questionClass(questionDict)
# load test cases into question
tests = filter(lambda t: re.match('[^#~]*\.test\Z', t),
os.listdir(subdir_path))
tests = map(lambda t: re.match('(.*)\.test\Z', t).group(1), tests)
for t in tests:
test_file = os.path.join(subdir_path, '%s.test' % t)
solution_file = os.path.join(subdir_path, '%s.solution' % t)
testDict = testParser.TestParser(test_file).parse()
if testDict.get("disabled", "false").lower() == "true":
continue
testClass = getattr(projectTestClasses, testDict['class'])
testCase = testClass(testDict)
def makefun(testCase, solution_file):
if generateSolutions:
# write solution file to disk
return lambda grades: testCase.writeSolution(
moduleDict, solution_file)
else:
# read in solution dictionary and pass as an argument
solutionDict = testParser.TestParser(solution_file).parse()
return lambda grades: testCase.execute(
grades, moduleDict, solutionDict)
question.addTestCase(testCase, makefun(testCase, solution_file))
# Note extra function is necessary for scoping reasons
def makefun(question):
return lambda grades: question.execute(grades)
setattr(sys.modules[__name__], i, makefun(question))
questions.append((i, question.getMaxPoints()))
grades = grading.Grades(projectParams.PROJECT_NAME,
questions,
edxOutput=edxOutput,
muteOutput=muteOutput)
grades.grade(sys.modules[__name__])
return grades.points
# the import of student code due to dependencies
import testParser
import testClasses
for module in moduleDict:
setattr(sys.modules[__name__], module, moduleDict[module])
questions = []
questionDicts = {}
test_subdirs = getTestSubdirs(testParser, testRoot, questionToGrade)
for q in test_subdirs:
subdir_path = os.path.join(testRoot, q)
if not os.path.isdir(subdir_path) or q[0] == '.':
continue
# create a question object
questionDict = testParser.TestParser(os.path.join(subdir_path, 'CONFIG')).parse()
questionClass = getattr(testClasses, questionDict['class'])
question = questionClass(questionDict, display)
questionDicts[q] = questionDict
# load test cases into question
tests = filter(lambda t: re.match('[^#~.].*\.test\Z', t), os.listdir(subdir_path))
tests = map(lambda t: re.match('(.*)\.test\Z', t).group(1), tests)
for t in sorted(tests):
test_file = os.path.join(subdir_path, '%s.test' % t)
solution_file = os.path.join(subdir_path, '%s.solution' % t)
test_out_file = os.path.join(subdir_path, '%s.test_output' % t)
testDict = testParser.TestParser(test_file).parse()
if testDict.get("disabled", "false").lower() == "true":
continue
testDict['test_out_file'] = test_out_file
def readPowerBayesNet(self):
netFile = os.path.join('.', 'powersBayesNet.txt')
testDict = testParser.TestParser(netFile).parse()
parseDict = parseBayesNetProblem(testDict)
return parseDict['problemBayesNet']