def test_interactive_bad_input(self):
"""Interactive autograder error handling, bad input
The interactive autograder needs to handle all sorts of
cases where the remote grader fails. The right behavior
is to retry until it can get a valid grade, and if it can't,
give up instead of scoring a failure.
"""
ag = AutoGrader(self.interactive_xml)
# we've seen a sick grader time out like this -- shouldn't grade
bad_return = r'{"score":0,"maximum-score":1,"feedback":[{"user_answer":"<result> {for $c1 in doc(\"countries.xml\")//country let $d1 := ($c1/@population div $c1/@area) for $c2 in doc(\"countries.xml\")//country let $d2:= ($c2/@population div $c2/@area) where $d1 >= $d2 return <highest density = \"{$d1}\">{data($c1/@name)} </highest>} {for $c1 in doc(\"countries.xml\")//country let $d1 := ($c1/@population div $c1/@area) for $c2 in doc(\"countries.xml\")//country let $d2:= ($c2/@population div $c2/@area) where $d1 <= $d2 return <lowest density = \"{$d1}\">{data($c1/@name)} </lowest>} </result>","score":0,"explanation":"Timeout Error"}]}'
with fake_remote_grader(bad_return):
with self.assertRaises(AutoGraderGradingException):
g = ag.grade("q1b", "should throw exception")
# we've seen a sick grader time out like this too
no_explanation_score0 = r'{"score":0, "maximum":1, "feedback":[{"explanation":""}]}'
with fake_remote_grader(no_explanation_score0):
with self.assertRaises(AutoGraderGradingException):
g = ag.grade("q1b", "should throw exception")
# but same thing with score=1 should be OK though (never penalize for no explanation)
no_explanation_score1 = r'{"score":1.0, "maximum":1, "feedback":[{"explanation":""}]}'
with fake_remote_grader(no_explanation_score1):
g = ag.grade("q1b", "should be OK")
self.assertEqual(g, {'correct': True, 'score': 1.0, 'feedback': [{"explanation": ""}] })
def test_interactive_bad_input(self):
"""Interactive autograder error handling, bad input
The interactive autograder needs to handle all sorts of
cases where the remote grader fails. The right behavior
is to retry until it can get a valid grade, and if it can't,
give up instead of scoring a failure.
"""
ag = AutoGrader(self.interactive_xml)
# we've seen a sick grader time out like this -- shouldn't grade
bad_return = r'{"score":0,"maximum-score":1,"feedback":[{"user_answer":"<result> {for $c1 in doc(\"countries.xml\")//country let $d1 := ($c1/@population div $c1/@area) for $c2 in doc(\"countries.xml\")//country let $d2:= ($c2/@population div $c2/@area) where $d1 >= $d2 return <highest density = \"{$d1}\">{data($c1/@name)} </highest>} {for $c1 in doc(\"countries.xml\")//country let $d1 := ($c1/@population div $c1/@area) for $c2 in doc(\"countries.xml\")//country let $d2:= ($c2/@population div $c2/@area) where $d1 <= $d2 return <lowest density = \"{$d1}\">{data($c1/@name)} </lowest>} </result>","score":0,"explanation":"Timeout Error"}]}'
with fake_remote_grader(bad_return):
with self.assertRaises(AutoGraderGradingException):
g = ag.grade("q1b", "should throw exception")
# we've seen a sick grader time out like this too
no_explanation_score0 = r'{"score":0, "maximum":1, "feedback":[{"explanation":""}]}'
with fake_remote_grader(no_explanation_score0):
with self.assertRaises(AutoGraderGradingException):
g = ag.grade("q1b", "should throw exception")
# but same thing with score=1 should be OK though (never penalize for no explanation)
no_explanation_score1 = r'{"score":1.0, "maximum":1, "feedback":[{"explanation":""}]}'
with fake_remote_grader(no_explanation_score1):
g = ag.grade("q1b", "should be OK")
self.assertEqual(g, {'correct': True, 'score': 1.0, 'feedback': [{"explanation": ""}] })
def test_interactive_bad_input(self):
"""Interactive autograder error handling, bad input
The interactive autograder needs to handle all sorts of
cases where the remote grader fails. The right behavior
is to retry until it can get a valid grade, and if it can't,
give up instead of scoring a failure.
"""
ag = AutoGrader(self.interactive_xml)
# this is on our watchword list
with fake_remote_grader('Timed Out'):
with self.assertRaises(AutoGraderGradingException):
g = ag.grade("q1b", "should throw exception")
# a watchword used elsewhere should succeed though. Let's say that "timeout"
# is a valid explanation for some reason.
fb = [{"user_answer": "timeout", "explanation": "Timed Out", "score": 1}]
fbstr = json.dumps(fb)
with fake_remote_grader('{"score":1, "maximum-score":1, "feedback":%s}' % fbstr):
g = ag.grade("q1b", "should succeed")
self.assertEqual(g, {'correct': True, 'score': 1, 'feedback': fb})
with fake_remote_grader_garbage(""):
with self.assertRaises(AutoGraderGradingException):
g = ag.grade("q1b", "should throw exception")
def test_interactive_grader_basic(self):
"""
Interactive autograder with fake remote endpoint (basic)
Uses some XML from a db class interactive exercise, but the actual values
aren't used since we just fake out the endpoint.
The trick here is overriding the global method that urllib2 uses to open
files. You have to remember to restore urllib2 to a good state before
finishing though otherwise urllib2 will be horked. Method cribbed from:
http://stackoverflow.com/questions/2276689/how-do-i-unit-test-a-module-that-relies-on-urllib2
Work for this is done in fake_remote_grader.py
"""
ag = AutoGrader(self.interactive_xml)
self.assertEqual(ag.points_possible, 2.0)
self.assertEqual(len(ag.grader_functions), 2)
# the feedback is opaque, so just carry that around
fb = [{"user_answer": "user-input", "explanation": "grader-output", "score": 0}]
fbstr = json.dumps(fb)
with fake_remote_grader('{"score":0, "maximum-score":1, "feedback":%s}' % fbstr):
g = ag.grade("q1b", "should_fail")
self.assertEqual(g, {'correct': False, 'score': 0, 'feedback': fb})
with fake_remote_grader('{"score":1, "maximum-score":1, "feedback":%s}' % fbstr):
g = ag.grade("q1b", "should_succeed")
self.assertEqual(g, {'correct': True, 'score': 1, 'feedback': fb})
with fake_remote_grader('{"score":10, "maximum-score":10, "feedback":%s}' % fbstr):
g = ag.grade("q1b", "should_succeed")
self.assertEqual(g, {'correct': True, 'score': 1, 'feedback': fb})
g = ag.grade("q2b", "should_succeed")
self.assertEqual(g, {'correct': True, 'score': 1, 'feedback': fb})
with fake_remote_grader('{"score":1, "feedback":%s}' % fbstr):
g = ag.grade("q1b", "should_succeed")
self.assertEqual(g, {'correct': True, 'score': 1, 'feedback': fb})
with fake_remote_grader('{"score":0, "feedback":%s}' % fbstr):
g = ag.grade("q1b", "should_fail")
self.assertEqual(g, {'correct': False, 'score': 0, 'feedback': fb})
def test_interactive_grader_basic(self):
"""
Interactive autograder with fake remote endpoint (basic)
Uses some XML from a db class interactive exercise, but the actual values
aren't used since we just fake out the endpoint.
The trick here is overriding the global method that urllib2 uses to open
files. You have to remember to restore urllib2 to a good state before
finishing though otherwise urllib2 will be horked. Method cribbed from:
http://stackoverflow.com/questions/2276689/how-do-i-unit-test-a-module-that-relies-on-urllib2
Work for this is done in fake_remote_grader.py
"""
ag = AutoGrader(self.interactive_xml)
self.assertEqual(ag.points_possible, 2.0)
self.assertEqual(len(ag.grader_functions), 2)
# the feedback is opaque, so just carry that around
fb = [{"user_answer": "user-input", "explanation": "grader-output", "score": 0}]
fbstr = json.dumps(fb)
with fake_remote_grader('{"score":0, "maximum-score":1, "feedback":%s}' % fbstr):
g = ag.grade("q1b", "should_fail")
self.assertEqual(g, {'correct': False, 'score': 0, 'feedback': fb})
with fake_remote_grader('{"score":1, "maximum-score":1, "feedback":%s}' % fbstr):
g = ag.grade("q1b", "should_succeed")
self.assertEqual(g, {'correct': True, 'score': 1, 'feedback': fb})
with fake_remote_grader('{"score":10, "maximum-score":10, "feedback":%s}' % fbstr):
g = ag.grade("q1b", "should_succeed")
self.assertEqual(g, {'correct': True, 'score': 1, 'feedback': fb})
g = ag.grade("q2b", "should_succeed")
self.assertEqual(g, {'correct': True, 'score': 1, 'feedback': fb})
with fake_remote_grader('{"score":1, "feedback":%s}' % fbstr):
g = ag.grade("q1b", "should_succeed")
self.assertEqual(g, {'correct': True, 'score': 1, 'feedback': fb})
with fake_remote_grader('{"score":0, "feedback":%s}' % fbstr):
g = ag.grade("q1b", "should_fail")
self.assertEqual(g, {'correct': False, 'score': 0, 'feedback': fb})
def test_interactive_grader_unicode(self):
"""Interactive autograder with fake remote endpoint (unicode)"""
ag = AutoGrader(self.interactive_xml)
ascii_string = "ascii test string"
unicode_string = u'娱乐资讯请点击' # "click infotainment" from china.com homepage
fb = [{"user_answer": "user-input", "explanation": "grader-output", "score": 0}]
fbstr = json.dumps(fb)
with fake_remote_grader('{"score":0, "feedback":%s}' % fbstr):
g = ag.grade("q1b", ascii_string)
self.assertEqual(g, {'correct': False, 'score': 0, 'feedback': fb})
with fake_remote_grader('{"score":0, "feedback":%s}' % fbstr):
g = ag.grade("q1b", unicode_string)
self.assertEqual(g, {'correct': False, 'score': 0, 'feedback': fb})
def test_interactive_grader_unicode(self):
"""Interactive autograder with fake remote endpoint (unicode)"""
ag = AutoGrader(self.interactive_xml)
ascii_string = "ascii test string"
unicode_string = u'娱乐资讯请点击' # "click infotainment" from china.com homepage
fb = [{"user_answer": "user-input", "explanation": "grader-output", "score": 0}]
fbstr = json.dumps(fb)
with fake_remote_grader('{"score":0, "feedback":%s}' % fbstr):
g = ag.grade("q1b", ascii_string)
self.assertEqual(g, {'correct': False, 'score': 0, 'feedback': fb})
with fake_remote_grader('{"score":0, "feedback":%s}' % fbstr):
g = ag.grade("q1b", unicode_string)
self.assertEqual(g, {'correct': False, 'score': 0, 'feedback': fb})
