def test_0(self):
studentL = {
"".join(s)
for s in exploreCFG(self.st.cfg, MAX_LENGTH_TO_CHECK + 3)
}
# The "+ 3" is for wiggle room
# See exploreCFG documentation for an explanation of why this is necessary
studentL = {s for s in studentL if len(s) <= MAX_LENGTH_TO_CHECK}
correctL = generateLanguage(MAX_LENGTH_TO_CHECK)
falsePositives = studentL.difference(correctL)
if falsePositives:
Feedback.add_feedback(
"Your CFG generated the string "
f"'{min(falsePositives, key=lambda s: len(s))}', "
"which is not in the language.")
return
falseNegatives = correctL.difference(studentL)
if falseNegatives:
Feedback.add_feedback(
"Your CFG did not generate the string "
f"'{min(falseNegatives, key=lambda s: len(s))}', "
"which is in the language.")
return
Feedback.set_score(1)
def test_0(self):
points = 0
# use the same input vector for both u and v to avoid ambiguity in the order of iteration
results = Feedback.call_user(
self.st.power_iteration, self.ref.M_test, self.ref.x_test)
if results is None:
Feedback.add_feedback(
'Your function is not returning any variables.')
self.fail()
if Feedback.check_numpy_array_allclose('The return value from your function power_iteration', self.ref.xc, results):
points += 0.5
results_hidden = Feedback.call_user(
self.st.power_iteration, self.ref.M_hidden, self.ref.x_hidden)
if Feedback.check_numpy_array_allclose('The return value from your function power_iteration (hidden test case)', self.ref.xc_hidden, results_hidden):
points += 0.5
Feedback.set_score(points)
def test_1(self):
for i in range(1, NUM_ELEMENTS_TO_CHECK + 1):
for j in range(i + 1, NUM_ELEMENTS_TO_CHECK + 1):
x = Feedback.call_user(self.st.getFoolingSetElement, i)
y = Feedback.call_user(self.st.getFoolingSetElement, j)
z = Feedback.call_user(self.st.getDistinguishingSuffix, i, j)
if not (isInLanguage(x + z) ^ isInLanguage(y + z)):
Feedback.add_feedback(
f"When i = {i} and j = {j}, the suffix\n\n"
f"z = '{z}'\n\n"
f"fails to distinguish the two fooling set elements\n\n"
f"x = '{x}'\n"
f"y = '{y}'")
Feedback.set_score(0)
return
Feedback.set_score(1)
def addError(self, test, err):
if isinstance(err[1], GradingComplete):
self.done_grading = True
self.addFailure(test, err)
elif isinstance(err[1], DoNotRun):
self.results[-1]['points'] = 0
self.results[-1]['max_points'] = 0
elif isinstance(err[1], UserCodeFailed):
# Student code raised Exception
tr_list = traceback.format_exception(*err[1].err)
name = 'Your code raised an Exception'
self.done_grading = True
if isinstance(err[1].err[1], SyntaxError) or isinstance(
err[1].err[1], NameError):
self.grading_succeeded = False
self.format_errors.append('Your code has a syntax error.')
Feedback.set_main_output()
else:
self.results.append({
'name': name,
'filename': 'error',
'max_points': 1,
'points': 0
})
Feedback.set_name('error')
Feedback.add_feedback(''.join(tr_list))
Feedback.add_feedback('\n\nYour code:\n\n')
print_student_code(st_code=Feedback.test.student_code_abs_path,
ipynb_key=Feedback.test.ipynb_key)
else:
tr_list = traceback.format_exception(*err)
test_id = test.id().split()[0]
if not test_id.startswith('test'):
# Error in setup code -- not recoverable
self.done_grading = True
self.grading_succeeded = False
self.results = []
self.results.append({
'name': 'Internal Grading Error',
'filename': 'error',
'max_points': 1,
'points': 0
})
Feedback.set_name('error')
Feedback.add_feedback(self.major_error_message +
''.join(tr_list))
else:
# Error in a single test -- keep going
unittest.TestResult.addError(self, test, err)
self.results[-1]['points'] = 0
Feedback.add_feedback(self.error_message + ''.join(tr_list))
def print_student_code(st_code='user_code.py',
ipynb_key='#grade',
as_feedback=True):
"""
Print the student's code, with syntax highlighting.
"""
with open(st_code, 'r', encoding='utf-8') as f:
filename, extension = splitext(st_code)
if extension == '.ipynb':
contents = extract_ipynb_contents(f, ipynb_key).strip()
lines = filter(lambda l: not l.strip().startswith(ipynb_key),
contents.split('\n'))
contents = '\n'.join(lines)
else:
contents = f.read().strip()
formatted = pygments.highlight(contents, PythonLexer(),
Terminal256Formatter(style='monokai'))
if as_feedback:
Feedback.add_feedback(formatted)
else:
return formatted
def test_0(self):
# Check type
for str_length in range(0, MAX_LENGTH_TO_CHECK + 1):
for char_list in itertools.product(alphabet, repeat=str_length):
x = "".join(char_list)
in_language = isInLanguage(x)
accepted = self.st.fa.accepts_input(x)
if in_language and not accepted:
Feedback.add_feedback(
f"Your DFA/NFA did not accept the string '{x}', "
"which is in the language.")
return
elif not in_language and accepted:
Feedback.add_feedback(
f"Your DFA/NFA accepted the string '{x}', "
"which is not in the language.")
return
Feedback.set_score(1)
def test_0(self):
user_val = Feedback.call_user(self.st.return_number)
if ((user_val % 2 == 0) and (self.ref.number_type == 'even')):
Feedback.set_score(1)
Feedback.add_feedback('Your answer is correct.')
elif ((user_val % 2 != 0) and (self.ref.number_type == 'odd')):
Feedback.set_score(1)
Feedback.add_feedback('Your answer is correct.')
else:
Feedback.set_score(0)
Feedback.add_feedback('The return value is not correct.')
def test_0(self):
allowed_chars = {'(', ')', '+', '*', 'e', ' '}.union(alphabet)
for char in self.st.reg_exp:
if char not in allowed_chars:
Feedback.add_feedback(
f"Your regular expression contains the character '{char}', "
"which is not permitted.")
return
processed_re = self.st.reg_exp.replace('e', '').replace('+',
'|').replace(
' ', '')
try:
compiled_re = re.compile(processed_re)
except re.error:
Feedback.add_feedback("Your regular expression failed to compile.")
return
for str_length in range(0, MAX_LENGTH_TO_CHECK + 1):
for char_list in itertools.product(alphabet, repeat=str_length):
x = "".join(char_list)
in_language = isInLanguage(x)
match = compiled_re.fullmatch(x)
if in_language and not match:
Feedback.add_feedback(
f"Your regular expression did not match the string '{x}', "
"which is in the language.")
return
elif not in_language and match:
Feedback.add_feedback(
f"Your regular expression matched the string '{x}', "
"which is not in the language.")
return
Feedback.set_score(1)
def test_1(self):
points = 0
results = feedback.call_user(self.st.array_to_scalar, self.ref.a1,
self.ref.a2)
## Testing if the return values are given correctly
if results is not None:
if hasattr(results, '__len__'):
if (len(results) != 2):
feedback.add_feedback(
'Your function is not returning the correct number of variables'
)
self.fail()
else:
(st_c, st_sumc) = results
else:
feedback.add_feedback(
'The return variables do not appear to be in tuple format')
self.fail()
else:
feedback.add_feedback(
'Your function is not returning any variables.')
self.fail()
if feedback.check_numpy_array_allclose('c',
self.ref.c,
st_c,
accuracy_critical=False):
points += 0.5
if feedback.check_scalar('sum_c',
self.ref.sum_c,
st_sumc,
accuracy_critical=True):
points += 0.5
feedback.set_score(points)
def run_code(result, run_req):
# {{{ silence matplotlib font cache warnings
import warnings
warnings.filterwarnings(
"ignore", message="Matplotlib is building the font cache.*")
# }}}
# {{{ disable exit
disable_exit_lines = re.sub("\n ",
"\n", r"""
def _monkey_patch_exit():
print("Using exit() is not allowed")
exit = _monkey_patch_exit
""")
# }}}
# {{{ compile code
if getattr(run_req, "setup_code", None):
try:
setup_code = compile(
run_req.setup_code,
"\n".join(["[setup_code]", disable_exit_lines]),
'exec')
except Exception:
package_exception(result, "setup_compile_error")
return
else:
setup_code = None
try:
user_code = compile(
run_req.user_code, "[user code]", 'exec')
except Exception:
package_exception(result, "user_compile_error")
return
if getattr(run_req, "test_code", None):
try:
test_code = compile(
run_req.test_code, "[test code]", 'exec')
except Exception:
package_exception(result, "test_compile_error")
return
else:
test_code = None
# }}}
if hasattr(run_req, "compile_only") and run_req.compile_only:
result["result"] = "success"
return
# {{{ run code
data_files = {}
if hasattr(run_req, "data_files"):
from base64 import b64decode
for name, contents in run_req.data_files.items():
data_files[name] = b64decode(contents.encode())
generated_html = []
result["html"] = generated_html
def output_html(s):
generated_html.append(s)
feedback = Feedback()
maint_ctx = {
"feedback": feedback,
"user_code": user_code,
"data_files": data_files,
"output_html": output_html,
"GradingComplete": GradingComplete,
}
if setup_code is not None:
try:
exec(setup_code, maint_ctx)
except Exception:
package_exception(result, "setup_error")
return
user_ctx = {}
if hasattr(run_req, "names_for_user"):
for name in run_req.names_for_user:
if name not in maint_ctx:
result["result"] = "setup_error"
result["message"] = "Setup code did not define '%s'." % name
user_ctx[name] = maint_ctx[name]
from copy import deepcopy
user_ctx = deepcopy(user_ctx)
try:
exec(user_code, user_ctx)
except Exception:
package_exception(result, "user_error")
return
# {{{ export plots
if "matplotlib" in sys.modules:
import matplotlib.pyplot as pt
from io import BytesIO
from base64 import b64encode
format = "png"
mime = "image/png"
figures = []
for fignum in pt.get_fignums():
pt.figure(fignum)
bio = BytesIO()
try:
pt.savefig(bio, format=format)
except Exception:
pass
else:
figures.append(
(fignum, mime, b64encode(bio.getvalue()).decode()))
result["figures"] = figures
# }}}
if hasattr(run_req, "names_from_user"):
for name in run_req.names_from_user:
if name not in user_ctx:
feedback.add_feedback(
"Required answer variable '%s' is not defined."
% name)
maint_ctx[name] = None
else:
maint_ctx[name] = user_ctx[name]
if test_code is not None:
try:
exec(test_code, maint_ctx)
except GradingComplete:
pass
except Exception:
package_exception(result, "test_error")
return
result["points"] = feedback.points
result["feedback"] = feedback.feedback_items
# }}}
result["result"] = "success"
def optional_test_plot_labels(self):
axes = self.plt.gca()
title = axes.get_title()
xlabel = axes.get_xlabel()
ylabel = axes.get_ylabel()
points = 0
if xlabel:
points += 1
Feedback.add_feedback('Plot has xlabel')
else:
Feedback.add_feedback('Plot is missing xlabel')
if title:
points += 1
Feedback.add_feedback('Plot has title')
else:
Feedback.add_feedback('Plot is missing title')
if ylabel:
points += 1
Feedback.add_feedback('Plot has ylabel')
else:
Feedback.add_feedback('Plot is missing ylabel')
Feedback.set_score(points / 3.0)
def run_code(result, run_req):
# {{{ silence matplotlib font cache warnings
import warnings
warnings.filterwarnings(
"ignore", message="Matplotlib is building the font cache.*")
# }}}
# {{{ compile code
if getattr(run_req, "setup_code", None):
try:
setup_code = compile(
run_req.setup_code, "[setup code]", 'exec')
except Exception:
package_exception(result, "setup_compile_error")
return
else:
setup_code = None
try:
user_code = compile(
run_req.user_code, "[user code]", 'exec')
except Exception:
package_exception(result, "user_compile_error")
return
if getattr(run_req, "test_code", None):
try:
test_code = compile(
run_req.test_code, "[test code]", 'exec')
except Exception:
package_exception(result, "test_compile_error")
return
else:
test_code = None
# }}}
if hasattr(run_req, "compile_only") and run_req.compile_only:
result["result"] = "success"
return
# {{{ run code
data_files = {}
if hasattr(run_req, "data_files"):
from base64 import b64decode
for name, contents in run_req.data_files.items():
data_files[name] = b64decode(contents.encode())
generated_html = []
result["html"] = generated_html
def output_html(s):
generated_html.append(s)
feedback = Feedback()
maint_ctx = {
"feedback": feedback,
"user_code": user_code,
"data_files": data_files,
"output_html": output_html,
"GradingComplete": GradingComplete,
}
if setup_code is not None:
try:
exec(setup_code, maint_ctx)
except Exception:
package_exception(result, "setup_error")
return
user_ctx = {}
if hasattr(run_req, "names_for_user"):
for name in run_req.names_for_user:
if name not in maint_ctx:
result["result"] = "setup_error"
result["message"] = "Setup code did not define '%s'." % name
user_ctx[name] = maint_ctx[name]
from copy import deepcopy
user_ctx = deepcopy(user_ctx)
try:
exec(user_code, user_ctx)
except Exception:
package_exception(result, "user_error")
return
# {{{ export plots
if "matplotlib" in sys.modules:
import matplotlib.pyplot as pt
from io import BytesIO
from base64 import b64encode
format = "png"
mime = "image/png"
figures = []
for fignum in pt.get_fignums():
pt.figure(fignum)
bio = BytesIO()
try:
pt.savefig(bio, format=format)
except Exception:
pass
else:
figures.append(
(fignum, mime, b64encode(bio.getvalue()).decode()))
result["figures"] = figures
# }}}
if hasattr(run_req, "names_from_user"):
for name in run_req.names_from_user:
if name not in user_ctx:
feedback.add_feedback(
"Required answer variable '%s' is not defined."
% name)
maint_ctx[name] = None
else:
maint_ctx[name] = user_ctx[name]
if test_code is not None:
try:
exec(test_code, maint_ctx)
except GradingComplete:
pass
except Exception:
package_exception(result, "test_error")
return
result["points"] = feedback.points
result["feedback"] = feedback.feedback_items
# }}}
result["result"] = "success"
