def test_0(self): studentL = { "".join(s) for s in exploreCFG(self.st.cfg, MAX_LENGTH_TO_CHECK + 3) } # The "+ 3" is for wiggle room # See exploreCFG documentation for an explanation of why this is necessary studentL = {s for s in studentL if len(s) <= MAX_LENGTH_TO_CHECK} correctL = generateLanguage(MAX_LENGTH_TO_CHECK) falsePositives = studentL.difference(correctL) if falsePositives: Feedback.add_feedback( "Your CFG generated the string " f"'{min(falsePositives, key=lambda s: len(s))}', " "which is not in the language.") return falseNegatives = correctL.difference(studentL) if falseNegatives: Feedback.add_feedback( "Your CFG did not generate the string " f"'{min(falseNegatives, key=lambda s: len(s))}', " "which is in the language.") return Feedback.set_score(1)
def test_0(self): points = 0 # use the same input vector for both u and v to avoid ambiguity in the order of iteration results = Feedback.call_user( self.st.power_iteration, self.ref.M_test, self.ref.x_test) if results is None: Feedback.add_feedback( 'Your function is not returning any variables.') self.fail() if Feedback.check_numpy_array_allclose('The return value from your function power_iteration', self.ref.xc, results): points += 0.5 results_hidden = Feedback.call_user( self.st.power_iteration, self.ref.M_hidden, self.ref.x_hidden) if Feedback.check_numpy_array_allclose('The return value from your function power_iteration (hidden test case)', self.ref.xc_hidden, results_hidden): points += 0.5 Feedback.set_score(points)
def test_1(self): for i in range(1, NUM_ELEMENTS_TO_CHECK + 1): for j in range(i + 1, NUM_ELEMENTS_TO_CHECK + 1): x = Feedback.call_user(self.st.getFoolingSetElement, i) y = Feedback.call_user(self.st.getFoolingSetElement, j) z = Feedback.call_user(self.st.getDistinguishingSuffix, i, j) if not (isInLanguage(x + z) ^ isInLanguage(y + z)): Feedback.add_feedback( f"When i = {i} and j = {j}, the suffix\n\n" f"z = '{z}'\n\n" f"fails to distinguish the two fooling set elements\n\n" f"x = '{x}'\n" f"y = '{y}'") Feedback.set_score(0) return Feedback.set_score(1)
def addError(self, test, err): if isinstance(err[1], GradingComplete): self.done_grading = True self.addFailure(test, err) elif isinstance(err[1], DoNotRun): self.results[-1]['points'] = 0 self.results[-1]['max_points'] = 0 elif isinstance(err[1], UserCodeFailed): # Student code raised Exception tr_list = traceback.format_exception(*err[1].err) name = 'Your code raised an Exception' self.done_grading = True if isinstance(err[1].err[1], SyntaxError) or isinstance( err[1].err[1], NameError): self.grading_succeeded = False self.format_errors.append('Your code has a syntax error.') Feedback.set_main_output() else: self.results.append({ 'name': name, 'filename': 'error', 'max_points': 1, 'points': 0 }) Feedback.set_name('error') Feedback.add_feedback(''.join(tr_list)) Feedback.add_feedback('\n\nYour code:\n\n') print_student_code(st_code=Feedback.test.student_code_abs_path, ipynb_key=Feedback.test.ipynb_key) else: tr_list = traceback.format_exception(*err) test_id = test.id().split()[0] if not test_id.startswith('test'): # Error in setup code -- not recoverable self.done_grading = True self.grading_succeeded = False self.results = [] self.results.append({ 'name': 'Internal Grading Error', 'filename': 'error', 'max_points': 1, 'points': 0 }) Feedback.set_name('error') Feedback.add_feedback(self.major_error_message + ''.join(tr_list)) else: # Error in a single test -- keep going unittest.TestResult.addError(self, test, err) self.results[-1]['points'] = 0 Feedback.add_feedback(self.error_message + ''.join(tr_list))
def print_student_code(st_code='user_code.py', ipynb_key='#grade', as_feedback=True): """ Print the student's code, with syntax highlighting. """ with open(st_code, 'r', encoding='utf-8') as f: filename, extension = splitext(st_code) if extension == '.ipynb': contents = extract_ipynb_contents(f, ipynb_key).strip() lines = filter(lambda l: not l.strip().startswith(ipynb_key), contents.split('\n')) contents = '\n'.join(lines) else: contents = f.read().strip() formatted = pygments.highlight(contents, PythonLexer(), Terminal256Formatter(style='monokai')) if as_feedback: Feedback.add_feedback(formatted) else: return formatted
def test_0(self): # Check type for str_length in range(0, MAX_LENGTH_TO_CHECK + 1): for char_list in itertools.product(alphabet, repeat=str_length): x = "".join(char_list) in_language = isInLanguage(x) accepted = self.st.fa.accepts_input(x) if in_language and not accepted: Feedback.add_feedback( f"Your DFA/NFA did not accept the string '{x}', " "which is in the language.") return elif not in_language and accepted: Feedback.add_feedback( f"Your DFA/NFA accepted the string '{x}', " "which is not in the language.") return Feedback.set_score(1)
def test_0(self): user_val = Feedback.call_user(self.st.return_number) if ((user_val % 2 == 0) and (self.ref.number_type == 'even')): Feedback.set_score(1) Feedback.add_feedback('Your answer is correct.') elif ((user_val % 2 != 0) and (self.ref.number_type == 'odd')): Feedback.set_score(1) Feedback.add_feedback('Your answer is correct.') else: Feedback.set_score(0) Feedback.add_feedback('The return value is not correct.')
def test_0(self): allowed_chars = {'(', ')', '+', '*', 'e', ' '}.union(alphabet) for char in self.st.reg_exp: if char not in allowed_chars: Feedback.add_feedback( f"Your regular expression contains the character '{char}', " "which is not permitted.") return processed_re = self.st.reg_exp.replace('e', '').replace('+', '|').replace( ' ', '') try: compiled_re = re.compile(processed_re) except re.error: Feedback.add_feedback("Your regular expression failed to compile.") return for str_length in range(0, MAX_LENGTH_TO_CHECK + 1): for char_list in itertools.product(alphabet, repeat=str_length): x = "".join(char_list) in_language = isInLanguage(x) match = compiled_re.fullmatch(x) if in_language and not match: Feedback.add_feedback( f"Your regular expression did not match the string '{x}', " "which is in the language.") return elif not in_language and match: Feedback.add_feedback( f"Your regular expression matched the string '{x}', " "which is not in the language.") return Feedback.set_score(1)
def test_1(self): points = 0 results = feedback.call_user(self.st.array_to_scalar, self.ref.a1, self.ref.a2) ## Testing if the return values are given correctly if results is not None: if hasattr(results, '__len__'): if (len(results) != 2): feedback.add_feedback( 'Your function is not returning the correct number of variables' ) self.fail() else: (st_c, st_sumc) = results else: feedback.add_feedback( 'The return variables do not appear to be in tuple format') self.fail() else: feedback.add_feedback( 'Your function is not returning any variables.') self.fail() if feedback.check_numpy_array_allclose('c', self.ref.c, st_c, accuracy_critical=False): points += 0.5 if feedback.check_scalar('sum_c', self.ref.sum_c, st_sumc, accuracy_critical=True): points += 0.5 feedback.set_score(points)
def run_code(result, run_req): # {{{ silence matplotlib font cache warnings import warnings warnings.filterwarnings( "ignore", message="Matplotlib is building the font cache.*") # }}} # {{{ disable exit disable_exit_lines = re.sub("\n ", "\n", r""" def _monkey_patch_exit(): print("Using exit() is not allowed") exit = _monkey_patch_exit """) # }}} # {{{ compile code if getattr(run_req, "setup_code", None): try: setup_code = compile( run_req.setup_code, "\n".join(["[setup_code]", disable_exit_lines]), 'exec') except Exception: package_exception(result, "setup_compile_error") return else: setup_code = None try: user_code = compile( run_req.user_code, "[user code]", 'exec') except Exception: package_exception(result, "user_compile_error") return if getattr(run_req, "test_code", None): try: test_code = compile( run_req.test_code, "[test code]", 'exec') except Exception: package_exception(result, "test_compile_error") return else: test_code = None # }}} if hasattr(run_req, "compile_only") and run_req.compile_only: result["result"] = "success" return # {{{ run code data_files = {} if hasattr(run_req, "data_files"): from base64 import b64decode for name, contents in run_req.data_files.items(): data_files[name] = b64decode(contents.encode()) generated_html = [] result["html"] = generated_html def output_html(s): generated_html.append(s) feedback = Feedback() maint_ctx = { "feedback": feedback, "user_code": user_code, "data_files": data_files, "output_html": output_html, "GradingComplete": GradingComplete, } if setup_code is not None: try: exec(setup_code, maint_ctx) except Exception: package_exception(result, "setup_error") return user_ctx = {} if hasattr(run_req, "names_for_user"): for name in run_req.names_for_user: if name not in maint_ctx: result["result"] = "setup_error" result["message"] = "Setup code did not define '%s'." % name user_ctx[name] = maint_ctx[name] from copy import deepcopy user_ctx = deepcopy(user_ctx) try: exec(user_code, user_ctx) except Exception: package_exception(result, "user_error") return # {{{ export plots if "matplotlib" in sys.modules: import matplotlib.pyplot as pt from io import BytesIO from base64 import b64encode format = "png" mime = "image/png" figures = [] for fignum in pt.get_fignums(): pt.figure(fignum) bio = BytesIO() try: pt.savefig(bio, format=format) except Exception: pass else: figures.append( (fignum, mime, b64encode(bio.getvalue()).decode())) result["figures"] = figures # }}} if hasattr(run_req, "names_from_user"): for name in run_req.names_from_user: if name not in user_ctx: feedback.add_feedback( "Required answer variable '%s' is not defined." % name) maint_ctx[name] = None else: maint_ctx[name] = user_ctx[name] if test_code is not None: try: exec(test_code, maint_ctx) except GradingComplete: pass except Exception: package_exception(result, "test_error") return result["points"] = feedback.points result["feedback"] = feedback.feedback_items # }}} result["result"] = "success"
def optional_test_plot_labels(self): axes = self.plt.gca() title = axes.get_title() xlabel = axes.get_xlabel() ylabel = axes.get_ylabel() points = 0 if xlabel: points += 1 Feedback.add_feedback('Plot has xlabel') else: Feedback.add_feedback('Plot is missing xlabel') if title: points += 1 Feedback.add_feedback('Plot has title') else: Feedback.add_feedback('Plot is missing title') if ylabel: points += 1 Feedback.add_feedback('Plot has ylabel') else: Feedback.add_feedback('Plot is missing ylabel') Feedback.set_score(points / 3.0)
def run_code(result, run_req): # {{{ silence matplotlib font cache warnings import warnings warnings.filterwarnings( "ignore", message="Matplotlib is building the font cache.*") # }}} # {{{ compile code if getattr(run_req, "setup_code", None): try: setup_code = compile( run_req.setup_code, "[setup code]", 'exec') except Exception: package_exception(result, "setup_compile_error") return else: setup_code = None try: user_code = compile( run_req.user_code, "[user code]", 'exec') except Exception: package_exception(result, "user_compile_error") return if getattr(run_req, "test_code", None): try: test_code = compile( run_req.test_code, "[test code]", 'exec') except Exception: package_exception(result, "test_compile_error") return else: test_code = None # }}} if hasattr(run_req, "compile_only") and run_req.compile_only: result["result"] = "success" return # {{{ run code data_files = {} if hasattr(run_req, "data_files"): from base64 import b64decode for name, contents in run_req.data_files.items(): data_files[name] = b64decode(contents.encode()) generated_html = [] result["html"] = generated_html def output_html(s): generated_html.append(s) feedback = Feedback() maint_ctx = { "feedback": feedback, "user_code": user_code, "data_files": data_files, "output_html": output_html, "GradingComplete": GradingComplete, } if setup_code is not None: try: exec(setup_code, maint_ctx) except Exception: package_exception(result, "setup_error") return user_ctx = {} if hasattr(run_req, "names_for_user"): for name in run_req.names_for_user: if name not in maint_ctx: result["result"] = "setup_error" result["message"] = "Setup code did not define '%s'." % name user_ctx[name] = maint_ctx[name] from copy import deepcopy user_ctx = deepcopy(user_ctx) try: exec(user_code, user_ctx) except Exception: package_exception(result, "user_error") return # {{{ export plots if "matplotlib" in sys.modules: import matplotlib.pyplot as pt from io import BytesIO from base64 import b64encode format = "png" mime = "image/png" figures = [] for fignum in pt.get_fignums(): pt.figure(fignum) bio = BytesIO() try: pt.savefig(bio, format=format) except Exception: pass else: figures.append( (fignum, mime, b64encode(bio.getvalue()).decode())) result["figures"] = figures # }}} if hasattr(run_req, "names_from_user"): for name in run_req.names_from_user: if name not in user_ctx: feedback.add_feedback( "Required answer variable '%s' is not defined." % name) maint_ctx[name] = None else: maint_ctx[name] = user_ctx[name] if test_code is not None: try: exec(test_code, maint_ctx) except GradingComplete: pass except Exception: package_exception(result, "test_error") return result["points"] = feedback.points result["feedback"] = feedback.feedback_items # }}} result["result"] = "success"