def setUp(self): """ On test start, initialise the points and set up the code feedback library to provide feedback for this test. """ self.points = 0 Feedback.set_test(self)
def test_4(self): if Feedback.check_numpy_array_allclose('right_side', self.ref.right_side, self.st.right_side): Feedback.set_score(1) else: Feedback.set_score(0)
def setUpClass(self): """ On start, run the user code and generate answer tuples. """ Feedback.set_test(self) base_dir = os.environ.get("MERGE_DIR") job_dir = os.environ.get("JOB_DIR") filenames_dir = os.environ.get("FILENAMES_DIR") self.student_code_abs_path = join(base_dir, self.student_code_file) # Load data so that we can use it in the test cases filenames_dir = os.environ.get("FILENAMES_DIR") with open(join(filenames_dir, 'data.json')) as f: self.data = json.load(f) ref_result, student_result, plot_value = execute_code( join(filenames_dir, 'ans.py'), join(base_dir, self.student_code_file), self.include_plt, join(base_dir, 'output.txt'), self.iter_num) answerTuple = namedtuple('answerTuple', ref_result.keys()) self.ref = answerTuple(**ref_result) studentTuple = namedtuple('studentTuple', student_result.keys()) self.st = studentTuple(**student_result) self.plt = plot_value if self.include_plt: self.display_plot()
def test_0(self): if self.ref.lcs("crows eat yay", "ows eat ") == self.st.lcs("crows eat yay", "ows eat "): Feedback.set_score(1) else: Feedback.set_score(0)
def test_2(self): user_dict = Feedback.call_user(self.st.bif_dict, 7) correct_dict = self.ref.bif_dict(7) if self.check(user_dict, correct_dict): Feedback.set_score(1) else: Feedback.set_score(0)
def test_0(self): if Feedback.check_plot('plot', self.ref.plot, self.st.plot, check_axes_scale='xy'): Feedback.set_score(1) else: Feedback.set_score(0)
def test_0(self): if Feedback.check_dataframe("df", self.ref.df, self.st.df, display_input=True): Feedback.set_score(1) else: Feedback.set_score(0)
def test_0(self): num_distinct_elements = len({ Feedback.call_user(self.st.getFoolingSetElement, i) for i in range(1, NUM_ELEMENTS_TO_CHECK + 1) }) if num_distinct_elements == NUM_ELEMENTS_TO_CHECK: Feedback.set_score(1) else: Feedback.set_score(0)
def test_3(self): points = 0 num_tests = 10 test_values = np.random.choice(np.arange(2, 30), size=num_tests, replace=False) for in_val in test_values: correct_val = self.ref.fib(in_val) user_val = Feedback.call_user(self.st.fib, in_val) if Feedback.check_scalar(f"fib({in_val})", correct_val, user_val): points += 1 Feedback.set_score(points / num_tests)
def check(self, user_dict, correct_dict): if not isinstance(user_dict, dict) or len(user_dict) != len(correct_dict): Feedback.set_score(0) return False for key in correct_dict: if key not in user_dict or correct_dict[key] != user_dict[key]: Feedback.set_score(0) return False return True
def test_0(self): sp19_enrollment = {"Murtaza": 10, "Lara": 20, "Mansi": 20, "Niki": 15, "Brendan": 10} sp19_attendance = {"Murtaza": [10, 9, 10, 10, 10, 2, 9, 10, 9, 1], "Lara": [19, 18, 16, 14, 12, 11, 10, 5, 5, 5], "Mansi": [25, 22, 23, 24, 25, 22, 22, 20, 25, 25], "Niki": [20, 15, 15, 12, 17, 17, 17, 16, 18, 20], "Brendan": [5, 5, 4, 2, 3, 10, 11, 12, 4, 2]} user_val = Feedback.call_user(self.st.TAs_with_at_least_half_discussions_full, sp19_enrollment, sp19_attendance) if self.ref.TAs_with_at_least_half_discussions_full(sp19_enrollment, sp19_attendance) == user_val: Feedback.set_score(1) else: Feedback.set_score(0)
def test_2(self): enrollment = {"A": 8, "B": 12, "C": 23, "D": 17, "E": 5} attendance = {"A": [8, 9, 8, 10, 10, 2, 6, 7, 3, 1], "B": [19, 18, 16, 14, 12, 11, 10, 5, 5, 5], "C": [25, 22, 23, 24, 25, 22, 22, 20, 25, 25], "D": [20, 15, 15, 12, 17, 17, 16, 16, 18, 20], "E": [5, 5, 4, 2, 3, 10, 11, 12, 4, 2]} user_val = Feedback.call_user(self.st.TAs_with_at_least_half_discussions_full, enrollment, attendance) if self.ref.TAs_with_at_least_half_discussions_full(enrollment, attendance) == user_val: Feedback.set_score(1) else: Feedback.set_score(0)
def test_3(self): points = 0 num_tests = 8 test_values = np.random.choice(np.arange(6, 20), size=num_tests, replace=False) for value in test_values: user_dict = Feedback.call_user(self.st.bif_dict, value) correct_dict = self.ref.bif_dict(value) if self.check(user_dict, correct_dict): points += 1 Feedback.set_score(points / num_tests)
def test_0(self): points = 0 a = np.random.rand(4) b = np.random.rand(4) user_val = Feedback.call_user(self.st.my_dot_product, a, b) ref_val = Feedback.call_user(self.ref.my_dot_product, a, b) if Feedback.check_scalar("my_dot_product return value", ref_val, user_val): points += 1 Feedback.set_score(points)
def test_1(self): sequence = "uc berkeley also cal" if self.ref.function == "most_common": user_val = Feedback.call_user(self.st.most_common, sequence) if self.ref.most_common(sequence) == user_val: Feedback.set_score(1) else: Feedback.set_score(0) elif self.ref.function == "least_common": user_val = Feedback.call_user(self.st.least_common, sequence) if self.ref.least_common(sequence) == user_val: Feedback.set_score(1) else: Feedback.set_score(0)
def test_3(self): sequence = "sphinxofquartz" if self.ref.function == "most_common": user_val = Feedback.call_user(self.st.most_common, sequence) if self.ref.most_common(sequence) == user_val: Feedback.set_score(1) else: Feedback.set_score(0) elif self.ref.function == "least_common": user_val = Feedback.call_user(self.st.least_common, sequence) if self.ref.least_common(sequence) == user_val: Feedback.set_score(1) else: Feedback.set_score(0)
def test_2(self): sequence = [1, 2, 3, 4, 5, 6, 7, 8] if self.ref.function == "most_common": user_val = Feedback.call_user(self.st.most_common, sequence) if self.ref.most_common(sequence) == user_val: Feedback.set_score(1) else: Feedback.set_score(0) elif self.ref.function == "least_common": user_val = Feedback.call_user(self.st.least_common, sequence) if self.ref.least_common(sequence) == user_val: Feedback.set_score(1) else: Feedback.set_score(0)
def test_0(self): points = 0 feedback.check_numpy_array_features('b', self.ref.a, self.st.b, accuracy_critical=True) suma = self.ref.beta * np.sum(self.ref.a) sumb = np.sum(self.st.b) if feedback.check_scalar('b', suma, sumb): points += 1 feedback.set_score(points)
def test_4(self): sequence = [] for i in range(20): sequence += [random.randint(0, 20)] if self.ref.function == "most_common": user_val = Feedback.call_user(self.st.most_common, sequence) if self.ref.most_common(sequence) == user_val: Feedback.set_score(1) else: Feedback.set_score(0) elif self.ref.function == "least_common": user_val = Feedback.call_user(self.st.least_common, sequence) if self.ref.least_common(sequence) == user_val: Feedback.set_score(1) else: Feedback.set_score(0)
def test_0(self): tests = 50 points = 0 xs = np.linspace(-1, 1, tests) alphas = np.linspace(0.1, 0.001, tests) np.random.shuffle(xs) np.random.shuffle(alphas) for i in range(tests): x = xs[i] alpha = alphas[i] ref_xnew = self.ref.gradient_descent(x, alpha) st_xnew = Feedback.call_user(self.st.gradient_descent, x, alpha) if Feedback.check_scalar(f'x={x}, alpha={alpha}', ref_xnew, st_xnew, report_success=True, report_failure=True): points += 1 Feedback.set_score(points / tests)
def setUpClass(cls): """ On start, run the user code and generate answer tuples. """ Feedback.set_test(cls) filenames_dir = os.environ.get("FILENAMES_DIR") base_dir = os.environ.get("MERGE_DIR") cls.student_code_abs_path = join(base_dir, cls.student_code_file) ref_result, student_result, plot_value = execute_code( join(filenames_dir, 'ans.py'), join(base_dir, cls.student_code_file), cls.include_plt, join(base_dir, 'output.txt'), cls.iter_num) answerTuple = namedtuple('answerTuple', ref_result.keys()) cls.ref = answerTuple(**ref_result) studentTuple = namedtuple('studentTuple', student_result.keys()) cls.st = studentTuple(**student_result) cls.plt = plot_value if cls.include_plt: cls.display_plot()
def test_0(self): studentL = { "".join(s) for s in exploreCFG(self.st.cfg, MAX_LENGTH_TO_CHECK + 3) } # The "+ 3" is for wiggle room # See exploreCFG documentation for an explanation of why this is necessary studentL = {s for s in studentL if len(s) <= MAX_LENGTH_TO_CHECK} correctL = generateLanguage(MAX_LENGTH_TO_CHECK) falsePositives = studentL.difference(correctL) if falsePositives: Feedback.add_feedback( "Your CFG generated the string " f"'{min(falsePositives, key=lambda s: len(s))}', " "which is not in the language.") return falseNegatives = correctL.difference(studentL) if falseNegatives: Feedback.add_feedback( "Your CFG did not generate the string " f"'{min(falseNegatives, key=lambda s: len(s))}', " "which is in the language.") return Feedback.set_score(1)
def test_1(self): for i in range(1, NUM_ELEMENTS_TO_CHECK + 1): for j in range(i + 1, NUM_ELEMENTS_TO_CHECK + 1): x = Feedback.call_user(self.st.getFoolingSetElement, i) y = Feedback.call_user(self.st.getFoolingSetElement, j) z = Feedback.call_user(self.st.getDistinguishingSuffix, i, j) if not (isInLanguage(x + z) ^ isInLanguage(y + z)): Feedback.add_feedback( f"When i = {i} and j = {j}, the suffix\n\n" f"z = '{z}'\n\n" f"fails to distinguish the two fooling set elements\n\n" f"x = '{x}'\n" f"y = '{y}'") Feedback.set_score(0) return Feedback.set_score(1)
def print_student_code(st_code='user_code.py', ipynb_key='#grade', as_feedback=True): """ Print the student's code, with syntax highlighting. """ with open(st_code, 'r', encoding='utf-8') as f: filename, extension = splitext(st_code) if extension == '.ipynb': contents = extract_ipynb_contents(f, ipynb_key).strip() lines = filter(lambda l: not l.strip().startswith(ipynb_key), contents.split('\n')) contents = '\n'.join(lines) else: contents = f.read().strip() formatted = pygments.highlight(contents, PythonLexer(), Terminal256Formatter(style='monokai')) if as_feedback: Feedback.add_feedback(formatted) else: return formatted
def test_3(self): enrollment = {"A": 8, "B": 12, "C": 23, "D": 17, "E": 5, "Murtaza": 10, "Lara": 20, "Mansi": 20, "Niki": 15, "Brendan": 10, "F": 10, "G": 20, "H": 30, "I": 20, "J": 10} attendance = {"A": [8, 9, 8, 10, 10, 2, 6, 7, 3, 1], "B": [19, 18, 16, 14, 12, 11, 10, 5, 5, 5], "C": [25, 22, 23, 24, 25, 22, 22, 20, 25, 25], "D": [20, 15, 15, 12, 17, 17, 16, 16, 18, 20], "E": [5, 5, 4, 2, 3, 10, 11, 12, 4, 2], "Murtaza": [10, 9, 10, 10, 10, 2, 9, 10, 9, 1], "Lara": [19, 18, 16, 14, 12, 11, 10, 5, 5, 5], "Mansi": [25, 22, 23, 24, 25, 22, 22, 20, 25, 25], "Niki": [20, 15, 15, 12, 17, 17, 17, 16, 18, 20], "Brendan": [5, 5, 4, 2, 3, 10, 11, 12, 4, 2], "F": [8, 9, 8, 10, 10, 2, 9, 10, 9, 1], "G": [19, 18, 16, 14, 12, 11, 10, 5, 5, 5], "H": [25, 22, 23, 24, 25, 22, 22, 20, 25, 25], "I": [20, 15, 15, 12, 17, 17, 17, 16, 18, 20], "J": [5, 5, 4, 2, 3, 10, 11, 12, 4, 2]} user_val = Feedback.call_user(self.st.TAs_with_at_least_half_discussions_full, enrollment, attendance) if self.ref.TAs_with_at_least_half_discussions_full(enrollment, attendance) == user_val: Feedback.set_score(1) else: Feedback.set_score(0)
def test_0(self): points = 0 # use the same input vector for both u and v to avoid ambiguity in the order of iteration results = Feedback.call_user( self.st.power_iteration, self.ref.M_test, self.ref.x_test) if results is None: Feedback.add_feedback( 'Your function is not returning any variables.') self.fail() if Feedback.check_numpy_array_allclose('The return value from your function power_iteration', self.ref.xc, results): points += 0.5 results_hidden = Feedback.call_user( self.st.power_iteration, self.ref.M_hidden, self.ref.x_hidden) if Feedback.check_numpy_array_allclose('The return value from your function power_iteration (hidden test case)', self.ref.xc_hidden, results_hidden): points += 0.5 Feedback.set_score(points)
def test_0(self): allowed_chars = {'(', ')', '+', '*', 'e', ' '}.union(alphabet) for char in self.st.reg_exp: if char not in allowed_chars: Feedback.add_feedback( f"Your regular expression contains the character '{char}', " "which is not permitted.") return processed_re = self.st.reg_exp.replace('e', '').replace('+', '|').replace( ' ', '') try: compiled_re = re.compile(processed_re) except re.error: Feedback.add_feedback("Your regular expression failed to compile.") return for str_length in range(0, MAX_LENGTH_TO_CHECK + 1): for char_list in itertools.product(alphabet, repeat=str_length): x = "".join(char_list) in_language = isInLanguage(x) match = compiled_re.fullmatch(x) if in_language and not match: Feedback.add_feedback( f"Your regular expression did not match the string '{x}', " "which is in the language.") return elif not in_language and match: Feedback.add_feedback( f"Your regular expression matched the string '{x}', " "which is not in the language.") return Feedback.set_score(1)
def test_0(self): points = 0 test_strings = ['', 'a', 'peak'] num_tests = len(test_strings) for s in test_strings: correct_val = self.ref.checkPalindrome(s) user_val = True if self.ref.function == 'is_palindrome_all': user_val = Feedback.call_user(self.st.is_palindrome_all, s) elif self.ref.function == 'is_palindrome_any': user_val = Feedback.call_user(self.st.is_palindrome_any, s) else: user_val = Feedback.call_user(self.st.is_palindrome_none, s) if Feedback.check_scalar(f"checkPalindrome({s})", correct_val, user_val): points += 1 if points < 3: points = 0 Feedback.set_score(points / num_tests)
def test_0(self): # Check type for str_length in range(0, MAX_LENGTH_TO_CHECK + 1): for char_list in itertools.product(alphabet, repeat=str_length): x = "".join(char_list) in_language = isInLanguage(x) accepted = self.st.fa.accepts_input(x) if in_language and not accepted: Feedback.add_feedback( f"Your DFA/NFA did not accept the string '{x}', " "which is in the language.") return elif not in_language and accepted: Feedback.add_feedback( f"Your DFA/NFA accepted the string '{x}', " "which is not in the language.") return Feedback.set_score(1)
def test_3(self): points = 0 test_strings = [ 'abc', 'c al', 'aabb', 'abcdefghi', 'palindrome', 'abbba', 'abcba', 'abca', 'refer', 'berkeley', 'abccba', 'abcdedcba' ] num_tests = 8 for s in test_strings: correct_val = self.ref.checkPalindrome(s) if isinstance(correct_val, bool): user_val = True if self.ref.function == 'is_palindrome_all': user_val = Feedback.call_user(self.st.is_palindrome_all, s) elif self.ref.function == 'is_palindrome_any': user_val = Feedback.call_user(self.st.is_palindrome_any, s) else: user_val = Feedback.call_user(self.st.is_palindrome_none, s) if Feedback.check_scalar(f"checkPalindrome({s})", correct_val, user_val): points += 1 if points <= 5: points = 0 Feedback.set_score(points / num_tests)
def run_code(result, run_req): # {{{ silence matplotlib font cache warnings import warnings warnings.filterwarnings( "ignore", message="Matplotlib is building the font cache.*") # }}} # {{{ compile code if getattr(run_req, "setup_code", None): try: setup_code = compile( run_req.setup_code, "[setup code]", 'exec') except Exception: package_exception(result, "setup_compile_error") return else: setup_code = None try: user_code = compile( run_req.user_code, "[user code]", 'exec') except Exception: package_exception(result, "user_compile_error") return if getattr(run_req, "test_code", None): try: test_code = compile( run_req.test_code, "[test code]", 'exec') except Exception: package_exception(result, "test_compile_error") return else: test_code = None # }}} if hasattr(run_req, "compile_only") and run_req.compile_only: result["result"] = "success" return # {{{ run code data_files = {} if hasattr(run_req, "data_files"): from base64 import b64decode for name, contents in run_req.data_files.items(): data_files[name] = b64decode(contents.encode()) generated_html = [] result["html"] = generated_html def output_html(s): generated_html.append(s) feedback = Feedback() maint_ctx = { "feedback": feedback, "user_code": user_code, "data_files": data_files, "output_html": output_html, "GradingComplete": GradingComplete, } if setup_code is not None: try: exec(setup_code, maint_ctx) except Exception: package_exception(result, "setup_error") return user_ctx = {} if hasattr(run_req, "names_for_user"): for name in run_req.names_for_user: if name not in maint_ctx: result["result"] = "setup_error" result["message"] = "Setup code did not define '%s'." % name user_ctx[name] = maint_ctx[name] from copy import deepcopy user_ctx = deepcopy(user_ctx) try: exec(user_code, user_ctx) except Exception: package_exception(result, "user_error") return # {{{ export plots if "matplotlib" in sys.modules: import matplotlib.pyplot as pt from io import BytesIO from base64 import b64encode format = "png" mime = "image/png" figures = [] for fignum in pt.get_fignums(): pt.figure(fignum) bio = BytesIO() try: pt.savefig(bio, format=format) except Exception: pass else: figures.append( (fignum, mime, b64encode(bio.getvalue()).decode())) result["figures"] = figures # }}} if hasattr(run_req, "names_from_user"): for name in run_req.names_from_user: if name not in user_ctx: feedback.add_feedback( "Required answer variable '%s' is not defined." % name) maint_ctx[name] = None else: maint_ctx[name] = user_ctx[name] if test_code is not None: try: exec(test_code, maint_ctx) except GradingComplete: pass except Exception: package_exception(result, "test_error") return result["points"] = feedback.points result["feedback"] = feedback.feedback_items # }}} result["result"] = "success"