def setUp(self):
"""
On test start, initialise the points and set up the code feedback library
to provide feedback for this test.
"""
self.points = 0
Feedback.set_test(self)
def test_4(self):
if Feedback.check_numpy_array_allclose('right_side',
self.ref.right_side,
self.st.right_side):
Feedback.set_score(1)
else:
Feedback.set_score(0)
def setUpClass(self):
"""
On start, run the user code and generate answer tuples.
"""
Feedback.set_test(self)
base_dir = os.environ.get("MERGE_DIR")
job_dir = os.environ.get("JOB_DIR")
filenames_dir = os.environ.get("FILENAMES_DIR")
self.student_code_abs_path = join(base_dir, self.student_code_file)
# Load data so that we can use it in the test cases
filenames_dir = os.environ.get("FILENAMES_DIR")
with open(join(filenames_dir, 'data.json')) as f:
self.data = json.load(f)
ref_result, student_result, plot_value = execute_code(
join(filenames_dir, 'ans.py'),
join(base_dir, self.student_code_file), self.include_plt,
join(base_dir, 'output.txt'), self.iter_num)
answerTuple = namedtuple('answerTuple', ref_result.keys())
self.ref = answerTuple(**ref_result)
studentTuple = namedtuple('studentTuple', student_result.keys())
self.st = studentTuple(**student_result)
self.plt = plot_value
if self.include_plt:
self.display_plot()
def test_0(self):
if self.ref.lcs("crows eat yay",
"ows eat ") == self.st.lcs("crows eat yay",
"ows eat "):
Feedback.set_score(1)
else:
Feedback.set_score(0)
def test_2(self):
user_dict = Feedback.call_user(self.st.bif_dict, 7)
correct_dict = self.ref.bif_dict(7)
if self.check(user_dict, correct_dict):
Feedback.set_score(1)
else:
Feedback.set_score(0)
def test_0(self):
if Feedback.check_plot('plot',
self.ref.plot,
self.st.plot,
check_axes_scale='xy'):
Feedback.set_score(1)
else:
Feedback.set_score(0)
def test_0(self):
if Feedback.check_dataframe("df",
self.ref.df,
self.st.df,
display_input=True):
Feedback.set_score(1)
else:
Feedback.set_score(0)
def test_0(self):
num_distinct_elements = len({
Feedback.call_user(self.st.getFoolingSetElement, i)
for i in range(1, NUM_ELEMENTS_TO_CHECK + 1)
})
if num_distinct_elements == NUM_ELEMENTS_TO_CHECK:
Feedback.set_score(1)
else:
Feedback.set_score(0)
def test_3(self):
points = 0
num_tests = 10
test_values = np.random.choice(np.arange(2, 30), size=num_tests, replace=False)
for in_val in test_values:
correct_val = self.ref.fib(in_val)
user_val = Feedback.call_user(self.st.fib, in_val)
if Feedback.check_scalar(f"fib({in_val})", correct_val, user_val):
points += 1
Feedback.set_score(points / num_tests)
def check(self, user_dict, correct_dict):
if not isinstance(user_dict,
dict) or len(user_dict) != len(correct_dict):
Feedback.set_score(0)
return False
for key in correct_dict:
if key not in user_dict or correct_dict[key] != user_dict[key]:
Feedback.set_score(0)
return False
return True
def test_0(self):
sp19_enrollment = {"Murtaza": 10, "Lara": 20, "Mansi": 20, "Niki": 15, "Brendan": 10}
sp19_attendance = {"Murtaza": [10, 9, 10, 10, 10, 2, 9, 10, 9, 1],
"Lara": [19, 18, 16, 14, 12, 11, 10, 5, 5, 5],
"Mansi": [25, 22, 23, 24, 25, 22, 22, 20, 25, 25],
"Niki": [20, 15, 15, 12, 17, 17, 17, 16, 18, 20],
"Brendan": [5, 5, 4, 2, 3, 10, 11, 12, 4, 2]}
user_val = Feedback.call_user(self.st.TAs_with_at_least_half_discussions_full, sp19_enrollment, sp19_attendance)
if self.ref.TAs_with_at_least_half_discussions_full(sp19_enrollment, sp19_attendance) == user_val:
Feedback.set_score(1)
else:
Feedback.set_score(0)
def test_2(self):
enrollment = {"A": 8, "B": 12, "C": 23, "D": 17, "E": 5}
attendance = {"A": [8, 9, 8, 10, 10, 2, 6, 7, 3, 1],
"B": [19, 18, 16, 14, 12, 11, 10, 5, 5, 5],
"C": [25, 22, 23, 24, 25, 22, 22, 20, 25, 25],
"D": [20, 15, 15, 12, 17, 17, 16, 16, 18, 20],
"E": [5, 5, 4, 2, 3, 10, 11, 12, 4, 2]}
user_val = Feedback.call_user(self.st.TAs_with_at_least_half_discussions_full, enrollment, attendance)
if self.ref.TAs_with_at_least_half_discussions_full(enrollment, attendance) == user_val:
Feedback.set_score(1)
else:
Feedback.set_score(0)
def test_3(self):
points = 0
num_tests = 8
test_values = np.random.choice(np.arange(6, 20),
size=num_tests,
replace=False)
for value in test_values:
user_dict = Feedback.call_user(self.st.bif_dict, value)
correct_dict = self.ref.bif_dict(value)
if self.check(user_dict, correct_dict):
points += 1
Feedback.set_score(points / num_tests)
def test_0(self):
points = 0
a = np.random.rand(4)
b = np.random.rand(4)
user_val = Feedback.call_user(self.st.my_dot_product, a, b)
ref_val = Feedback.call_user(self.ref.my_dot_product, a, b)
if Feedback.check_scalar("my_dot_product return value", ref_val,
user_val):
points += 1
Feedback.set_score(points)
def test_1(self):
sequence = "uc berkeley also cal"
if self.ref.function == "most_common":
user_val = Feedback.call_user(self.st.most_common, sequence)
if self.ref.most_common(sequence) == user_val:
Feedback.set_score(1)
else:
Feedback.set_score(0)
elif self.ref.function == "least_common":
user_val = Feedback.call_user(self.st.least_common, sequence)
if self.ref.least_common(sequence) == user_val:
Feedback.set_score(1)
else:
Feedback.set_score(0)
def test_3(self):
sequence = "sphinxofquartz"
if self.ref.function == "most_common":
user_val = Feedback.call_user(self.st.most_common, sequence)
if self.ref.most_common(sequence) == user_val:
Feedback.set_score(1)
else:
Feedback.set_score(0)
elif self.ref.function == "least_common":
user_val = Feedback.call_user(self.st.least_common, sequence)
if self.ref.least_common(sequence) == user_val:
Feedback.set_score(1)
else:
Feedback.set_score(0)
def test_2(self):
sequence = [1, 2, 3, 4, 5, 6, 7, 8]
if self.ref.function == "most_common":
user_val = Feedback.call_user(self.st.most_common, sequence)
if self.ref.most_common(sequence) == user_val:
Feedback.set_score(1)
else:
Feedback.set_score(0)
elif self.ref.function == "least_common":
user_val = Feedback.call_user(self.st.least_common, sequence)
if self.ref.least_common(sequence) == user_val:
Feedback.set_score(1)
else:
Feedback.set_score(0)
def test_0(self):
points = 0
feedback.check_numpy_array_features('b',
self.ref.a,
self.st.b,
accuracy_critical=True)
suma = self.ref.beta * np.sum(self.ref.a)
sumb = np.sum(self.st.b)
if feedback.check_scalar('b', suma, sumb):
points += 1
feedback.set_score(points)
def test_4(self):
sequence = []
for i in range(20):
sequence += [random.randint(0, 20)]
if self.ref.function == "most_common":
user_val = Feedback.call_user(self.st.most_common, sequence)
if self.ref.most_common(sequence) == user_val:
Feedback.set_score(1)
else:
Feedback.set_score(0)
elif self.ref.function == "least_common":
user_val = Feedback.call_user(self.st.least_common, sequence)
if self.ref.least_common(sequence) == user_val:
Feedback.set_score(1)
else:
Feedback.set_score(0)
def test_0(self):
tests = 50
points = 0
xs = np.linspace(-1, 1, tests)
alphas = np.linspace(0.1, 0.001, tests)
np.random.shuffle(xs)
np.random.shuffle(alphas)
for i in range(tests):
x = xs[i]
alpha = alphas[i]
ref_xnew = self.ref.gradient_descent(x, alpha)
st_xnew = Feedback.call_user(self.st.gradient_descent, x, alpha)
if Feedback.check_scalar(f'x={x}, alpha={alpha}', ref_xnew, st_xnew, report_success=True, report_failure=True):
points += 1
Feedback.set_score(points / tests)
def setUpClass(cls):
"""
On start, run the user code and generate answer tuples.
"""
Feedback.set_test(cls)
filenames_dir = os.environ.get("FILENAMES_DIR")
base_dir = os.environ.get("MERGE_DIR")
cls.student_code_abs_path = join(base_dir, cls.student_code_file)
ref_result, student_result, plot_value = execute_code(
join(filenames_dir, 'ans.py'),
join(base_dir, cls.student_code_file), cls.include_plt,
join(base_dir, 'output.txt'), cls.iter_num)
answerTuple = namedtuple('answerTuple', ref_result.keys())
cls.ref = answerTuple(**ref_result)
studentTuple = namedtuple('studentTuple', student_result.keys())
cls.st = studentTuple(**student_result)
cls.plt = plot_value
if cls.include_plt:
cls.display_plot()
def test_0(self):
studentL = {
"".join(s)
for s in exploreCFG(self.st.cfg, MAX_LENGTH_TO_CHECK + 3)
}
# The "+ 3" is for wiggle room
# See exploreCFG documentation for an explanation of why this is necessary
studentL = {s for s in studentL if len(s) <= MAX_LENGTH_TO_CHECK}
correctL = generateLanguage(MAX_LENGTH_TO_CHECK)
falsePositives = studentL.difference(correctL)
if falsePositives:
Feedback.add_feedback(
"Your CFG generated the string "
f"'{min(falsePositives, key=lambda s: len(s))}', "
"which is not in the language.")
return
falseNegatives = correctL.difference(studentL)
if falseNegatives:
Feedback.add_feedback(
"Your CFG did not generate the string "
f"'{min(falseNegatives, key=lambda s: len(s))}', "
"which is in the language.")
return
Feedback.set_score(1)
def test_1(self):
for i in range(1, NUM_ELEMENTS_TO_CHECK + 1):
for j in range(i + 1, NUM_ELEMENTS_TO_CHECK + 1):
x = Feedback.call_user(self.st.getFoolingSetElement, i)
y = Feedback.call_user(self.st.getFoolingSetElement, j)
z = Feedback.call_user(self.st.getDistinguishingSuffix, i, j)
if not (isInLanguage(x + z) ^ isInLanguage(y + z)):
Feedback.add_feedback(
f"When i = {i} and j = {j}, the suffix\n\n"
f"z = '{z}'\n\n"
f"fails to distinguish the two fooling set elements\n\n"
f"x = '{x}'\n"
f"y = '{y}'")
Feedback.set_score(0)
return
Feedback.set_score(1)
def print_student_code(st_code='user_code.py',
ipynb_key='#grade',
as_feedback=True):
"""
Print the student's code, with syntax highlighting.
"""
with open(st_code, 'r', encoding='utf-8') as f:
filename, extension = splitext(st_code)
if extension == '.ipynb':
contents = extract_ipynb_contents(f, ipynb_key).strip()
lines = filter(lambda l: not l.strip().startswith(ipynb_key),
contents.split('\n'))
contents = '\n'.join(lines)
else:
contents = f.read().strip()
formatted = pygments.highlight(contents, PythonLexer(),
Terminal256Formatter(style='monokai'))
if as_feedback:
Feedback.add_feedback(formatted)
else:
return formatted
def test_3(self):
enrollment = {"A": 8, "B": 12, "C": 23, "D": 17, "E": 5, "Murtaza": 10, "Lara": 20, "Mansi": 20, "Niki": 15, "Brendan": 10, "F": 10, "G": 20, "H": 30, "I": 20, "J": 10}
attendance = {"A": [8, 9, 8, 10, 10, 2, 6, 7, 3, 1],
"B": [19, 18, 16, 14, 12, 11, 10, 5, 5, 5],
"C": [25, 22, 23, 24, 25, 22, 22, 20, 25, 25],
"D": [20, 15, 15, 12, 17, 17, 16, 16, 18, 20],
"E": [5, 5, 4, 2, 3, 10, 11, 12, 4, 2],
"Murtaza": [10, 9, 10, 10, 10, 2, 9, 10, 9, 1],
"Lara": [19, 18, 16, 14, 12, 11, 10, 5, 5, 5],
"Mansi": [25, 22, 23, 24, 25, 22, 22, 20, 25, 25],
"Niki": [20, 15, 15, 12, 17, 17, 17, 16, 18, 20],
"Brendan": [5, 5, 4, 2, 3, 10, 11, 12, 4, 2],
"F": [8, 9, 8, 10, 10, 2, 9, 10, 9, 1],
"G": [19, 18, 16, 14, 12, 11, 10, 5, 5, 5],
"H": [25, 22, 23, 24, 25, 22, 22, 20, 25, 25],
"I": [20, 15, 15, 12, 17, 17, 17, 16, 18, 20],
"J": [5, 5, 4, 2, 3, 10, 11, 12, 4, 2]}
user_val = Feedback.call_user(self.st.TAs_with_at_least_half_discussions_full, enrollment, attendance)
if self.ref.TAs_with_at_least_half_discussions_full(enrollment, attendance) == user_val:
Feedback.set_score(1)
else:
Feedback.set_score(0)
def test_0(self):
points = 0
# use the same input vector for both u and v to avoid ambiguity in the order of iteration
results = Feedback.call_user(
self.st.power_iteration, self.ref.M_test, self.ref.x_test)
if results is None:
Feedback.add_feedback(
'Your function is not returning any variables.')
self.fail()
if Feedback.check_numpy_array_allclose('The return value from your function power_iteration', self.ref.xc, results):
points += 0.5
results_hidden = Feedback.call_user(
self.st.power_iteration, self.ref.M_hidden, self.ref.x_hidden)
if Feedback.check_numpy_array_allclose('The return value from your function power_iteration (hidden test case)', self.ref.xc_hidden, results_hidden):
points += 0.5
Feedback.set_score(points)
def test_0(self):
allowed_chars = {'(', ')', '+', '*', 'e', ' '}.union(alphabet)
for char in self.st.reg_exp:
if char not in allowed_chars:
Feedback.add_feedback(
f"Your regular expression contains the character '{char}', "
"which is not permitted.")
return
processed_re = self.st.reg_exp.replace('e', '').replace('+',
'|').replace(
' ', '')
try:
compiled_re = re.compile(processed_re)
except re.error:
Feedback.add_feedback("Your regular expression failed to compile.")
return
for str_length in range(0, MAX_LENGTH_TO_CHECK + 1):
for char_list in itertools.product(alphabet, repeat=str_length):
x = "".join(char_list)
in_language = isInLanguage(x)
match = compiled_re.fullmatch(x)
if in_language and not match:
Feedback.add_feedback(
f"Your regular expression did not match the string '{x}', "
"which is in the language.")
return
elif not in_language and match:
Feedback.add_feedback(
f"Your regular expression matched the string '{x}', "
"which is not in the language.")
return
Feedback.set_score(1)
def test_0(self):
points = 0
test_strings = ['', 'a', 'peak']
num_tests = len(test_strings)
for s in test_strings:
correct_val = self.ref.checkPalindrome(s)
user_val = True
if self.ref.function == 'is_palindrome_all':
user_val = Feedback.call_user(self.st.is_palindrome_all, s)
elif self.ref.function == 'is_palindrome_any':
user_val = Feedback.call_user(self.st.is_palindrome_any, s)
else:
user_val = Feedback.call_user(self.st.is_palindrome_none, s)
if Feedback.check_scalar(f"checkPalindrome({s})", correct_val,
user_val):
points += 1
if points < 3:
points = 0
Feedback.set_score(points / num_tests)
def test_0(self):
# Check type
for str_length in range(0, MAX_LENGTH_TO_CHECK + 1):
for char_list in itertools.product(alphabet, repeat=str_length):
x = "".join(char_list)
in_language = isInLanguage(x)
accepted = self.st.fa.accepts_input(x)
if in_language and not accepted:
Feedback.add_feedback(
f"Your DFA/NFA did not accept the string '{x}', "
"which is in the language.")
return
elif not in_language and accepted:
Feedback.add_feedback(
f"Your DFA/NFA accepted the string '{x}', "
"which is not in the language.")
return
Feedback.set_score(1)
def test_3(self):
points = 0
test_strings = [
'abc', 'c al', 'aabb', 'abcdefghi', 'palindrome', 'abbba', 'abcba',
'abca', 'refer', 'berkeley', 'abccba', 'abcdedcba'
]
num_tests = 8
for s in test_strings:
correct_val = self.ref.checkPalindrome(s)
if isinstance(correct_val, bool):
user_val = True
if self.ref.function == 'is_palindrome_all':
user_val = Feedback.call_user(self.st.is_palindrome_all, s)
elif self.ref.function == 'is_palindrome_any':
user_val = Feedback.call_user(self.st.is_palindrome_any, s)
else:
user_val = Feedback.call_user(self.st.is_palindrome_none,
s)
if Feedback.check_scalar(f"checkPalindrome({s})", correct_val,
user_val):
points += 1
if points <= 5:
points = 0
Feedback.set_score(points / num_tests)
def run_code(result, run_req):
# {{{ silence matplotlib font cache warnings
import warnings
warnings.filterwarnings(
"ignore", message="Matplotlib is building the font cache.*")
# }}}
# {{{ compile code
if getattr(run_req, "setup_code", None):
try:
setup_code = compile(
run_req.setup_code, "[setup code]", 'exec')
except Exception:
package_exception(result, "setup_compile_error")
return
else:
setup_code = None
try:
user_code = compile(
run_req.user_code, "[user code]", 'exec')
except Exception:
package_exception(result, "user_compile_error")
return
if getattr(run_req, "test_code", None):
try:
test_code = compile(
run_req.test_code, "[test code]", 'exec')
except Exception:
package_exception(result, "test_compile_error")
return
else:
test_code = None
# }}}
if hasattr(run_req, "compile_only") and run_req.compile_only:
result["result"] = "success"
return
# {{{ run code
data_files = {}
if hasattr(run_req, "data_files"):
from base64 import b64decode
for name, contents in run_req.data_files.items():
data_files[name] = b64decode(contents.encode())
generated_html = []
result["html"] = generated_html
def output_html(s):
generated_html.append(s)
feedback = Feedback()
maint_ctx = {
"feedback": feedback,
"user_code": user_code,
"data_files": data_files,
"output_html": output_html,
"GradingComplete": GradingComplete,
}
if setup_code is not None:
try:
exec(setup_code, maint_ctx)
except Exception:
package_exception(result, "setup_error")
return
user_ctx = {}
if hasattr(run_req, "names_for_user"):
for name in run_req.names_for_user:
if name not in maint_ctx:
result["result"] = "setup_error"
result["message"] = "Setup code did not define '%s'." % name
user_ctx[name] = maint_ctx[name]
from copy import deepcopy
user_ctx = deepcopy(user_ctx)
try:
exec(user_code, user_ctx)
except Exception:
package_exception(result, "user_error")
return
# {{{ export plots
if "matplotlib" in sys.modules:
import matplotlib.pyplot as pt
from io import BytesIO
from base64 import b64encode
format = "png"
mime = "image/png"
figures = []
for fignum in pt.get_fignums():
pt.figure(fignum)
bio = BytesIO()
try:
pt.savefig(bio, format=format)
except Exception:
pass
else:
figures.append(
(fignum, mime, b64encode(bio.getvalue()).decode()))
result["figures"] = figures
# }}}
if hasattr(run_req, "names_from_user"):
for name in run_req.names_from_user:
if name not in user_ctx:
feedback.add_feedback(
"Required answer variable '%s' is not defined."
% name)
maint_ctx[name] = None
else:
maint_ctx[name] = user_ctx[name]
if test_code is not None:
try:
exec(test_code, maint_ctx)
except GradingComplete:
pass
except Exception:
package_exception(result, "test_error")
return
result["points"] = feedback.points
result["feedback"] = feedback.feedback_items
# }}}
result["result"] = "success"
