def test_exercise(sct, student_data, solution_data, success_msg=None):
"""
"""
assert isinstance(sct, list)
assert isinstance(student_data, dict)
assert isinstance(solution_data, dict)
rep = Reporter()
for single_sct in sct:
state = State(
student_data=student_data,
solution_data=solution_data,
sct_range=single_sct.get("range"),
reporter=rep,
)
SCT_CTX["Ex"].root_state = state
try:
exec("\n".join(single_sct.get("sct", [])), SCT_CTX)
except TestFail as tf:
return tf.payload
if success_msg and isinstance(success_msg, str):
rep.success_msg = success_msg
return rep.build_final_payload()
def test_multiple_state_init(conn):
state1 = State(
student_code="SELECT * FROM company",
solution_code="SELECT * FROM company",
pre_exercise_code="",
student_result={
"id": [1],
"name": ["greg"]
},
solution_result={
"id": [0],
"name": ["greg"]
},
student_conn=conn,
solution_conn=None,
reporter=Reporter(),
)
state2 = State(
student_code="SELECT * FROM company",
solution_code="SELECT * FROM company",
pre_exercise_code="",
student_result={
"id": [1],
"name": ["greg"]
},
solution_result={
"id": [0],
"name": ["greg"]
},
student_conn=conn,
solution_conn=None,
reporter=Reporter(),
)
def test_test_runner_proxy():
r = Reporter()
assert len(r.tests) == 0
assert not r.has_failed
assert not r.failures
result, feedback = r.do_test(Success("msg"))
assert len(r.tests) == 1
assert not r.has_failed
assert not r.failures
assert isinstance(result, bool)
assert feedback is None
failing_test = Fail("msg")
results = r.do_tests([failing_test, Success("msg")])
assert len(r.tests) == 3
assert r.has_failed
assert len(r.failures) == 1
assert failing_test in r.failures
assert isinstance(results, list)
assert r.tests == r.runner.tests
assert r.runner.has_failed
assert r.failures == r.runner.failures
def test_highlighting_path_no_position():
r = Reporter()
f = Feedback(FeedbackComponent("msg"), path=Path("test.py"))
payload = r.build_failed_payload(f)
expected_payload = {"correct": False, "message": "msg", "path": "test.py"}
assert payload == expected_payload
def test_reporter_line_info_1_based_cols():
rep = Reporter()
rep.failed_test = True
rep.feedback = Feedback("MESSAGE", ast.parse("\nSELECT x FROM y"))
payload = rep.build_payload()
pos = [2, 1, 2, 15]
pos_names = ['line_start', 'column_start', 'line_end', 'column_end']
for ii, k in enumerate(pos_names):
assert payload[k] == pos[ii]
def test_highlighting_offset_proxy():
r = Reporter(Reporter(), highlight_offset=highlight_range_2)
f = FeedbackTest("msg")
f.highlight = highlight_range_1
payload = r.build_failed_payload(f)
expected_payload = {"correct": False, "message": "msg", **highlight_combined}
assert payload == expected_payload
def test_highlighting_offset(offset, highlight, payload_highlight_info):
r = Reporter(highlight_offset=offset)
f = FeedbackTest("msg")
f.highlight = highlight
payload = r.build_failed_payload(f)
expected_payload = {"correct": False, "message": "msg", **payload_highlight_info}
assert payload == expected_payload
def test_highlighting_offset(offset, highlight, payload_highlight_info):
r = Reporter()
f = Feedback(
FeedbackComponent("msg"),
highlight=Highlight(highlight),
highlight_offset=offset,
)
payload = r.build_failed_payload(f)
expected_payload = {"correct": False, "message": "msg", **payload_highlight_info}
assert payload == expected_payload
def test_highlighting_offset_proxy():
r = Reporter()
f = Feedback(
FeedbackComponent("msg"),
highlight=Highlight(highlight_range_1),
highlight_offset=highlight_range_2,
)
payload = r.build_failed_payload(f)
expected_payload = {"correct": False, "message": "msg", **highlight_combined}
assert payload == expected_payload
def test_get_dispatcher_oracle():
# Given
conn2 = MagicMock()
dialect = MagicMock()
conn2.dialect = dialect
dialect.name = "oracle"
state = State(
student_code="SELECT * FROM company",
solution_code="SELECT * FROM company",
pre_exercise_code="",
student_result={
"id": [1],
"name": ["greg"]
},
solution_result={
"id": [1],
"name": ["greg"]
},
student_conn=conn2,
solution_conn=None,
reporter=Reporter(),
)
# When
output = state.get_dispatcher()
# Then
assert output.ast_mod.grammar.__spec__.name == 'antlr_plsql.antlr_py'
def test_exercise(sct,
student_code,
student_result,
student_conn,
solution_code,
solution_result,
solution_conn,
pre_exercise_code,
ex_type,
error,
debug = False # currently unused
):
"""
"""
# TODO: put reporter on state
state = State(
student_code = student_code,
solution_code = solution_code,
pre_exercise_code = pre_exercise_code,
student_conn = student_conn,
solution_conn = solution_conn,
student_result = student_result,
solution_result = solution_result,
reporter = Reporter(error))
SCT_CTX['Ex'].root_state = state
try:
exec(sct, SCT_CTX)
except TestFail: pass
return(state.reporter.build_payload())
def test_exercise(sct,
student_code,
student_result,
student_conn,
solution_code,
solution_result,
solution_conn,
pre_exercise_code,
ex_type,
error,
force_diagnose = False,
debug = False # currently unused
):
"""
"""
state = State(
student_code = student_code,
solution_code = solution_code,
pre_exercise_code = pre_exercise_code,
student_conn = student_conn,
solution_conn = solution_conn,
student_result = student_result,
solution_result = solution_result,
reporter = Reporter(error),
force_diagnose = force_diagnose)
SCT_CTX['Ex'].root_state = state
try:
exec(sct, SCT_CTX)
except TestFail as tf:
return tf.payload
return state.reporter.build_final_payload()
def test_state_dec_instant_eval():
state = State("student_code", "", "", None, None, {}, {}, Reporter())
@state_dec
def stu_code(state, x='x'):
return state.student_code + x
assert stu_code(state) == "student_codex"
def state():
return State(student_code="some code\x1b[39;49m",
solution_code="some code",
pre_exercise_code="",
student_conn=replwrap.bash(),
solution_conn=None,
student_result="stdout stuff",
solution_result=None,
reporter=Reporter())
def state():
return State(student_code="echo a $b ${c}",
solution_code="echo a $b ${c} unique",
pre_exercise_code="",
student_conn=None,
solution_conn=None,
student_result="",
solution_result="",
reporter=Reporter())
def test_exercise(
sct,
student_code,
solution_code,
pre_exercise_code,
student_process,
solution_process,
raw_student_output,
ex_type,
error,
force_diagnose=False,
):
"""
Point of interaction with the Python backend.
Args:
sct (str): The solution corectness test as a string of code.
student_code (str): The code which is entered by the student.
solution_code (str): The code which is in the solution.
pre_exercise_code (str): The code which is executed pre exercise.
student_process (Process): Process in which the student code was executed.
solution_process (Process): Process in which the solution code was executed.
raw_student_output (str): The output which is given by executing the student's program.
ex_type (str): The type of the exercise.
error (tuple): A tuple with some information on possible errors.
Returns:
dict: Returns dict with correct - whether the SCT passed, message - the feedback message and
tags - the tags belonging to the SCT execution.
"""
try:
state = State(
student_code=check_str(student_code),
solution_code=check_str(solution_code),
pre_exercise_code=check_str(pre_exercise_code),
student_process=check_process(student_process),
solution_process=check_process(solution_process),
raw_student_output=check_str(raw_student_output),
force_diagnose=force_diagnose,
reporter=Reporter([error] if error else [])
)
State.root_state = state
tree, sct_cntxt = prep_context()
# Actually execute SCTs
exec(sct, sct_cntxt)
# Run remaining nodes on tree (v1 only)
if tree:
for test in tree.crnt_node:
test(state)
except TestFail as e:
return e.payload
return state.reporter.build_final_payload()
def prepare_state(sol_result, stu_result):
return State(
student_code = "",
solution_code = "",
reporter = Reporter(),
# args below should be ignored
pre_exercise_code = "",
student_result = stu_result, solution_result = sol_result,
student_conn = None, solution_conn = None)
def test_highlighting_path():
r = Reporter()
f = Feedback(
FeedbackComponent("msg"),
highlight=Highlight(highlight_range_1),
path=Path("test.py"),
)
payload = r.build_failed_payload(f)
expected_payload = {
"correct": False,
"message": "msg",
"path": "test.py",
**highlight_payload_1,
}
assert payload == expected_payload
def test_initial_state():
State(student_code={'script.py': '1'},
solution_code={'script.py': '1'},
reporter=Reporter(),
pre_exercise_code="",
student_result="",
solution_result="",
student_conn=None,
solution_conn=None,
ast_dispatcher=Dispatcher(DUMMY_NODES, ParseHey()))
def prepare_state(student_code):
return State(
student_code = "",
reporter = Reporter(),
# args below should be ignored
solution_code = "NA", pre_exercise_code = "NA",
solution_ast = "NA", student_ast = "NA",
student_result = [], solution_result = [],
student_conn = None, solution_conn = None,
ast_dispatcher = "NA")
def prepare_state(student_code=''):
s = State(student_code=student_code,
solution_code="",
reporter=Reporter(),
pre_exercise_code="",
student_result=None,
solution_result=None,
student_conn=None,
solution_conn=None)
s.tc_msg = "Fail"
return (s)
def test_pec_parsing_error():
with pytest.raises(InstructorError):
State(
student_code="parses",
solution_code="parses",
pre_exercise_code="does not parse",
student_process=None,
solution_process=None,
reporter=Reporter(),
raw_student_output=None,
)
def state():
return State(
student_code="",
solution_code="",
reporter=Reporter(),
# args below should be ignored
pre_exercise_code="NA",
student_result={'a': [1]},
solution_result={'b': [2]},
student_conn=None,
solution_conn=None)
def test_multiple_state_init(conn):
state1 = State(
student_code = "SELECT * FROM company",
solution_code = "SELECT * FROM company",
pre_exercise_code = "",
student_result = {'id': [1], 'name': ['greg']},
solution_result = {'id': [0], 'name': ['greg']},
student_conn = conn,
solution_conn = None,
reporter= Reporter())
state2 = State(
student_code = "SELECT * FROM company",
solution_code = "SELECT * FROM company",
pre_exercise_code = "",
student_result = {'id': [1], 'name': ['greg']},
solution_result = {'id': [0], 'name': ['greg']},
student_conn = conn,
solution_conn = None,
reporter= Reporter())
def state():
return State(
# only Reporter and Dispatcher are used
student_code="",
solution_code="",
reporter=Reporter(),
pre_exercise_code="",
student_result="",
solution_result="",
student_conn=None,
solution_conn=None,
)
def state():
return State(
student_code="",
solution_code="",
reporter=Reporter(),
# args below should be ignored
pre_exercise_code="NA",
student_result="",
solution_result="",
student_conn=None,
solution_conn=None,
ast_dispatcher=Dispatcher(DUMMY_NODES, ParseHey()))
def state():
return State(
student_code='student_code',
solution_code='solution_code',
reporter=Reporter(),
# args below should be ignored
pre_exercise_code="NA",
student_result='stu_result',
solution_result=None,
student_conn=replwrap.bash(),
solution_conn=None
)
def prepare_state(sol_result, stu_result):
conn = Connection('postgresql')
return State(
student_code="",
solution_code="",
reporter=Reporter(),
# args below should be ignored
pre_exercise_code="NA",
student_result=stu_result,
solution_result=sol_result,
student_conn=conn,
solution_conn=None)
def prepare_state(solution_code, student_code, dialect='postgresql'):
dispatcher = Dispatcher.from_module(PARSER_MODULES[dialect])
return State(
student_code=student_code,
solution_code=solution_code,
reporter=Reporter(),
# args below should be ignored
pre_exercise_code="NA",
student_result=[],
solution_result=[],
student_conn=None,
solution_conn=None,
ast_dispatcher=dispatcher)
def state():
return State(
# only Reporter and Dispatcher are used
student_code="",
solution_code="",
reporter=Reporter(),
pre_exercise_code="",
student_result="",
solution_result="",
student_conn=None,
solution_conn=None,
ast_dispatcher=Dispatcher(ast.AST, DUMMY_NODES, ParseHey()),
)
