def test_objects(language: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "objects", language, tmp_path,
"plan.tson", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"] * 2
assert len(updates.find_all("start-testcase")) == 3
def test_simple_programmed_eval_wrong(language: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "echo", language, tmp_path,
"one-programmed-wrong.tson", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["wrong"]
def test_counter_chained(language: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "counter", language, tmp_path,
"chained.yaml", "solution")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"] * 3
assert len(updates.find_all("start-testcase")) == 4
def test_programmed_evaluation(language: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "echo-function", language, tmp_path,
"programmed.tson", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"] * 4
assert len(updates.find_all("append-message")) == 4
def test_io_function_exercise_haskell_io(language: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "echo-function", language, tmp_path,
"one.tson", "correct_io")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"]
def test_missing_key_types_detected(lang: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "objects", lang, tmp_path,
"missing_key_types.yaml", "solution")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 0
assert updates.find_status_enum() == ["internal error"]
def test_language_evaluator_exception(lang: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "division", lang, tmp_path, "plan.json",
"wrong")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["wrong"]
assert len(updates.find_all("append-message")) == 1
def test_programmed_evaluator_lotto(lang: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "lotto", lang, tmp_path,
"one-programmed-python.tson", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 1
assert updates.find_status_enum() == ["correct"]
def test_named_parameters_not_supported(language, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "echo-function", language, tmp_path,
"one-named.tson", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["internal error"]
def test_heterogeneous_arguments_are_detected(lang: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "isbn", lang, tmp_path, "full.tson",
"solution")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 0
assert updates.find_status_enum() == ["internal error"]
def test_too_much_output(tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "echo", "python", tmp_path, "two.tson",
"output_limit", {"output_limit": 1000})
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
# 4 times: two times for the tests, one escalate and one judgement.
assert updates.find_status_enum() == ["output limit exceeded"] * 4
assert len(updates.find_all("close-test")) == 2
def test_javascript_async(exercise: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, exercise, "javascript", tmp_path,
"one.tson", "correct-async")
workdir = Path(conf.resources).parent / "workdir"
if workdir.exists():
shutil.copytree(workdir, tmp_path, dirs_exist_ok=True)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"]
def test_program_params(lang: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "sum", lang, tmp_path, "short.tson",
"correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == [
'correct', 'correct', 'correct', 'correct'
]
assert len(updates.find_all("start-testcase")) == 3
assert len(updates.find_all("start-test")) == 4
def test_io_function_file_exercise(language: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "echo-function-file", language,
tmp_path, "one.tson", "correct")
shutil.copytree(Path(conf.resources).parent / "workdir",
tmp_path,
dirs_exist_ok=True)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"]
def test_hide_expected_wrong(language: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "echo", language, tmp_path,
"one-hide-expected.tson", "wrong")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["wrong"]
assert len(
list(
filter(lambda x: not bool(x["expected"]),
updates.find_all("start-test")))) == 1
def test_assignment_and_use_in_expression_list(lang: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "isbn-list", lang, tmp_path,
"one-with-assignment.tson", "solution")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
# Assert that the empty context testcase is not shown, while the assignment
# and expression testcase are shown.
assert len(updates.find_all("start-testcase")) == 2
# Assert the only one test was executed.
assert updates.find_status_enum() == ["correct"]
assert len(updates.find_all("start-test")) == 1
def test_batch_compilation_no_fallback(language: str, tmp_path: Path,
pytestconfig):
config_ = {"options": {"allow_fallback": False}}
conf = configuration(pytestconfig, "echo", language, tmp_path, "two.tson",
"run-error", config_)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 2
# One wrong status for every stderr + stdout
assert len(updates.find_status_enum()) >= 4
# There could be more wrongs: some languages might modify the exit code
assert all(s == "wrong" for s in updates.find_status_enum())
def test_crashing_assignment_with_before(lang: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "isbn", lang, tmp_path,
f"one-with-crashing-assignment-{lang}.tson",
"solution")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
# Only the assignment was started.
assert len(updates.find_all("start-testcase")) == 1
assert updates.find_status_enum() == ["wrong"]
# Assert the exception is included.
assert updates.find_next("start-test")["channel"] == "exception"
def test_javascript_exception_missing_message(tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig,
"js-exceptions",
"javascript",
tmp_path,
"plan.yaml",
"wrong-message",
backward_compatibility_plan=True)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["wrong"]
assert len(updates.find_all("append-message")) == 1
def test_full_echo(lang: str, tmp_path: Path, pytestconfig):
config_ = {"options": {"parallel": True}}
conf = configuration(pytestconfig,
"echo",
lang,
tmp_path,
"full.tson",
"correct",
options=config_)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 50
assert updates.find_status_enum() == ["correct"] * 50
def test_batch_compilation_fallback(language: str, tmp_path: Path,
pytestconfig, mocker):
lang_class = LANGUAGES[language]
class_instance = lang_class()
mocker.patch.object(lang_class,
'compilation',
wraps=class_instance.compilation)
conf = configuration(pytestconfig, "echo", language, tmp_path, "two.tson",
"comp-error")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 2
assert updates.find_status_enum() == ["compilation error"] * 2
assert class_instance.compilation.call_count == 3
def test_context_compilation(language: str, tmp_path: Path, pytestconfig,
mocker):
config_ = {"options": {"mode": "context"}}
# Mock the compilation callback to ensure we call it for every context.
lang_class = LANGUAGES[language]
class_instance = lang_class()
mocker.patch.object(lang_class,
'compilation',
wraps=class_instance.compilation)
conf = configuration(pytestconfig, "echo", language, tmp_path, "two.tson",
"correct", config_)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 2
assert updates.find_status_enum() == ["correct"] * 2
assert class_instance.compilation.call_count == 2
def test_io_function_display_multiline_exercise(language: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "echo-function", language, tmp_path,
"one-display-multiline.tson", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"]
start_test = updates.find_all('start-test')
close_test = updates.find_all('close-test')
assert 1 == len(start_test)
assert 1 == len(close_test)
assert "return (String)" == start_test[0].get("channel", '')
expected, actual = start_test[0].get("expected", ''), close_test[0].get(
"generated", '')
quote = quotes[language]
assert expected[0] != quote and expected[-1] != quote
assert actual[0] != quote and actual[-1] != quote
def test_timeout(language_and_time: Tuple[str, int], tmp_path: Path,
pytestconfig):
config_ = {
"time_limit": language_and_time[1] # seconds
}
conf = configuration(pytestconfig, "echo", language_and_time[0], tmp_path,
"full.tson", "correct", config_)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 50
status = updates.find_status_enum()
correct = [x for x in status if x == "correct"]
exceeded = [x for x in status if x == "time limit exceeded"]
wrong = [x for x in status if x == "wrong"]
# We should have at least one good result.
assert len(correct) >= 1
assert len(wrong) <= 2
assert len(exceeded) >= 1
# Once for every status, plus one escalation, plus one judgement-close
assert len(wrong + correct + exceeded) == 50 + 1 + 1
def test_pylint(tmp_path: Path, config, pytestconfig):
conf = configuration(pytestconfig, "counter", "python", tmp_path,
"plan.yaml", "solution-pylint", config)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("annotate-code")) > 0
def test_language_evaluator_exception(lang: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "division", lang, tmp_path, "plan.json",
"correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"]
def test_global_variable_yaml(language: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "global", language, tmp_path,
"plan.yaml", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"]
def test_shellcheck(tmp_path: Path, config, pytestconfig):
conf = configuration(pytestconfig, "echo", "bash", tmp_path, "one.tson",
"wrong", config)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("annotate-code")) > 0
def test_objects_error(language: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "objects", language, tmp_path,
"plan.tson", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["internal error"]
def test_io_exercise_wrong(language: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "echo", language, tmp_path, "one.tson",
"wrong")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["wrong"]
