def test_counter_chained(language: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "counter", language, tmp_path,
"chained.yaml", "solution")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"] * 3
assert len(updates.find_all("start-testcase")) == 4
def test_programmed_evaluator_lotto(lang: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "lotto", lang, tmp_path,
"one-programmed-python.tson", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 1
assert updates.find_status_enum() == ["correct"]
def test_io_function_exercise_haskell_io(language: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "echo-function", language, tmp_path,
"one.tson", "correct_io")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"]
def test_missing_key_types_detected(lang: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "objects", lang, tmp_path,
"missing_key_types.yaml", "solution")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 0
assert updates.find_status_enum() == ["internal error"]
def test_simple_programmed_eval_wrong(language: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "echo", language, tmp_path,
"one-programmed-wrong.tson", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["wrong"]
def test_programmed_evaluation(language: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "echo-function", language, tmp_path,
"programmed.tson", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"] * 4
assert len(updates.find_all("append-message")) == 4
def test_named_parameters_not_supported(language, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "echo-function", language, tmp_path,
"one-named.tson", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["internal error"]
def test_language_evaluator_exception(lang: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "division", lang, tmp_path, "plan.json",
"wrong")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["wrong"]
assert len(updates.find_all("append-message")) == 1
def test_special_numbers(language, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "", language, tmp_path)
plan = Plan()
bundle = create_bundle(conf, sys.stdout, plan)
type_map = bundle.lang_config.type_support_map()
# Create a list of basic types we want to test.
types = []
for t, n in itertools.product([
BasicNumericTypes.RATIONAL, AdvancedNumericTypes.DOUBLE_PRECISION,
AdvancedNumericTypes.SINGLE_PRECISION
], [
SpecialNumbers.NOT_A_NUMBER, SpecialNumbers.POS_INFINITY,
SpecialNumbers.NEG_INFINITY
]):
if type_map[t] == TypeSupport.SUPPORTED:
types.append(NumberType(type=t, data=n))
# Run the encode templates.
results = run_encoder(bundle, tmp_path, types)
assert len(results) == len(types)
for result, expected in zip(results, types):
actual = as_basic_type(parse_value(result))
expected = as_basic_type(expected)
assert expected.type == actual.type
py_expected = to_python_comparable(expected)
py_actual = to_python_comparable(actual)
assert py_expected == py_actual
def test_objects(language: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "objects", language, tmp_path,
"plan.tson", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"] * 2
assert len(updates.find_all("start-testcase")) == 3
def test_too_much_output(tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "echo", "python", tmp_path, "two.tson",
"output_limit", {"output_limit": 1000})
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
# 4 times: two times for the tests, one escalate and one judgement.
assert updates.find_status_enum() == ["output limit exceeded"] * 4
assert len(updates.find_all("close-test")) == 2
def test_heterogeneous_arguments_are_detected(lang: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "isbn", lang, tmp_path, "full.tson",
"solution")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 0
assert updates.find_status_enum() == ["internal error"]
def test_javascript_async(exercise: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, exercise, "javascript", tmp_path,
"one.tson", "correct-async")
workdir = Path(conf.resources).parent / "workdir"
if workdir.exists():
shutil.copytree(workdir, tmp_path, dirs_exist_ok=True)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"]
def test_escape(language, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "", language, tmp_path)
plan = Plan()
bundle = create_bundle(conf, sys.stdout, plan)
assert_serialisation(
bundle, tmp_path, StringType(type=BasicStringTypes.TEXT,
data='"hallo"'))
assert_serialisation(bundle, tmp_path,
StringType(type=AdvancedStringTypes.CHAR, data="'"))
def test_program_params(lang: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "sum", lang, tmp_path, "short.tson",
"correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == [
'correct', 'correct', 'correct', 'correct'
]
assert len(updates.find_all("start-testcase")) == 3
assert len(updates.find_all("start-test")) == 4
def test_io_function_file_exercise(language: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "echo-function-file", language,
tmp_path, "one.tson", "correct")
shutil.copytree(Path(conf.resources).parent / "workdir",
tmp_path,
dirs_exist_ok=True)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"]
def test_hide_expected_wrong(language: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "echo", language, tmp_path,
"one-hide-expected.tson", "wrong")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["wrong"]
assert len(
list(
filter(lambda x: not bool(x["expected"]),
updates.find_all("start-test")))) == 1
def evaluator_config(tmp_path: Path, pytestconfig, options=None) -> EvaluatorConfig:
if options is None:
options = dict()
conf = configuration(pytestconfig, "", "python", tmp_path)
plan = Plan()
bundle = create_bundle(conf, sys.stdout, plan)
return EvaluatorConfig(
bundle=bundle,
options=options,
context_dir=tmp_path
)
def test_javascript_exception_missing_message(tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig,
"js-exceptions",
"javascript",
tmp_path,
"plan.yaml",
"wrong-message",
backward_compatibility_plan=True)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["wrong"]
assert len(updates.find_all("append-message")) == 1
def test_batch_compilation_no_fallback(language: str, tmp_path: Path,
pytestconfig):
config_ = {"options": {"allow_fallback": False}}
conf = configuration(pytestconfig, "echo", language, tmp_path, "two.tson",
"run-error", config_)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 2
# One wrong status for every stderr + stdout
assert len(updates.find_status_enum()) >= 4
# There could be more wrongs: some languages might modify the exit code
assert all(s == "wrong" for s in updates.find_status_enum())
def test_crashing_assignment_with_before(lang: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "isbn", lang, tmp_path,
f"one-with-crashing-assignment-{lang}.tson",
"solution")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
# Only the assignment was started.
assert len(updates.find_all("start-testcase")) == 1
assert updates.find_status_enum() == ["wrong"]
# Assert the exception is included.
assert updates.find_next("start-test")["channel"] == "exception"
def test_assignment_and_use_in_expression_list(lang: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "isbn-list", lang, tmp_path,
"one-with-assignment.tson", "solution")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
# Assert that the empty context testcase is not shown, while the assignment
# and expression testcase are shown.
assert len(updates.find_all("start-testcase")) == 2
# Assert the only one test was executed.
assert updates.find_status_enum() == ["correct"]
assert len(updates.find_all("start-test")) == 1
def test_full_echo(lang: str, tmp_path: Path, pytestconfig):
config_ = {"options": {"parallel": True}}
conf = configuration(pytestconfig,
"echo",
lang,
tmp_path,
"full.tson",
"correct",
options=config_)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 50
assert updates.find_status_enum() == ["correct"] * 50
def test_basic_types(language, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "", language, tmp_path)
plan = Plan()
bundle = create_bundle(conf, sys.stdout, plan)
type_map = bundle.lang_config.type_support_map()
# Create a list of basic types we want to test.
types = []
if type_map[BasicNumericTypes.INTEGER] != TypeSupport.UNSUPPORTED:
types.append(NumberType(type=BasicNumericTypes.INTEGER, data=5))
if type_map[BasicNumericTypes.RATIONAL] != TypeSupport.UNSUPPORTED:
types.append(NumberType(type=BasicNumericTypes.RATIONAL, data=5.5))
if type_map[BasicStringTypes.TEXT] != TypeSupport.UNSUPPORTED:
types.append(StringType(type=BasicStringTypes.TEXT, data="hallo"))
if type_map[BasicBooleanTypes.BOOLEAN] != TypeSupport.UNSUPPORTED:
types.append(BooleanType(type=BasicBooleanTypes.BOOLEAN, data=True))
if type_map[BasicSequenceTypes.SEQUENCE] != TypeSupport.UNSUPPORTED:
types.append(
SequenceType(
type=BasicSequenceTypes.SEQUENCE,
data=[NumberType(type=BasicNumericTypes.INTEGER, data=20)]))
if type_map[BasicSequenceTypes.SET] != TypeSupport.UNSUPPORTED:
types.append(
SequenceType(
type=BasicSequenceTypes.SET,
data=[NumberType(type=BasicNumericTypes.INTEGER, data=20)]))
if type_map[BasicObjectTypes.MAP] != TypeSupport.UNSUPPORTED:
types.append(
ObjectType(type=BasicObjectTypes.MAP,
data=[
ObjectKeyValuePair(
key=StringType(type=BasicStringTypes.TEXT,
data="data"),
value=NumberType(type=BasicNumericTypes.INTEGER,
data=5))
]))
if type_map[BasicNothingTypes.NOTHING] != TypeSupport.UNSUPPORTED:
types.append(NothingType())
# Run the encode templates.
results = run_encoder(bundle, tmp_path, types)
assert len(results) == len(types)
for result, expected in zip(results, types):
actual = as_basic_type(parse_value(result))
assert expected.type == actual.type
py_expected = to_python_comparable(expected)
py_actual = to_python_comparable(actual)
assert py_expected == py_actual
def test_batch_compilation_fallback(language: str, tmp_path: Path,
pytestconfig, mocker):
lang_class = LANGUAGES[language]
class_instance = lang_class()
mocker.patch.object(lang_class,
'compilation',
wraps=class_instance.compilation)
conf = configuration(pytestconfig, "echo", language, tmp_path, "two.tson",
"comp-error")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 2
assert updates.find_status_enum() == ["compilation error"] * 2
assert class_instance.compilation.call_count == 3
def test_context_compilation(language: str, tmp_path: Path, pytestconfig,
mocker):
config_ = {"options": {"mode": "context"}}
# Mock the compilation callback to ensure we call it for every context.
lang_class = LANGUAGES[language]
class_instance = lang_class()
mocker.patch.object(lang_class,
'compilation',
wraps=class_instance.compilation)
conf = configuration(pytestconfig, "echo", language, tmp_path, "two.tson",
"correct", config_)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 2
assert updates.find_status_enum() == ["correct"] * 2
assert class_instance.compilation.call_count == 2
def test_io_function_display_multiline_exercise(language: str, tmp_path: Path,
pytestconfig):
conf = configuration(pytestconfig, "echo-function", language, tmp_path,
"one-display-multiline.tson", "correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"]
start_test = updates.find_all('start-test')
close_test = updates.find_all('close-test')
assert 1 == len(start_test)
assert 1 == len(close_test)
assert "return (String)" == start_test[0].get("channel", '')
expected, actual = start_test[0].get("expected", ''), close_test[0].get(
"generated", '')
quote = quotes[language]
assert expected[0] != quote and expected[-1] != quote
assert actual[0] != quote and actual[-1] != quote
def test_timeout(language_and_time: Tuple[str, int], tmp_path: Path,
pytestconfig):
config_ = {
"time_limit": language_and_time[1] # seconds
}
conf = configuration(pytestconfig, "echo", language_and_time[0], tmp_path,
"full.tson", "correct", config_)
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert len(updates.find_all("start-testcase")) == 50
status = updates.find_status_enum()
correct = [x for x in status if x == "correct"]
exceeded = [x for x in status if x == "time limit exceeded"]
wrong = [x for x in status if x == "wrong"]
# We should have at least one good result.
assert len(correct) >= 1
assert len(wrong) <= 2
assert len(exceeded) >= 1
# Once for every status, plus one escalation, plus one judgement-close
assert len(wrong + correct + exceeded) == 50 + 1 + 1
def test_advanced_types(language, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "", language, tmp_path)
plan = Plan()
bundle = create_bundle(conf, sys.stdout, plan)
type_map = bundle.lang_config.type_support_map()
# Create a list of basic types we want to test.
types = []
# The only advanced type undefined that should be tested
if type_map[AdvancedNothingTypes.UNDEFINED] == TypeSupport.SUPPORTED:
types.append(NothingType(type=AdvancedNothingTypes.UNDEFINED))
# Run the encode templates.
results = run_encoder(bundle, tmp_path, types)
assert len(results) == len(types)
for result, expected in zip(results, types):
actual = parse_value(result)
assert expected.type == actual.type
py_expected = to_python_comparable(expected)
py_actual = to_python_comparable(actual)
assert py_expected == py_actual
def test_language_evaluator_exception(lang: str, tmp_path: Path, pytestconfig):
conf = configuration(pytestconfig, "division", lang, tmp_path, "plan.json",
"correct")
result = execute_config(conf)
updates = assert_valid_output(result, pytestconfig)
assert updates.find_status_enum() == ["correct"]
