def test_invalid_sandbox(self):
results = grader.test_code("",
"",
sandbox_cmd="invalid_command_123 4 5 6")
assert not results["success"], results
self.assertIn("Invalid command: invalid_command_123 4 5 6",
results["reason"])
def test_solution_syntaxerror_does_not_raise(self):
results = grader.test_code(dedent("""\
from grader import *
@test
def empty(m): pass
"""), "invalid_syntax")
assert results["success"], results
def test_grader_invalid(self):
tester = dedent("""\
from grader import *
from grader.extensions import ast
template = '''
b = 0
if not ____:
i = 5
...
j = 6
assert j * i * b == 30
...
'''
ast.template_test(template_code=template)
""")
solution_invalid = dedent("""\
b = 0
if not False:
b = 1
i = 5
j = 6
assert j * i * b == 30
""")
results = grader.test_code(tester, solution_invalid)["results"][0]
assert not results["success"], results
self.assertIn("Program code does not match template.",
results["error_message"])
def test_grader_invalid(self):
tester = dedent("""\
from grader import *
from grader.extensions import ast
template = '''
b = 0
if not ____:
i = 5
...
j = 6
assert j * i * b == 30
...
'''
ast.template_test(template_code=template)
""")
solution_invalid = dedent("""\
b = 0
if not False:
b = 1
i = 5
j = 6
assert j * i * b == 30
""")
results = grader.test_code(tester, solution_invalid)["results"][0]
assert not results["success"], results
self.assertIn("Program code does not match template.", results["error_message"])
def test_grader_valid(self):
tester = dedent("""\
from grader import *
from grader.extensions import ast
template = '''
b = 0
if not ____:
i = 5
...
j = 6
assert j * i * b == 30
...
'''
ast.template_test(template_code=template)
""")
solution_valid = dedent("""\
b = 0
if not False:
i = 5
b = 1
j = 6
assert j * i * b == 30
""")
results = grader.test_code(tester, solution_valid)["results"][0]
assert results["success"], results
def test_grader_valid(self):
tester = dedent("""\
from grader import *
from grader.extensions import ast
template = '''
b = 0
if not ____:
i = 5
...
j = 6
assert j * i * b == 30
...
'''
ast.template_test(template_code=template)
""")
solution_valid = dedent("""\
b = 0
if not False:
i = 5
b = 1
j = 6
assert j * i * b == 30
""")
results = grader.test_code(tester, solution_valid)["results"][0]
assert results["success"], results
def test_docker_sandbox(self):
results = grader.test_code(dedent("""\
from grader import *
@test
def empty(m): pass
"""), "", sandbox_cmd="docker")
assert results["success"], results
def test_solution_syntaxerror_does_not_raise(self):
results = grader.test_code(
dedent("""\
from grader import *
@test
def empty(m): pass
"""), "invalid_syntax")
assert results["success"], results
def test_docker_sandbox(self):
results = grader.test_code(dedent("""\
from grader import *
@test
def empty(m): pass
"""),
"",
sandbox_cmd="docker")
assert results["success"], results
def test_solution2():
data = request.json
app.logger.info(data)
answer = grader.test_code(data['tester_code'],
data['solution_code'],
data.get('assets', []),
sandbox_cmd='docker')
app.logger.debug(answer)
return jsonify(answer)
def test_solution2():
data = request.json
app.logger.info(data)
answer = grader.test_code(
data['tester_code'],
data['solution_code'],
data.get('assets', []),
sandbox_cmd='docker'
)
app.logger.debug(answer)
return jsonify(answer)
def test_assets_available_while_running(self):
# TODO: check sys.path size
asset_code = {"filename": "file.txt", "contents": "secret"}
results = grader.test_code(
dedent("""\
from grader import *
@test
def empty(m):
assert m.stdout.read().strip() == 'secret'
"""), "print(open('file.txt').read())", [asset_code])
assert results["success"], results
assert results["results"][0]["success"], results["results"][0]
def tests_assets_are_importable_at_test_registration(self):
# TODO: check sys.path size
asset_code = {"filename": "asset_file.py", "contents": "exists = True"}
results = grader.test_code(
dedent("""\
from grader import *
import asset_file
@test
def empty(m):
assert asset_file.exists
"""), "", [asset_code])
assert results["success"], results
assert results["results"][0]["success"], results
def test_assets_available_while_running(self):
# TODO: check sys.path size
asset_code = {
"filename": "file.txt",
"contents": "secret"
}
results = grader.test_code(dedent("""\
from grader import *
@test
def empty(m):
assert m.stdout.read().strip() == 'secret'
"""), "print(open('file.txt').read())", [asset_code])
assert results["success"], results
assert results["results"][0]["success"], results["results"][0]
def tests_assets_are_importable_at_test_registration(self):
# TODO: check sys.path size
asset_code = {
"filename": "asset_file.py",
"contents": "exists = True"
}
results = grader.test_code(dedent("""\
from grader import *
import asset_file
@test
def empty(m):
assert asset_file.exists
"""), "", [asset_code])
assert results["success"], results
assert results["results"][0]["success"], results
def test_tester_no_tests(self):
results = grader.test_code("", "")
assert not results["success"]
self.assertEquals(results["reason"], "No tests found in tester",
results)
def test_tester_invalid_syntax(self):
results = grader.test_code("some invalid code", "")
assert not results["success"]
self.assertEquals(results["reason"], "Load tests failure", results)
self.assertIn("SyntaxError:", results["extra_info"]["error_message"])
def run_code(tester_code, solution_code):
return grader.test_code(tester_code, solution_code)["results"]
def test_invalid_sandbox(self):
results = grader.test_code("", "", sandbox_cmd="invalid_command_123 4 5 6")
assert not results["success"], results
self.assertIn("Invalid command: invalid_command_123 4 5 6", results["reason"])
def test_tester_no_tests(self):
results = grader.test_code("", "")
assert not results["success"]
self.assertEquals(results["reason"], "No tests found in tester", results)
def run_code(tester_code, solution_code):
return grader.test_code(tester_code, solution_code)["results"]
def test_tester_invalid_syntax(self):
results = grader.test_code("some invalid code", "")
assert not results["success"]
self.assertEquals(results["reason"], "Load tests failure", results)
self.assertIn("SyntaxError:", results["extra_info"]["error_message"])
