def test_json_conversion_no_sandbox(self):
'''test json conversion with no sandbox'''
pgm1 = Program("print('hello world')", 'python3', FireJail())
jsobj = pgm1.to_json_object()
del jsobj["sandbox"]
pgm2 = Program.from_json_object(json.loads(json.dumps(jsobj)))
assert pgm2.sandbox == FireJail()
def test_c_compilation_error(self):
'''Test compilation error in C'''
code = '''
#include<stdio.h>
int main(){
printf("hello world")
return 0;
}
'''
pgm = Program(code, 'C', self.sandbox)
pgm.compile()
def test_no_workspace(self, tmp_path: Path):
'''Test if workspace does not exist'''
with Program("print('hello world')", 'python3', self.sandbox) as pgm:
Settings.load_added_settings(
f'workspace: {str(tmp_path.joinpath("fake"))}')
pgm.compile()
Settings.load_added_settings('workspace: playground')
def test_context_manager_usage(self):
'''Test usage of Program class as context manager'''
with Program("print('hello world')", 'python3', FireJail()) as pgm:
pgm.compile()
testcase = TestCase()
pgm.execute(testcase)
assert testcase.real_output.strip() == "hello world"
def test_json_conversion(self):
'''test json conversion'''
pgm1 = Program("print('hello world')", 'python3', FireJail())
testcases = [TestCase(), TestCase("h", "h")]
ev1 = Evaluation(pgm1, testcases)
jsobj = ev1.to_json_object()
ev2 = Evaluation.from_json_object(json.loads(json.dumps(jsobj)))
assert ev1 == ev2
def from_json_object(data: Dict[str, Any]) -> 'Evaluation':
'''Generate TestCase object from JSON object'''
from executioner.program import Program # pylint: disable=C0415
assert all([x in data for x in ["program", "testcases",
"metrics"]]), "invalid Evaluation JSON"
return Evaluation(
Program.from_json_object(data["program"]),
testcases=[TestCase.from_json_object(
test_obj) for test_obj in data["testcases"]],
metrics=[BaseMetrics.from_json_object(
metric_obj) for metric_obj in data['metrics'] if metric_obj]
)
def test_basic_usage(self):
'''test basic usage of Program class'''
pgm = Program("print('hello world')", 'python3', FireJail())
pgm.compile()
testcase = TestCase()
pgm.execute(testcase)
assert testcase.real_output.strip() == "hello world"
def test_memory_limit(self):
'''Test memory limit of sandbox'''
code = '''a = [i for i in range(1024*1024*1024)]'''
pgm = Program(code, 'python3', self.sandbox)
pgm.compile()
testcase = TestCase('')
pgm.execute(testcase)
assert isinstance(testcase.error, MemoryOutError)
def test_python_success(self):
'''test successful python code'''
code = '''print("hello world")'''
pgm = Program(code, 'python3', self.sandbox)
pgm.compile()
testcase = TestCase(testcase_output="hello world")
pgm.execute(testcase)
assert testcase.real_output.strip() == testcase.output
def test_python_error(self):
'''test runtime error in python'''
code = '''a = 1/0'''
pgm = Program(code, 'python3', self.sandbox)
pgm.compile()
testcase = TestCase()
pgm.execute(testcase)
assert isinstance(testcase.error, RunTimeError)
def test_evaluate_success(self):
'''Test evaluate success'''
code = '''
print("hello world")
'''
pgm = Program(code, 'python3', FireJail())
testcases = [TestCase(testcase_output='hello world')]
ev_obj = Evaluation(
pgm,
testcases,
[Equal()],
)
ev_obj.evaluate()
assert testcases[0].scores == {'Equal': 1}
def test_tight_memory_limit(self):
'''Test memory limit tightly'''
code = '''a = [i for i in range({mem_limit})]'''
pgm = Program(code.format(mem_limit=100*1024*1024),
'python3', self.sandbox)
pgm.compile()
testcase = TestCase('')
pgm.execute(testcase)
assert isinstance(testcase.error, MemoryOutError)
def test_evaluate_compilation(self):
'''Test evaluate compilation error'''
code = '''
print("hello world"
'''
pgm = Program(code, 'python3', FireJail())
testcases = [TestCase(testcase_output='hello world')]
ev_obj = Evaluation(
pgm,
testcases,
[Equal()],
)
ev_obj.evaluate()
assert all(
[isinstance(test.error, CompilationError) for test in testcases])
def test_runtime_limit(self):
'''Test runtime limit of sandbox'''
code = '''
while(True):
pass
'''
pgm = Program(code, 'python3', self.sandbox)
pgm.compile()
testcase = TestCase('')
pgm.execute(testcase)
assert isinstance(testcase.error, TimeOutError)
def test_network_access(self):
'''Test network access of sandbox'''
code = '''
import http
conn = http.client.HTTPConnection('www.python.org')
conn.request("HEAD", "/index.html")
'''
pgm = Program(code, 'python3', self.sandbox)
pgm.compile()
testcase = TestCase('')
pgm.execute(testcase)
assert isinstance(testcase.error, RunTimeError)
def test_tight_runtime_limit(self):
'''
Test the runtime limit tightly
'''
code = '''
import time
time.sleep(10)
'''
pgm = Program(code, 'python3', self.sandbox)
pgm.compile()
testcase = TestCase()
pgm.execute(testcase)
assert isinstance(testcase.error, TimeOutError)
def test_c_success(self):
'''test successful C code'''
code = '''
#include<stdio.h>
int main(){
printf("hello world");
return 0;
}
'''
pgm = Program(code, 'C', self.sandbox)
pgm.compile()
testcase = TestCase(testcase_output="hello world")
pgm.execute(testcase)
assert testcase.real_output.strip() == testcase.output
def test_c_error(self):
'''test runtime error in C'''
code = '''
#include<stdio.h>
int main(){
int a, b;
scanf("%d", &b);
a = 10/b;
return 0;
}
'''
pgm = Program(code, 'C', self.sandbox)
pgm.compile()
testcase = TestCase("0")
pgm.execute(testcase)
assert isinstance(testcase.error, RunTimeError)
def test_file_access(self):
'''Test file access above workspace folder'''
code = '''
f = open('../a.txt', 'w')
f.write("hello world")
f.close()
'''
pgm = Program(code, 'python3', self.sandbox)
pgm.compile()
testcase = TestCase('')
pgm.execute(testcase)
file_path = path.join(path.dirname(
pgm.sandbox.file_location), '../a.txt')
if path.exists(file_path):
with open(file_path, 'r') as file_obj:
assert file_obj.read().strip() != "hello world"
else:
assert True
def test_multithread_runtime_limit(self):
'''Test the runtime limit for multithreaded code'''
code = '''
from multiprocessing import Pool
def f(t):
a = 0
for i in range(int(10**t)):
a += 1
return a
with Pool(2) as p:
print(p.map(f, [8.1, 8.1]))'''
pgm = Program(code, 'python3', self.sandbox)
pgm.compile()
testcase = TestCase()
pgm.execute(testcase)
assert isinstance(testcase.error, TimeOutError)
def test_json_conversion_non_program(self):
'''test json conversion of non program class'''
jsobj = ["executioner.evaluate", "TestCase"]
with pytest.raises(AssertionError):
Program.from_json_object(jsobj)
def test_json_conversion(self):
'''test json conversion'''
pgm1 = Program("print('hello world')", 'python3', FireJail())
jsobj = pgm1.to_json_object()
pgm2 = Program.from_json_object(json.loads(json.dumps(jsobj)))
assert pgm1 == pgm2