def test_function(capsys):
'''
Test judge.main
Sample data is supposed to be created with insufficient arguments.
Judging the created sample data is supposed to get a grade of 30.
'''
cmd = 'judge.py'
iters = 2 # Verify idempotence
for _ in range(iters):
# Create sample data
answer_path, result_path, action = judge.main([
cmd,
])
assert action == 'demo'
_ = capsys.readouterr()
# Judge with created data
_, _, action = judge.main([cmd, answer_path, result_path])
assert action == 'judge'
captured = capsys.readouterr()
ret = json.loads(captured.out)
assert ret['data'] == pytest.approx(0.3, 1e-4), ret['message']
result_path = 'result.json'
judge._demo(answer_path, result_path) # pylint: disable=protected-access
_ = capsys.readouterr()
# Judge with created data
_, _, action = judge.main([cmd, answer_path, result_path])
assert action == 'judge'
captured = capsys.readouterr()
ret = json.loads(captured.out)
assert ret['data'] == pytest.approx(0.3, 1e-4), ret['message']
def test_function(capsys):
'''
Test judge.main
Sample data is supposed to be created with insufficient arguments.
Judging the created sample data is supposed to get a grade of 120.
'''
argv = ['judge.py', ]
# Create sample data
answer_path, result_path, action = judge.main(argv)
assert action == 'demo'
_ = capsys.readouterr()
# Judge with created data
_, _, action = judge.main(argv + [answer_path, result_path])
assert action == 'judge'
captured = capsys.readouterr()
ret = json.loads(captured.out)
assert ret['data'] == 120
def main(session):
args = parser.parse_args()
problems = get_problems(session)
teams = get_team_names(session)
directory, target_files, problem_name = setup_submission(
args.submission_id, problems, teams)
if args.extract:
paths = extract(directory, target_files)
test_file = setup_testall(paths)
else:
paths = glob.glob(os.path.join(directory, '*.java'))
paths = [p for p in paths if not p.endswith('/RunTestAll.java')]
test_files = [os.path.basename(path) for path in paths
if any('testAll()' in line for line in open(path))]
assert len(test_files) == 1, test_files
test_file, = test_files
print("Stored in %s" % directory)
if directory:
remove_package(paths)
if args.patch:
patch_test_file(directory, test_file, problem_name,
args.override_version)
if args.instrument:
for path in paths:
instrument(path)
subprocess.check_call(['javac'] + glob.glob('%s/*.java' % directory))
subprocess.check_call(('java', '-cp', directory, 'RunTestAll'))
main_files = [os.path.basename(path) for path in paths
if any('void main(' in line for line in open(path))]
main_file, = main_files
for testcase in get_testcases(problem_name):
print(testcase)
try:
judge.main([os.path.join(directory, main_file), testcase])
except SystemExit as e:
print(e.args[0])
def test_function():
'''SmokeTest for judge.main'''
judge.main(['judge.py', SAMPLE_ANSWER, SAMPLE_RESULT])
from judge import main
if __name__ == '__main__':
main()
def fetch_problem_and_test(self, problems_root, problem_name, language,
solution_status):
problem_root = "{0}/{1}".format(problems_root.strip("/"), problem_name)
args = {}
args['problem_file_url'] = "{0}/problem.zip".format(problem_root)
args[
'submission_file_url'] = "{0}/solutions/{1}/{2}/solution{3}".format(
problem_root, language, solution_status, info.file_ext)
args['callback_url'] = 'https://example.com/callback'
try:
expected_result = requests.get(
"{0}/solutions/{1}/{2}/result.json".format(
problem_root, language, solution_status))
expected_result.raise_for_status()
expected_result = expected_result.json()
except requests.exceptions.HTTPError:
expected_result = None
result = judge.main(args)
orig_result = copy.deepcopy(result)
if expected_result == None:
print(result)
return
problem_manifest_file = open("problem/manifest.yaml", "r")
problem_manifest = yaml.safe_load(problem_manifest_file)
problem_manifest_file.close()
time_limit = problem_manifest['metadata']['limit']['time']
memory_limit = problem_manifest['metadata']['limit']['memory']
try:
del expected_result['resource']
except KeyError:
print("Resource field is not found on the expected result.")
del result['resource']
if result['status'] == 'CE':
del result['message']
if 'message' in expected_result:
del expected_result['message']
for batch in range(0, len(expected_result['batches'])):
try:
del expected_result['batches'][batch]['resource']
except KeyError:
print("Resource field is not found on the expected result.")
del result['batches'][batch]['resource']
for testcase in range(
0, len(expected_result['batches'][batch]['testcases'])):
if result['batches'][batch]['testcases'][testcase][
'status'] in ['AC', 'WA', 'IR']:
self.assertTrue(
result['batches'][batch]['testcases'][testcase]
['resource']['time'] <= time_limit)
self.assertTrue(
result['batches'][batch]['testcases'][testcase]
['resource']['memory'] <= memory_limit)
if result['batches'][batch]['testcases'][testcase][
'status'] == 'TLE':
self.assertTrue(
result['batches'][batch]['testcases'][testcase]
['resource']['time'] >= time_limit)
self.assertTrue(
result['batches'][batch]['testcases'][testcase]
['resource']['memory'] <= memory_limit)
if result['batches'][batch]['testcases'][testcase][
'status'] == 'MLE':
self.assertTrue(
result['batches'][batch]['testcases'][testcase]
['resource']['time'] <= time_limit)
self.assertTrue(
result['batches'][batch]['testcases'][testcase]
['resource']['memory'] >= memory_limit)
try:
del expected_result['batches'][batch]['testcases'][
testcase]['resource']
except KeyError:
print(
"Resource field is not found on the expected result.")
del result['batches'][batch]['testcases'][testcase]['resource']
if not expected_result == result:
print(solution_status)
print(json.dumps(orig_result))
self.assertEqual(expected_result, result)