def create_success_output(binaries, tests_result):
logger.debug('Creating json with tests results')
teachers = {
'compilation': [{
'binary':
binary,
'configurations': [{
'name': name,
'result': result
} for name, result in configurations.items()]
} for binary, configurations in binaries.items()],
'tests': [{
'configuration':
configuration,
'cases': [{
'name': name,
'result': result
} for name, result in cases.items()]
} for configuration, cases in tests_result.items()]
}
with open(Config.teachers_json(), 'w') as f:
json.dump(teachers, f, cls=EnhancedJSONEncoder, indent=4)
students = {
'compilation': [{
'binary':
binary,
'configurations': [{
'name':
name,
'result':
result.compiler_output if Config.get_visibility(
name, Visibility.BUILD) else ''
} for name, result in configurations.items()]
} for binary, configurations in binaries.items()],
'tests': [{
'configuration':
configuration,
'cases': [{
'name':
name,
'status':
result.get_status() == tester.tests.TestResultStatus.SUCCESS,
'result':
result.stdout if Config.get_visibility(
configuration, Visibility.TESTS) else ''
} for name, result in cases.items()]
} for configuration, cases in tests_result.items()]
}
with open(Config.students_json(), 'w') as f:
json.dump(students, f, cls=EnhancedJSONEncoder, indent=4)
def build_tests():
# copy submission to main test file
if Config.get_mode() == SubmissionMode.COPY:
# this is quite a hack, input file is always named main.cpp, so
# we need to change that to header file
shutil.copy2(os.path.join(Config.submission_path(), 'main.cpp'),
os.path.join(Config.tests_path(), 'submission.h'))
result = {}
for configuration in ['debug', 'release']:
result[configuration] = build(
os.path.join(Config.tests_path(), 'build-' + configuration))
return result
def configure():
logging.config.dictConfig({
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format':
'{levelname:8s} {asctime} {module:10s} {funcName:20s} {message}',
'style': '{',
}
},
'handlers': {
'file': {
'level': 'DEBUG',
'class': 'logging.FileHandler',
'filename': os.path.join(Config.output_path(),
'main-tester.log'),
'mode': 'w',
'formatter': 'verbose',
},
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'verbose',
},
},
'root': {
'level': 'NOTSET',
'handlers': ['console', 'file']
},
})
def build(project_path):
build_result = compiler.compile_cmake_project(project_path)
build_output = os.path.join(Config.build_output_path(),
project_path.replace('/', '_') + '.txt')
with open(build_output, "w") as text_file:
text_file.write(build_result.compiler_output)
return build_result
def main():
"""
Main entry point of our python tester. It will first of all
load settings, compile sources, run tests and collect results,
then it will pack those and send everything to output folder.
"""
# start logger
tester.logger.configure()
logger.info('Tester started...')
logger.debug(Config.dumps())
compiler.check_cmake()
test_results = {}
binaries = {'tests': build_tests()}
if Config.get_mode() == SubmissionMode.BUILD:
binaries['submission'] = build_submission()
for configuration, binary in binaries['tests'].items():
if binary.errno != 0:
continue # build failed
test_results[configuration] = {}
tests = tester.tests.Tests(binary.output_path, configuration)
for test_case in tests.test_cases:
submission_binary = binaries.get('submission',
{}).get(configuration, None)
if submission_binary:
if submission_binary.errno != 0:
continue # cannot compile submission
test_result = tests.run_test(test_case,
submission_binary.output_path)
else:
test_result = tests.run_test(test_case)
test_results[configuration][test_case] = test_result
create_success_output(binaries, test_results)
logger.info('Finished.')
def handle_exception(exc_type, exc_value, exc_traceback):
try:
logger.exception('Uncaught exception',
exc_info=(exc_type, exc_value, exc_traceback))
logger.debug('Creating json with errors')
output = {
'status':
'Exception',
'text':
'This should never happen, please report this incident to your friendly administrators.',
}
with open(Config.teachers_json(), 'w') as f:
json.dump(output, f)
with open(Config.students_json(), 'w') as f:
json.dump(output, f)
finally:
sys.exit(0)
def build_submission():
if Config.get_mode() != SubmissionMode.BUILD:
return {}
shutil.copy2(os.path.join(Config.submission_path(), 'main.cpp'),
Config.submission_project())
result = {}
for configuration in ['debug', 'release']:
project_result = compiler.compile_cmake_lists(
Config.submission_project(), configuration)
build_output = os.path.join(
Config.build_output_path(),
Config.submission_project().replace('/', '_') + '-' +
configuration + '-cmake-lists.txt')
with open(build_output, "w") as text_file:
text_file.write(project_result.compiler_output)
if project_result.errno != 0:
return {}
result[configuration] = build(project_result.output_path)
return result
def run_test(self, test_case, submission_binary=None):
logger.info('Running test "%s" in configuration "%s".', test_case,
self.configuration)
temp_dir = tempfile.mkdtemp()
logger.debug('Using folder "%s"', temp_dir)
catch_path = os.path.join(temp_dir, self.CATCH_EXEC_NAME)
shutil.copy2(self.binary, catch_path)
os.chmod(temp_dir, 0o777) # everyone os allowed to do everything
os.chmod(catch_path, 0o777)
pw_record = pwd.getpwnam("apc-test")
user_uid = pw_record.pw_uid
user_gid = pw_record.pw_gid
env = {}
if submission_binary:
submission_path = os.path.join(temp_dir, self.SUBMISSION_EXEC_NAME)
shutil.copy2(submission_binary, submission_path)
env = {
'SUBMISSIONPATH': submission_path,
'DATAPATH': Config.data_path(),
}
args = [*self._options, test_case.replace(',', '\,')
] # comma in test is not allowed, you need to escape it
logger.debug(
'Starting tests file %s, with arguments "%s" current working directory "%s"',
catch_path, ', '.join(args), temp_dir)
def demote(user_uid, user_gid):
os.setgid(user_gid)
os.setuid(user_uid)
with TimeoutManager() as timeout:
try:
catch = subprocess.run(
[catch_path, *args],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=lambda: demote(user_uid, user_gid),
timeout=timeout,
cwd=temp_dir,
env=env)
stdout = catch.stdout.decode('utf-8')
stderr = catch.stderr.decode('utf-8')
logger.info('Test finished errno: %d', catch.returncode)
logger.debug('Test stdout: "%s"\n stderr: "%s"', stdout,
stderr)
return TestResult(catch.returncode, stdout, stderr)
except subprocess.TimeoutExpired:
logger.info('Test timeouted.')
# first negative number that cannot be represented with 32 bit signed int (assuming 2-complement)
return TestResult(-2147483649, '',
'Subprocess timeout expired!')
def __init__(self, binary, configuration):
self.binary = binary
self.configuration = configuration
self._options = Config.get_catch2_configuration(configuration)
self.test_cases = self._list_tests()