def run_test_memcheck(tool: str,
compiler: Optional[Compiler],
reporter: Reporter,
tests: List[str],
config: Config,
timeout: Optional[float] = None) -> bool:
reporter.log(f'Running tests with cuda-memcheck --tool {tool}', 'title')
runner = MemcheckRunner(tool)
compiler = config.find_compiler() if compiler is None else compiler
if compiler is None:
raise RuntimeError("Could not find a suitable compiler")
compiler = config.common_flags(compiler).add_flag('-O3').add_flag(
'-g').add_flag('-Xcompiler', '-rdynamic').add_flag('-lineinfo')
orig_tests = tests
tests = expand_glob(tests, ['tests/*medium*'])
if not tests:
no_tests_error(orig_tests, ['tests'])
return run_test(compiler=compiler,
runner=runner,
reporter=reporter,
name=f'test-memcheck-{tool}',
tests=tests,
config=config,
timeout=timeout)
def run_test_asan(compiler: Optional[Compiler],
reporter: Reporter,
tests: List[str],
config: Config,
timeout: Optional[float] = None) -> bool:
reporter.log('Running tests with address sanitizer', 'title')
runner = AsanRunner()
compiler = config.find_compiler() if compiler is None else compiler
if compiler is None:
sys.exit("I'm sorry, I could not find a suitable compiler.")
if isinstance(compiler, NvccCompiler):
compiler = config.common_flags(compiler).add_flag(
'-Xcompiler',
'"-fsanitize=address"').add_flag('-Xcompiler',
'"-fsanitize=undefined"')
runner.env[
'ASAN_OPTIONS'] = 'protect_shadow_gap=0:replace_intrin=0:detect_leaks=0'
else:
compiler = config.common_flags(compiler).add_flag(
'-fsanitize=address').add_flag('-fsanitize=undefined')
compiler = compiler.add_flag('-g')
orig_tests = tests
tests = expand_glob(tests, ['tests/*'])
if not tests:
no_tests_error(orig_tests, ['tests'])
return run_test(compiler=compiler,
runner=runner,
reporter=reporter,
name='test-asan',
tests=tests,
config=config,
timeout=timeout)
def run_test_tsan(compiler: Compiler,
reporter: Reporter,
tests: List[str],
config: Config,
timeout: Optional[float] = None) -> bool:
reporter.log('Running tests with thread sanitizer', 'title')
reporter.log('Not implemented')
return True
runner = TsanRunner()
compiler = config.find_compiler() if compiler is None else compiler
if not isinstance(compiler, ClangCompiler):
reporter.log(
'Thread sanitizer is supported only with clang. Skipping test')
return True
compiler = common_flags(compiler).add_flag('-fsanitize=thread')
orig_tests = tests
tests = expand_glob(tests, ['tests/*'])
if not tests:
no_tests_error(orig_tests, ['tests'])
return run_test(compiler=compiler,
runner=runner,
reporter=reporter,
name='test-tsan',
tests=tests,
timeout=timeout)
def run_test_uninit(compiler: Optional[Compiler],
reporter: Reporter,
tests: List[str],
config: Config,
timeout: Optional[float] = None) -> bool:
reporter.log('Running tests with uninitialized variable check', 'title')
runner = Runner()
compiler = find_clang_compiler() if compiler is None else compiler
if compiler is None:
sys.exit("I'm sorry, I could not find a suitable clang compiler.")
if isinstance(compiler, ClangCompiler) and compiler.version[0] < 8:
sys.exit(f"I'm sorry, but the clang compiler {compiler} is too old.")
compiler = config.common_flags(compiler).add_flag('-O3').add_flag(
'-g').add_flag('-ftrivial-auto-var-init=pattern')
orig_tests = tests
tests = expand_glob(tests, ['tests/*', 'benchmarks/*'])
if not tests:
no_tests_error(orig_tests, ['tests', 'benchmarks'])
return run_test(compiler=compiler,
runner=runner,
reporter=reporter,
name='test-uninit',
tests=tests,
config=config,
timeout=timeout)
def compile_assembly(compiler: Compiler,
reporter: Reporter,
tests: List[str],
config: Config,
timeout: Optional[float] = None) -> bool:
reporter.log('Compiling to assembly', 'title')
compiler = config.find_compiler() if compiler is None else compiler
if compiler is None:
raise RuntimeError("Could not find a suitable compiler")
compiler = config.common_flags(compiler).add_flag('-O3').add_flag(
'-S').add_flag('-fverbose-asm')
rep = reporter.analysis_group('assembly')
output = rep.compilation(compiler.add_source(
config.source)).compile(out_file=config.binary)
if not output.is_success():
return False
rep.analyze(open(config.binary, 'r').read())
return True
def run_test_plain(compiler: Optional[Compiler],
reporter: Reporter,
tests: List[str],
config: Config,
timeout: Optional[float] = None) -> bool:
reporter.log('Running tests', 'title')
runner = Runner()
compiler = config.find_compiler() if compiler is None else compiler
if compiler is None:
sys.exit("I'm sorry, I could not find a suitable compiler.")
compiler = config.common_flags(compiler).add_flag('-O3').add_flag('-g')
orig_tests = tests
tests = expand_glob(tests, ['tests/*', 'benchmarks/*'])
if not tests:
no_tests_error(orig_tests, ['tests', 'benchmarks'])
return run_test(compiler=compiler,
runner=runner,
reporter=reporter,
name='test-plain',
tests=tests,
config=config,
timeout=timeout)
def run_benchmark_all(compiler: Optional[Compiler],
reporter: Reporter,
tests: List[str],
config: Config,
timeout: Optional[float] = None) -> bool:
reporter.log('Running benchmark', 'title')
runner = Runner()
compiler = config.find_compiler() if compiler is None else compiler
if compiler is None:
raise RuntimeError("Could not find a suitable compiler")
compiler = config.common_flags(compiler).add_flag('-O3').add_flag('-g')
orig_tests = tests
tests = expand_glob(tests, ['benchmarks/*'])
if not tests:
no_tests_error(orig_tests, ['benchmarks'])
return run_benchmark(compiler=compiler,
runner=runner,
reporter=reporter,
name='benchmark-all',
tests=tests,
config=config,
timeout=timeout)
def compile_assembly(compiler: Compiler,
reporter: Reporter,
tests: List[str],
config: Config,
timeout: Optional[float] = None) -> bool:
reporter.log('Compiling to assembly', 'title')
compiler = config.find_compiler() if compiler is None else compiler
if compiler is None:
raise RuntimeError("Could not find a suitable compiler")
compiler = config.common_flags(compiler).add_flag('-O3').add_flag(
'-S').add_flag('-fverbose-asm')
rep = reporter.analysis_group('assembly')
output = rep.compilation(compiler.add_source(
config.source)).compile(out_file=config.binary)
if not output.is_success():
return False
assembly = open(config.binary, 'r').read()
if len(assembly.encode('utf-8')) > MAX_ASSEMBLY_OUTPUT:
assembly = "Generated assembly was too long and wasn't stored"
rep.analyze(assembly)
return True
def run_benchmark(compiler: Compiler,
runner: Runner,
reporter: Reporter,
name: str,
tests: List[str],
config: Config,
timeout: Optional[float] = None) -> bool:
rep = reporter.benchmark_group(name, tests)
output = rep.compilation(
compiler.add_source(config.tester).add_source(
config.source)).compile(out_file=config.binary)
if not output.is_success():
return False
for test in tests:
runner_output = runner.run(config,
config.benchmark_command(test),
timeout=parse_timeout(test, timeout))
rep.benchmark(test, runner_output)
if runner_output.errors or not runner_output.run_successful:
return False
return True
def cli(config: Config):
parser = argparse.ArgumentParser(
usage=f'{sys.argv[0]} [-h|--help] [options] command [tests ...]',
description=f'''
PPC grading tool
Run all tests:
{sys.argv[0]} test
Run all benchmarks
{sys.argv[0]} benchmark
Run tests with address sanitizer only
{sys.argv[0]} test-asan
Run only tests 001 and 017
{sys.argv[0]} test tests/001 tests/017
''',
allow_abbrev=False,
formatter_class=argparse.RawTextHelpFormatter,
add_help=False,
)
# We'll handle help manually to add tests argument in help but not when
# parsing for the first time. Otherwise argparse asks to give both command
# and tests even though giving zero tests suffices.
parser.add_argument('-h',
'--help',
action=HelpAction,
nargs=0,
dest='help',
help='show this help message and exit')
parser.add_argument('--json',
action='store_const',
dest='reporter',
default=TerminalReporter,
const=JsonReporter,
help=argparse.SUPPRESS)
parser.add_argument('--timeout', type=float, help='timeout for each test')
compiler_group = parser.add_mutually_exclusive_group()
compiler_group.add_argument(
'--clang',
dest='compiler',
nargs='?',
default=None,
const='',
help=
'use given clang compiler, or if left empty try to find a suitable compiler',
type=clang_compiler)
compiler_group.add_argument(
'--gcc',
dest='compiler',
nargs='?',
default=None,
const='',
help=
'use given gcc compiler, or if left empty try to find a suitable compiler',
type=gcc_compiler)
compiler_group.add_argument(
'--nvcc',
dest='compiler',
nargs='?',
default=None,
const='',
help=
'use given nvcc compiler, or if left empty try to find a suitable compiler',
type=nvcc_compiler)
parser.add_argument('--file',
dest='file',
default=config.source,
help='file to be graded',
type=str)
parser.add_argument('--binary',
dest='binary',
default=config.binary,
help=argparse.SUPPRESS,
type=str)
parser.add_argument(
'--ignore-errors',
action='store_const',
dest='ignore_errors',
default=False,
const=True,
help='run through all tests without stopping at the first error',
)
parser.add_argument(
'-v',
'--verbose',
action=VerboseAction,
nargs=0,
help=
'show the commands that I run (give twice to make me more talkative)')
parser.add_argument(
'commands',
metavar='command',
type=command_type(config.gpu),
help='command to run, for example \'test\' or \'benchmark\'')
# Rest of the arguments are considered to be tests
args, tests = parser.parse_known_args()
# Check that there aren't any extra flags
for test in tests:
if test.startswith('-'):
sys.exit(f'{sys.argv[0]}: error: Unknown flag {test}')
reporter = args.reporter(config)
compiler = args.compiler
timeout = args.timeout
config.source = args.file
config.binary = args.binary
config.ignore_errors = args.ignore_errors
for command in args.commands:
passed = command(compiler=compiler,
reporter=reporter,
tests=tests,
timeout=timeout,
config=config)
if not passed:
break
reporter.finalize()