def run_lli(input_f, out_f, err_f):
additional_clang_input = [
mx_subst.path_substitutions.substitute(ci)
for ci in parsed_args.clang_input or []
]
toolchain_clang = mx_sulong._get_toolchain_tool("native,CC")
mx_sulong.llvm_tool([
toolchain_clang, "-O0", "-Wno-everything", "-o", tmp_out,
input_f
] + additional_clang_input)
with open(out_f, 'w') as o, open(err_f, 'w') as e:
mx.command_function('lli')(parsed_args.lli_arg + [tmp_out],
timeout=lli_timeout,
nonZeroIsFatal=False,
out=o,
err=e)
def fuzz(args=None, out=None):
parser = ArgumentParser(prog='mx fuzz', description='')
parser.add_argument('--seed',
help='Seed used for randomness.',
metavar='<seed>',
type=int,
default=int(time.time()))
parser.add_argument(
'--size',
help=
'Approximate size for the generated testcases in lines of code. (default: %(default)s)',
metavar='<size>',
type=int,
default=30)
parser.add_argument(
'--timeout',
help=
'Timeout for running the generated program. (default: %(default)s)',
metavar='<timeout>',
type=int,
default=10)
parser.add_argument(
'--generator',
help='Tool used for generating the testcases. (default: %(default)s)',
choices=("llvm-stress", "csmith"),
default="llvm-stress")
parser.add_argument(
'--nrtestcases',
help='Number of testcases to be generated. (default: %(default)s)',
metavar='<nrtestcases>',
type=int,
default=10)
parser.add_argument('outdir',
help='The output directory.',
metavar='<outdir>')
parsed_args = parser.parse_args(args)
tmp_dir = None
try:
tmp_dir = tempfile.mkdtemp()
tmp_ll = os.path.join(tmp_dir, 'tmp.ll')
tmp_main_ll = os.path.join(tmp_dir, 'tmp.main.ll')
tmp_c = os.path.join(tmp_dir, 'tmp.c')
tmp_out = os.path.join(tmp_dir, 'tmp.out')
tmp_sulong_out = os.path.join(tmp_dir, 'tmp_sulong_out.txt')
tmp_bin_out = os.path.join(tmp_dir, 'tmp_bin_out.txt')
tmp_sulong_err = os.path.join(tmp_dir, 'tmp_sulong_err.txt')
tmp_bin_err = os.path.join(tmp_dir, 'tmp_bin_err.txt')
rand = Random(parsed_args.seed)
passed = 0
invalid = 0
gen = []
for _ in range(parsed_args.nrtestcases):
toolchain_clang = mx_sulong._get_toolchain_tool("native,CC")
if parsed_args.generator == "llvm-stress":
_run_fuzz_tool("llvm-stress", [
"-o", tmp_ll, "--size",
str(parsed_args.size), "--seed",
str(rand.randint(0, 10000000))
])
fuzz_main = os.path.join(
mx.dependency('SULONG_TOOLS',
fatalIfMissing=True).get_output(), "src",
"fuzzmain.c")
mx.run([
toolchain_clang, "-O0", "-Wno-everything", "-o", tmp_out,
tmp_ll, fuzz_main
])
mx_sulong.llvm_tool([
"clang", "-O0", "-Wno-everything", "-S", "-emit-llvm",
"-o", tmp_main_ll, fuzz_main
])
mx_sulong.llvm_tool(
["llvm-link", "-o", tmp_ll, tmp_ll, tmp_main_ll])
mx_sulong.llvm_tool(["llvm-dis", "-o", tmp_ll, tmp_ll])
else:
csmith_exe = "csmith"
csmith_headers = mx.get_env('CSMITH_HEADERS', None)
if not csmith_headers:
mx.abort("Environment variable `CSMITH_HEADERS` not set")
mx.run([
csmith_exe, "-o", tmp_c, "--seed",
str(rand.randint(0, 10000000))
])
mx.run([
toolchain_clang, "-O0", "-Wno-everything",
"-I" + csmith_headers, "-o", tmp_out, tmp_c
])
mx_sulong.llvm_tool([
"clang", "-O0", "-Wno-everything", "-S", "-emit-llvm",
"-I" + csmith_headers, "-o", tmp_ll, tmp_c
])
gen.append((tmp_c, 'autogen.c'))
timeout = parsed_args.timeout
with open(tmp_sulong_out, 'w') as o, open(tmp_sulong_err,
'w') as e:
mx_sulong.runLLVM([
'--llvm.llDebug', '--llvm.traceIR',
'--experimental-options', tmp_out
],
timeout=timeout,
nonZeroIsFatal=False,
out=o,
err=e)
with open(tmp_bin_out, 'w') as o, open(tmp_bin_err, 'w') as e:
try:
mx.run([tmp_out], timeout=timeout, out=o, err=e)
except SystemExit:
invalid += 1
continue
if all(
filecmp.cmp(sulong_f, bin_f, shallow=False)
for sulong_f, bin_f in ((tmp_sulong_out, tmp_bin_out),
(tmp_sulong_err, tmp_bin_err))):
passed += 1
else:
now = str(datetime.datetime.now())
now = now.replace(":", "_").replace(" ", "_")
current_out_dir = os.path.join(
parsed_args.outdir, now + "_" + parsed_args.generator)
os.makedirs(current_out_dir)
gen += [
(tmp_ll, 'autogen.ll'),
(tmp_out, 'autogen'),
(tmp_sulong_out, 'sulong_out.txt'),
(tmp_bin_out, 'bin_out.txt'),
(tmp_sulong_err, 'sulong_err.txt'),
(tmp_bin_err, 'bin_err.txt'),
]
for tmp_f, gen_f_name in gen:
shutil.copy(tmp_f, os.path.join(current_out_dir,
gen_f_name))
finally:
if tmp_dir:
shutil.rmtree(tmp_dir)
mx.log("Test report")
mx.log("total testcases: {} seed: {}".format(parsed_args.nrtestcases,
parsed_args.seed))
mx.log("interesting testcases: {} invalid testcases: {}".format(
parsed_args.nrtestcases - invalid - passed, invalid))
def check_interesting(args=None, out=None):
parser = ArgumentParser(prog='mx check-interesting', description='')
parser.add_argument('input', help='The input file.', metavar='<input>')
parser.add_argument(
'--bin-startswith',
help=
'Prefix the reference output of interesting testprograms has to start with.',
metavar='<refstartswith>',
default=None)
parser.add_argument(
'--sulong-startswith',
help=
'Prefix the output of interesting testprograms has to start with when executed on Sulong.',
metavar='<teststartswith>',
default=None)
parsed_args = parser.parse_args(args)
def _not_interesting(msg=None):
if msg:
mx.logv(
mx.colorize("mx check-interesting: no ({})".format(msg),
color="blue"))
sys.exit(0)
def _interesting(msg=None):
if msg:
mx.logv(
mx.colorize("mx check-interesting: yes ({})".format(msg),
color="cyan"))
sys.exit(1)
def _files_match_pattern(prefix, out_file, err_file):
with open(out_file, 'r') as o, open(err_file, 'r') as e:
if not any(
fl.startswith(prefix)
for fl in (next(o, ""), next(e, ""))):
return False
return True
tmp_dir = None
try:
tmp_dir = tempfile.mkdtemp()
tmp_out = os.path.join(tmp_dir, 'tmp.out')
tmp_out_o3 = os.path.join(tmp_dir, 'tmp.o3.out')
tmp_sulong_out = os.path.join(tmp_dir, 'tmp_sulong_out.txt')
tmp_bin_out = os.path.join(tmp_dir, 'tmp_bin_out.txt')
tmp_sulong_err = os.path.join(tmp_dir, 'tmp_sulong_err.txt')
tmp_bin_err = os.path.join(tmp_dir, 'tmp_bin_err.txt')
tmp_bin_out_o3 = os.path.join(tmp_dir, 'tmp_bin_out_o3.txt')
tmp_bin_err_o3 = os.path.join(tmp_dir, 'tmp_bin_err_o3.txt')
try:
toolchain_clang = mx_sulong._get_toolchain_tool("native,CC")
mx.run([
toolchain_clang, "-O0", "-Wno-everything", "-o", tmp_out,
parsed_args.input
])
mx.run([
toolchain_clang, "-O3", "-Wno-everything", "-o", tmp_out_o3,
parsed_args.input
])
except SystemExit:
_not_interesting("Compiling the input file failed!")
with open(tmp_sulong_out, 'w') as o, open(tmp_sulong_err, 'w') as e:
mx_sulong.runLLVM([tmp_out],
timeout=10,
nonZeroIsFatal=False,
out=o,
err=e)
with open(tmp_bin_out, 'w') as o, open(tmp_bin_err, 'w') as e:
try:
mx.run([tmp_out], timeout=10, out=o, err=e)
except SystemExit:
_not_interesting(
"Running the O0 compiled input files natively failed!")
with open(tmp_bin_out_o3, 'w') as o, open(tmp_bin_err_o3, 'w') as e:
try:
mx.run([tmp_out_o3], timeout=10, out=o, err=e)
except SystemExit:
_not_interesting(
"Running the O3 compiled input files natively failed!")
if not all(
filecmp.cmp(bin_f, bin_f_o3, shallow=False)
for bin_f, bin_f_o3 in ((tmp_bin_out, tmp_bin_out_o3),
(tmp_bin_err, tmp_bin_err_o3))):
_not_interesting("The result of O0 and O3 is different!")
if all(
filecmp.cmp(sulong_f, bin_f, shallow=False)
for sulong_f, bin_f in ((tmp_sulong_out, tmp_bin_out),
(tmp_sulong_err, tmp_bin_err))):
_not_interesting("The result of native and sulong is the same!")
if parsed_args.bin_startswith and not _files_match_pattern(
parsed_args.bin_startswith, tmp_bin_out, tmp_bin_err):
_not_interesting("The native result does not match the pattern")
if parsed_args.sulong_startswith and not _files_match_pattern(
parsed_args.sulong_startswith, tmp_sulong_out, tmp_sulong_err):
_not_interesting("The sulong result does not match the pattern")
_interesting(
"Results are different and match the pattern (if provided)")
finally:
if tmp_dir:
shutil.rmtree(tmp_dir)