def MakeNonsfiRuntime():
"""Builds just the nonsfi runtime."""
# Translate tempdir/szrt.ll and tempdir/szrt_ll.ll to
# szrt_nonsfi_{target}.tmp.o.
Translate(ll_files,
['-mtriple=' + target_info.triple] + target_info.llc_flags +
['-relocation-model=pic', '-force-tls-non-pic', '-malign-double'],
TmpFile('{dir}/szrt_nonsfi_{target}.tmp.o'),
verbose, target_info.target)
# Assemble srcdir/szrt_asm_{target}.s to tempdir/szrt_asm_{target}.o.
shellcmd(['llvm-mc',
'-triple=' + target_info.triple, '--defsym NONSFI=1',
'-filetype=obj',
'-o', TmpFile('{dir}/szrt_nonsfi_asm_{target}.o'),
'{srcdir}/szrt_asm_{target}.s'.format(
srcdir=srcdir, target=target_info.target)
], echo=verbose)
# Write full szrt_nonsfi_{target}.o.
PartialLink([TmpFile('{dir}/szrt_nonsfi_{target}.tmp.o'),
TmpFile('{dir}/szrt_nonsfi_asm_{target}.o')],
['-m {ld_emu}'.format(ld_emu=target_info.ld_emu)],
OutFile('{rtdir}/szrt_nonsfi_{target}.o'),
verbose)
shellcmd([GetObjcopyCmd(target_info.target),
'--strip-symbol=NONSFI',
OutFile('{rtdir}/szrt_nonsfi_{target}.o')])
def MakeSandboxedRuntime():
"""Builds just the sandboxed runtime."""
# Translate tempdir/szrt.ll and tempdir/szrt_ll.ll to szrt_sb_{target}.o.
# The sandboxed library does not get the profiler helper function as the
# binaries are linked with -nostdlib.
Translate(ll_files,
['-mtriple=' + targets.ConvertTripleToNaCl(target_info.triple)] +
target_info.llc_flags,
TmpFile('{dir}/szrt_sb_{target}.tmp.o'),
verbose,target_info.target)
# Assemble srcdir/szrt_asm_{target}.s to tempdir/szrt_asm_{target}.o.
shellcmd(['llvm-mc',
'-triple=' + targets.ConvertTripleToNaCl(target_info.triple),
'--defsym NACL=1',
'-filetype=obj',
'-o', TmpFile('{dir}/szrt_sb_asm_{target}.o'),
'{srcdir}/szrt_asm_{target}.s'.format(
srcdir=srcdir, target=target_info.target)
], echo=verbose)
PartialLink([TmpFile('{dir}/szrt_sb_{target}.tmp.o'),
TmpFile('{dir}/szrt_sb_asm_{target}.o')],
['-m {ld_emu}'.format(ld_emu=target_info.sb_emu)],
OutFile('{rtdir}/szrt_sb_{target}.o'),
verbose)
shellcmd([GetObjcopyCmd(target_info.target),
'--strip-symbol=NACL',
OutFile('{rtdir}/szrt_sb_{target}.o')])
def run(is_cpp):
"""Passes its arguments directly to pnacl-clang.
If -fsanitize-address is specified, extra information is passed to
pnacl-clang to ensure that later instrumentation in pnacl-sz can be
performed. For example, clang automatically inlines many memory allocation
functions, so this script will redefine them at compile time to make sure
they can be correctly instrumented by pnacl-sz.
"""
pnacl_root = FindBaseNaCl()
dummy_subs = {'calloc': {'sig': ['void *', 'size_t', 'size_t'],
'sub': '__asan_dummy_calloc'},
'_calloc': {'sig': ['void *', 'size_t', 'size_t'],
'sub': '__asan_dummy_calloc'}}
subs_src = (
'{root}/toolchain_build/src/subzero/pydir/sz_clang_dummies.c'
).format(root=pnacl_root)
clang = (
'{root}/toolchain/linux_x86/pnacl_newlib_raw/bin/pnacl-clang{pp}'
).format(root=pnacl_root, pp='++' if is_cpp else '')
args = sys.argv
args[0] = clang
tmp_dir = ''
if '-fsanitize-address' in args:
args.remove('-fsanitize-address')
include_dirs = set()
tmp_dir = tempfile.mkdtemp()
for i, arg in enumerate(args[1:], 1):
if not os.path.isfile(arg):
continue
src = os.path.basename(arg)
ext = os.path.splitext(arg)[1]
if ext in ['.c', '.cc', '.cpp']:
include_dirs |= {os.path.dirname(arg)}
dest_name = os.path.join(tmp_dir, src)
with open(dest_name, 'w') as dest:
dest.write(subsToMacros(dummy_subs, arg))
with open(arg) as src:
for line in src:
dest.write(line)
args[i] = dest_name
# If linking (not single file compilation) then add dummy definitions
if not ('-o' in args and
('-c' in args or '-S' in args or '-E' in args)):
args.append(subs_src)
for d in include_dirs:
args.append('-iquote {d}'.format(d=d))
if '-fno-inline' not in args:
args.append('-fno-inline')
err_code = 0
try:
shellcmd(args, echo=True)
except subprocess.CalledProcessError as e:
print e.output
err_code = e.returncode
if tmp_dir != '':
shutil.rmtree(tmp_dir)
exit(err_code)
def check_blas(self):
print "Checking if provided BLAS works...",
# This function simply generates a FORTRAN program
# that contains few calls to BLAS routine and then
# checks if compilation, linking and execution are succesful
# Try to detect which BLAS is used
if re.search('mkl', self.config.blaslib, re.IGNORECASE):
self.config.blasname = "mkl"
elif re.search('acml', self.config.blaslib, re.IGNORECASE):
self.config.blasname = "acml"
if self.config.blasname != "Unknown":
if self.config.compiler == "Intel":
self.config.cflags += '' #" -openmp"
# self.config.ldflags_c += " -openmp"
# self.config.ldflags_fc += " -openmp"
elif self.config.compiler == "GNU":
self.config.cflags += '' #" -fopenmp"
# self.config.ldflags_c += " -fopenmp"
# self.config.ldflags_fc += " -fopenmp"
sys.stdout.flush()
writefile(
'tmpf.f', """
program ftest
double precision da, dx(1)
dx(1)=1
da = 2
call dscal(1,da,dx,1)
stop
end\n""")
#fcomm = self.config.fc+' -o tmpf '+'tmpf.f '+self.config.blaslib+' '+self.config.ldflags_fc+' -lm'
fcomm = self.config.fc + ' -o tmpf ' + 'tmpf.f ' + self.config.blaslib + ' -lm'
(output, error, retz) = shellcmd(fcomm)
if (retz != 0):
print '\n\nBLAS: provided BLAS cannot be used! aborting...'
print 'error is:\n', '*' * 50, '\n', fcomm, '\n', error, '\n', '*' * 50
sys.exit()
comm = './tmpf'
(output, error, retz) = shellcmd(comm)
if (retz != 0):
print '\n\nBLAS: provided BLAS cannot be used! aborting...'
print 'error is:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50
sys.exit()
delfiles(['tmpf.f', 'tmpf'])
print 'yes'
return 0
def set_mangling(self):
""" Sets the INTFACE variable in Bmake.inc """
# This one generates a program equivalent to that in BLACS/INSTALL
# that checks the mangling in FORTRAN function symbols
print 'Setting Fortran mangling...',
sys.stdout.flush()
writefile(
'tmpf.f', """
program intface
external c_intface
integer i
call c_intface(i)
stop
end\n""")
writefile(
'tmpc.c', """
#include <stdio.h>
void c_intface_(int *i){fprintf(stdout, \"-DADD_\");fflush(stdout);}
void c_intface(int *i){fprintf(stdout, \"-DNOCHANGE\");fflush(stdout);}
void c_intface__(int *i){fprintf(stdout, \"-DfcIsF2C\");fflush(stdout);}
void C_INTFACE(int *i){fprintf(stdout, \"-DUPCASE\");fflush(stdout);}\n"""
)
ccomm = self.config.cc + ' ' + self.config.cflags + ' -c tmpc.c -o tmpc.o'
fcomm = self.config.fc + ' ' + self.config.fflags + ' tmpf.f tmpc.o -o xintface'
(output, error, retz) = shellcmd(ccomm)
if retz:
print '\n\nCOMMON: in set_mangling: cannot compile'
print 'error is:\n', '*' * 50, '\n', error, '\n', '*' * 50
#sys.exit()
(output, error, retz) = shellcmd(fcomm)
if retz:
print '\n\nCOMMON: in set_mangling: cannot compile'
print 'error is:\n', '*' * 50, '\n', error, '\n', '*' * 50
#sys.exit()
comm = os.path.join(os.getcwd(), 'xintface')
(output, error, retz) = shellcmd(comm)
if retz:
print '\n\nCOMMON: in set_mangling: cannot run xintface'
print 'error is:\n', '*' * 50, '\n', error, '\n', '*' * 50
#sys.exit()
self.mangling = output
delfiles(['xintface', 'tmpf.f', 'tmpf.o', 'tmpc.c', 'tmpc.o'])
print self.mangling
return 1
def set_download(self):
""" Figures out how to download files """
print 'Setting download command...'
wget = 0
urllib = 0
if urllib == 0:
# if urllib2 is not present checks if wget is present
# in the PATH and if yes it sets the download command
# to be wget
print "Checking availablility of wget...",
path = str(os.getenv('PATH')).split(os.pathsep)
for i in path:
if (os.path.isfile(os.path.join(i, 'wget'))):
print "available"
wget = 1
break
if wget:
# test wget
print "Testing wget...",
comm = 'wget --tries=2 --timeout=5 http://www.netlib.org/lapack/index'
(output, error, retz) = shellcmd(comm)
if (retz != 0):
print 'not working.'
wget = -1
else:
print "working"
self.downcmd = "wget"
os.remove("index")
return
else:
# wget not available
print "not available"
wget = 0
def cleanup(self):
" Cleans up the installer directory "
print "Cleaning up...",
sys.stdout.flush()
builddir = os.path.join(self.build)
os.chdir(builddir)
print "build dir is ", builddir
comm = 'make clean'
retz = subprocess.call(comm,
shell=True,
stdout=sys.stdout,
stderr=sys.stderr)
#(output, error, retz) = shellcmd(comm)
comm = 'rm -f ../script/*.pyc; rm -rf download'
(output, error, retz) = shellcmd(comm)
retz = subprocess.call(comm,
shell=True,
stdout=sys.stdout,
stderr=sys.stderr)
print "done."
def fc_is_xlf(self):
comm = self.config.fc
(output, error, retz) = shellcmd(comm)
isifort = string.find(output, 'xlf')
if isifort != -1:
return 1
return 0
def cc_is_intel(self):
comm = self.config.cc + ' -V'
(output, error, retz) = shellcmd(comm)
isifort = string.find(error, 'Intel(R) C')
if isifort != -1:
return 1
return 0
def cc_is_gnu(self):
comm = self.config.cc + ' -v'
(output, error, retz) = shellcmd(comm)
isifort = string.find(error, 'gcc')
if isifort != -1:
return 1
return 0
def cc_is_xlc(self):
comm = self.config.cc + ' -qversion'
(output, error, retz) = shellcmd(comm)
isifort = string.find(output, 'XL')
if isifort != -1:
return 1
return 0
def check_cc(self):
""" checking if cc works """
# simply generates a C program containing a couple of calls
# to MPI routines and checks if the compilation and execution
# are succesful
print 'Checking if cc works...',
sys.stdout.flush()
# generate
writefile(
'tmpc.c', """
#include <stdio.h>
int main(int argc, char **argv){
int iam;
fprintf(stdout, \"success\" );fflush(stdout);
return 0;
}\n""")
# compile
#ccomm = self.config.cc+" "+self.config.cflags+" "+self.config.ldflags_c+" -o tmpc "+os.path.join(os.getcwd(),"tmpc.c")
import re
cflags_ = re.sub(
'-O3', '-O0', self.config.cflags
) # -O3 would take too much time. Do not need optimization here....
ccomm = self.config.cc + " " + cflags_ + " -o tmpc " + os.path.join(
os.getcwd(), "tmpc.c")
(output, error, retz) = shellcmd(ccomm)
if retz:
print '\n\nCOMMON: C compiler not working! aborting...'
print 'stderr:\n', '*' * 50, '\n', error, '\n', '*' * 50
sys.exit()
# run
comm = './tmpc'
(output, error, retz) = shellcmd(comm)
if retz:
print '\n\nCOMMON: cc not working! aborting...'
print 'error is:\n', '*' * 50, '\n', error, '\n', '*' * 50
sys.exit()
# cleanup
delfiles(['tmpc.c', 'tmpc'])
print 'yes'
return 0
def Translate(ll_files, extra_args, obj, verbose, target):
"""Translate a set of input bitcode files into a single object file.
Use pnacl-llc to translate textual bitcode input ll_files into object file
obj, using extra_args as the architectural flags.
"""
externalize = ['-externalize']
shellcmd(['cat'] + ll_files + ['|',
'pnacl-llc',
'-function-sections',
'-O2',
'-filetype=obj',
'-bitcode-format=llvm',
'-o', obj
] + extra_args + externalize, echo=verbose)
localize_syms = ['nacl_tp_tdb_offset', 'nacl_tp_tls_offset']
shellcmd([GetObjcopyCmd(target), obj] +
[('--localize-symbol=' + sym) for sym in localize_syms])
def check_fc(self):
""" check if the Fortran compiler works """
# simply generates a F77 program and checks if the compilation and execution
# are succesful
print "Checking if the Fortran compiler works...",
sys.stdout.flush()
# generate
writefile(
"tmpf.f", """
program ftest
integer i
print*,'success'
stop
end\n""")
# compile
#fcomm = self.config.fc+' '+self.config.fcflags+" "+self.config.ldflags_fc+' -o tmpf '+'tmpf.f'
fcomm = self.config.fc + ' ' + self.config.fflags + ' -o tmpf ' + 'tmpf.f'
(output, error, retz) = shellcmd(fcomm)
if retz:
print '\n\nCOMMON: the Fortran compiler is not working! aborting...'
print 'error is:\n', '*' * 50, '\n', error, '\n', '*' * 50
sys.exit()
# run
comm = './tmpf'
(output, error, retz) = shellcmd(comm)
if retz:
print '\n\nCOMMON: the Fortran compiler is not working! aborting...'
print 'error is:\n', '*' * 50, '\n', error, '\n', '*' * 50
sys.exit()
# cleanup
delfiles(['tmpf.f', 'tmpf', 'tmpf.o'])
print 'yes'
return 0
def main():
"""Run the specified command only if conditions are met.
Two conditions are checked. First, the CONDITION must be true.
Secondly, all NEED names must be in the set of HAVE names.
If both conditions are met, the command defined by the remaining
arguments is run in a shell.
"""
argparser = argparse.ArgumentParser(
description=' ' + main.__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
argparser.add_argument('--cond',
choices={'true', 'false'},
required=False,
default='true',
metavar='CONDITION',
help='Condition to test.')
argparser.add_argument('--need',
required=False,
default=[],
action='append',
metavar='NEED',
help='Needed name. May be repeated.')
argparser.add_argument('--have',
required=False,
default=[],
action='append',
metavar='HAVE',
help='Name you have. May be repeated.')
argparser.add_argument('--echo-cmd',
required=False,
action='store_true',
help='Trace the command before running.')
argparser.add_argument('--command',
nargs=argparse.REMAINDER,
help='Command to run if attributes found.')
args = argparser.parse_args()
# Quit early if no command to run.
if not args.command:
raise RuntimeError("No command argument(s) specified for ifatts")
if args.cond == 'true' and set(args.need) <= set(args.have):
stdout_result = shellcmd(args.command, echo=args.echo_cmd)
if not args.echo_cmd:
sys.stdout.write(stdout_result)
def cc_is_pgi(self):
comm = self.config.cc + ' -V'
(output, error, retz) = shellcmd(comm)
isifort = string.find(error, 'pgicc')
if isifort != -1:
return 1
isifort = string.find(error, 'Portland')
if isifort != -1:
return 1
isifort = string.find(output, 'pgicc')
if isifort != -1:
return 1
isifort = string.find(output, 'Portland')
if isifort != -1:
return 1
return 0
def MakeNativeRuntime():
"""Builds just the native runtime."""
# Translate tempdir/szrt.ll and tempdir/szrt_ll.ll to
# szrt_native_{target}.tmp.o.
Translate(ll_files,
['-mtriple=' + target_info.triple] + target_info.llc_flags,
TmpFile('{dir}/szrt_native_{target}.tmp.o'),
verbose, target_info.target)
# Compile srcdir/szrt_profiler.c to
# tempdir/szrt_profiler_native_{target}.o.
shellcmd(['clang',
'-O2',
'-target=' + target_info.triple,
'-c',
'{srcdir}/szrt_profiler.c'.format(srcdir=srcdir),
'-o', TmpFile('{dir}/szrt_native_profiler_{target}.o')
], echo=verbose)
# Assemble srcdir/szrt_asm_{target}.s to tempdir/szrt_asm_{target}.o.
shellcmd(['llvm-mc',
'-triple=' + target_info.triple, '--defsym NATIVE=1',
'-filetype=obj',
'-o', TmpFile('{dir}/szrt_native_asm_{target}.o'),
'{srcdir}/szrt_asm_{target}.s'.format(
srcdir=srcdir, target=target_info.target)
], echo=verbose)
# Write full szrt_native_{target}.o.
PartialLink([TmpFile('{dir}/szrt_native_{target}.tmp.o'),
TmpFile('{dir}/szrt_native_asm_{target}.o'),
TmpFile('{dir}/szrt_native_profiler_{target}.o')],
['-m {ld_emu}'.format(ld_emu=target_info.ld_emu)],
OutFile('{rtdir}/szrt_native_{target}.o'),
verbose)
shellcmd([GetObjcopyCmd(target_info.target),
'--strip-symbol=NATIVE',
OutFile('{rtdir}/szrt_native_{target}.o')])
# Compile srcdir/szrt_asan.c to szrt_asan_{target}.o
shellcmd(['clang',
'-O2',
'-target=' + target_info.triple,
'-c',
'{srcdir}/szrt_asan.c'.format(srcdir=srcdir),
'-o', OutFile('{rtdir}/szrt_asan_{target}.o')
], echo=verbose)
def get_pid(process_name):
command = "adb shell ps | grep %s" % (process_name)
fp = shellcmd(command)
if fp == -1:
errMsg = "Don't get the PID of process %s, please make sure the given name is right\n" % (process_name)
sys.stderr.write(errMsg)
return -1
rs = fp.readlines()
if len(rs) != 1:
pids = []
for i in rs:
p = i.split()
if p[len(p) - 1] == process_name:
return int(p[1])
pids.append(i.split()[1])
errMsg = "Get more then one PID (%s), please given a more accurate process name\n" % (", ".join(pids))
sys.stderr.write(errMsg)
return -1
return int(rs[0].split()[1])
def get_so_maps(pid):
command = "adb shell cat /proc/%d/maps" % (pid)
fp = shellcmd(command)
if fp == -1:
print("error in get so maps")
return None
rs = fp.readlines()
so_maps = {}
for i in rs:
if i.find("/data/data/") == -1:
continue;
pos = i.find("-")
if (pos == -1):
continue
addr = int(i[0:pos], 16)
pos = i.rfind("/")
if (pos == -1):
continue
so_name = i[pos+1:].strip()
if not so_name in so_maps.keys():
so_maps[so_name] = addr
return so_maps
def main():
"""Run the specified command only if conditions are met.
Two conditions are checked. First, the CONDITION must be true.
Secondly, all NEED names must be in the set of HAVE names.
If both conditions are met, the command defined by the remaining
arguments is run in a shell.
"""
argparser = argparse.ArgumentParser(
description=' ' + main.__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
argparser.add_argument('--cond', choices={'true', 'false'} , required=False,
default='true', metavar='CONDITION',
help='Condition to test.')
argparser.add_argument('--need', required=False, default=[],
action='append', metavar='NEED',
help='Needed name. May be repeated.')
argparser.add_argument('--have', required=False, default=[],
action='append', metavar='HAVE',
help='Name you have. May be repeated.')
argparser.add_argument('--echo-cmd', required=False,
action='store_true',
help='Trace the command before running.')
argparser.add_argument('--command', nargs=argparse.REMAINDER,
help='Command to run if attributes found.')
args = argparser.parse_args()
# Quit early if no command to run.
if not args.command:
raise RuntimeError("No command argument(s) specified for ifatts")
if args.cond == 'true' and set(args.need) <= set(args.have):
stdout_result = shellcmd(args.command, echo=args.echo_cmd)
if not args.echo_cmd:
sys.stdout.write(stdout_result)
def main():
"""Builds a cross-test binary for comparing Subzero and llc translation.
Each --test argument is compiled once by llc and once by Subzero. C/C++
tests are first compiled down to PNaCl bitcode using pnacl-clang and
pnacl-opt. The --prefix argument ensures that symbol names are different
between the two object files, to avoid linking errors.
There is also a --driver argument that specifies the C/C++ file that calls
the test functions with a variety of interesting inputs and compares their
results.
"""
# arch_map maps a Subzero target string to TargetInfo (e.g., triple).
arch_map = {
'x8632': targets.X8632Target,
'x8664': targets.X8664Target,
'arm32': targets.ARM32Target,
'mips32': targets.MIPS32Target
}
arch_sz_flags = {
'x8632': [],
'x8664': [],
# For ARM, test a large stack offset as well. +/- 4095 is
# the limit, so test somewhere near that boundary.
'arm32': ['--test-stack-extra', '4084'],
'mips32': ['--test-stack-extra', '4084']
}
arch_llc_flags_extra = {
# Use sse2 instructions regardless of input -mattr
# argument to avoid differences in (undefined) behavior of
# converting NaN to int.
'x8632': ['-mattr=sse2'],
'x8664': ['-mattr=sse2'],
'arm32': [],
'mips32': [],
}
desc = 'Build a cross-test that compares Subzero and llc translation.'
argparser = argparse.ArgumentParser(description=desc)
argparser.add_argument('--test',
required=True,
action='append',
metavar='TESTFILE_LIST',
help='List of C/C++/.ll files with test functions')
argparser.add_argument('--driver',
required=True,
metavar='DRIVER',
help='Driver program')
argparser.add_argument('--target',
required=False,
default='x8632',
choices=arch_map.keys(),
metavar='TARGET',
help='Translation target architecture.' +
' Default %(default)s.')
argparser.add_argument('-O',
required=False,
default='2',
dest='optlevel',
choices=['m1', '-1', '0', '1', '2'],
metavar='OPTLEVEL',
help='Optimization level for llc and Subzero ' +
'(m1 and -1 are equivalent).' +
' Default %(default)s.')
argparser.add_argument('--clang-opt',
required=False,
default=True,
dest='clang_opt')
argparser.add_argument(
'--mattr',
required=False,
default='sse2',
dest='attr',
choices=['sse2', 'sse4.1', 'neon', 'hwdiv-arm', 'base'],
metavar='ATTRIBUTE',
help='Target attribute. Default %(default)s.')
argparser.add_argument('--sandbox',
required=False,
default=0,
type=int,
dest='sandbox',
help='Use sandboxing. Default "%(default)s".')
argparser.add_argument('--nonsfi',
required=False,
default=0,
type=int,
dest='nonsfi',
help='Use Non-SFI mode. Default "%(default)s".')
argparser.add_argument('--prefix',
required=True,
metavar='SZ_PREFIX',
help='String prepended to Subzero symbol names')
argparser.add_argument('--output',
'-o',
required=True,
metavar='EXECUTABLE',
help='Executable to produce')
argparser.add_argument('--dir',
required=False,
default='.',
metavar='OUTPUT_DIR',
help='Output directory for all files.' +
' Default "%(default)s".')
argparser.add_argument('--filetype',
default='obj',
dest='filetype',
choices=['obj', 'asm', 'iasm'],
help='Output file type. Default %(default)s.')
argparser.add_argument('--sz',
dest='sz_args',
action='append',
default=[],
help='Extra arguments to pass to pnacl-sz.')
args = argparser.parse_args()
nacl_root = FindBaseNaCl()
bindir = ('{root}/toolchain/linux_x86/pnacl_newlib_raw/bin'.format(
root=nacl_root))
target_info = arch_map[args.target]
triple = target_info.triple
if args.sandbox:
triple = targets.ConvertTripleToNaCl(triple)
llc_flags = target_info.llc_flags + arch_llc_flags_extra[args.target]
if args.nonsfi:
llc_flags.extend([
'-relocation-model=pic', '-malign-double', '-force-tls-non-pic',
'-mtls-use-call'
])
mypath = os.path.abspath(os.path.dirname(sys.argv[0]))
# Construct a "unique key" for each test so that tests can be run in
# parallel without race conditions on temporary file creation.
key = '{sb}.O{opt}.{attr}.{target}'.format(target=args.target,
sb=get_sfi_string(
args, 'sb', 'nonsfi',
'nat'),
opt=args.optlevel,
attr=args.attr)
objs = []
for arg in args.test:
base, ext = os.path.splitext(arg)
if ext == '.ll':
bitcode = arg
else:
# Use pnacl-clang and pnacl-opt to produce PNaCl bitcode.
bitcode_nonfinal = os.path.join(args.dir, base + '.' + key + '.bc')
bitcode = os.path.join(args.dir, base + '.' + key + '.pnacl.ll')
shellcmd([
'{bin}/pnacl-clang'.format(bin=bindir),
('-O2' if args.clang_opt else '-O0'),
('-DARM32' if args.target == 'arm32' else ''), '-c', arg,
('-DMIPS32' if args.target == 'mips32' else ''), '-o',
bitcode_nonfinal
])
shellcmd([
'{bin}/pnacl-opt'.format(bin=bindir),
'-pnacl-abi-simplify-preopt', '-pnacl-abi-simplify-postopt',
'-pnaclabi-allow-debug-metadata', '-strip-metadata',
'-strip-module-flags', '-strip-debug', bitcode_nonfinal, '-S',
'-o', bitcode
])
base_sz = '{base}.{key}'.format(base=base, key=key)
asm_sz = os.path.join(args.dir, base_sz + '.sz.s')
obj_sz = os.path.join(args.dir, base_sz + '.sz.o')
obj_llc = os.path.join(args.dir, base_sz + '.llc.o')
shellcmd([
'{path}/pnacl-sz'.format(path=os.path.dirname(mypath)),
] + args.sz_args + [
'-O' + args.optlevel, '-mattr=' + args.attr, '--target=' +
args.target, '--sandbox=' + str(args.sandbox), '--nonsfi=' +
str(args.nonsfi), '--prefix=' + args.prefix,
'-allow-uninitialized-globals', '-externalize', '-filetype=' +
args.filetype, '-o=' +
(obj_sz if args.filetype == 'obj' else asm_sz), bitcode
] + arch_sz_flags[args.target])
if args.filetype != 'obj':
shellcmd([
'{bin}/llvm-mc'.format(bin=bindir),
'-triple=' + ('mipsel-linux-gnu' if args.target == 'mips32'
and args.sandbox else triple), '-filetype=obj',
'-o=' + obj_sz, asm_sz
])
# Each separately translated Subzero object file contains its own
# definition of the __Sz_block_profile_info profiling symbol. Avoid
# linker errors (multiply defined symbol) by making all copies weak.
# (This could also be done by Subzero if it supported weak symbol
# definitions.) This approach should be OK because cross tests are
# currently the only situation where multiple translated files are
# linked into the executable, but when PNaCl supports shared nexe
# libraries, this would need to change. (Note: the same issue applies
# to the __Sz_revision symbol.)
shellcmd([
'{bin}/{objcopy}'.format(bin=bindir,
objcopy=GetObjcopyCmd(args.target)),
'--weaken-symbol=__Sz_block_profile_info',
'--weaken-symbol=__Sz_revision',
'--strip-symbol=nacl_tp_tdb_offset',
'--strip-symbol=nacl_tp_tls_offset', obj_sz
])
objs.append(obj_sz)
shellcmd([
'{bin}/pnacl-llc'.format(bin=bindir), '-mtriple=' + triple,
'-externalize', '-filetype=obj', '-bitcode-format=llvm', '-o=' +
obj_llc, bitcode
] + llc_flags)
strip_syms = [] if args.target == 'mips32' else [
'nacl_tp_tdb_offset', 'nacl_tp_tls_offset'
]
shellcmd([
'{bin}/{objcopy}'.format(
bin=bindir, objcopy=GetObjcopyCmd(args.target)), obj_llc
] + [('--strip-symbol=' + sym) for sym in strip_syms])
objs.append(obj_llc)
# Add szrt_sb_${target}.o or szrt_native_${target}.o.
if not args.nonsfi:
objs.append(('{root}/toolchain_build/src/subzero/build/runtime/' +
'szrt_{sb}_' + args.target + '.o').format(
root=nacl_root,
sb=get_sfi_string(args, 'sb', 'nonsfi', 'native')))
target_params = []
if args.target == 'arm32':
target_params.append('-DARM32')
target_params.append('-static')
if args.target == 'mips32':
target_params.append('-DMIPS32')
pure_c = os.path.splitext(args.driver)[1] == '.c'
if not args.nonsfi:
# Set compiler to clang, clang++, pnacl-clang, or pnacl-clang++.
compiler = '{bin}/{prefix}{cc}'.format(
bin=bindir,
prefix=get_sfi_string(args, 'pnacl-', '', ''),
cc='clang' if pure_c else 'clang++')
sb_native_args = ([
'-O0', '--pnacl-allow-native', '-arch', target_info.compiler_arch,
'-Wn,-defsym=__Sz_AbsoluteZero=0'
] if args.sandbox else [
'-g', '-target=' +
triple, '-lm', '-lpthread', '-Wl,--defsym=__Sz_AbsoluteZero=0'
] + target_info.cross_headers)
shellcmd([compiler] + target_params + [args.driver] + objs +
['-o', os.path.join(args.dir, args.output)] + sb_native_args)
return 0
base, ext = os.path.splitext(args.driver)
bitcode_nonfinal = os.path.join(args.dir, base + '.' + key + '.bc')
bitcode = os.path.join(args.dir, base + '.' + key + '.pnacl.ll')
asm_sz = os.path.join(args.dir, base + '.' + key + '.s')
obj_llc = os.path.join(args.dir, base + '.' + key + '.o')
compiler = '{bin}/{prefix}{cc}'.format(bin=bindir,
prefix='pnacl-',
cc='clang' if pure_c else 'clang++')
shellcmd([compiler] + target_params +
[args.driver, '-O2', '-o', bitcode_nonfinal, '-Wl,-r'])
shellcmd([
'{bin}/pnacl-opt'.format(bin=bindir), '-pnacl-abi-simplify-preopt',
'-pnacl-abi-simplify-postopt', '-pnaclabi-allow-debug-metadata',
'-strip-metadata', '-strip-module-flags', '-strip-debug',
'-disable-opt', bitcode_nonfinal, '-S', '-o', bitcode
])
shellcmd([
'{bin}/pnacl-llc'.format(bin=bindir), '-mtriple=' +
triple, '-externalize', '-filetype=obj', '-O2', '-bitcode-format=llvm',
'-o', obj_llc, bitcode
] + llc_flags)
if not args.sandbox and not args.nonsfi:
shellcmd([
'{bin}/{objcopy}'.format(bin=bindir,
objcopy=GetObjcopyCmd(args.target)),
'--redefine-sym', '_start=_user_start', obj_llc
])
objs.append(obj_llc)
if args.nonsfi:
LinkNonsfi(objs, os.path.join(args.dir, args.output), args.target)
elif args.sandbox:
LinkSandbox(objs, os.path.join(args.dir, args.output), args.target)
else:
LinkNative(objs, os.path.join(args.dir, args.output), args.target)
def run(is_cpp):
"""Passes its arguments directly to pnacl-clang.
If -fsanitize-address is specified, extra information is passed to
pnacl-clang to ensure that later instrumentation in pnacl-sz can be
performed. For example, clang automatically inlines many memory allocation
functions, so this script will redefine them at compile time to make sure
they can be correctly instrumented by pnacl-sz.
"""
pnacl_root = FindBaseNaCl()
dummy_subs = {
'calloc': {
'sig': ['void *', 'size_t', 'size_t'],
'sub': '__asan_dummy_calloc'
},
'_calloc': {
'sig': ['void *', 'size_t', 'size_t'],
'sub': '__asan_dummy_calloc'
}
}
subs_src = (
'{root}/toolchain_build/src/subzero/pydir/sz_clang_dummies.c').format(
root=pnacl_root)
clang = ('{root}/toolchain/linux_x86/pnacl_newlib_raw/bin/pnacl-clang{pp}'
).format(root=pnacl_root, pp='++' if is_cpp else '')
args = sys.argv
args[0] = clang
tmp_dir = ''
if '-fsanitize-address' in args:
args.remove('-fsanitize-address')
include_dirs = set()
tmp_dir = tempfile.mkdtemp()
for i, arg in enumerate(args[1:], 1):
if not os.path.isfile(arg):
continue
src = os.path.basename(arg)
ext = os.path.splitext(arg)[1]
if ext in ['.c', '.cc', '.cpp']:
include_dirs |= {os.path.dirname(arg)}
dest_name = os.path.join(tmp_dir, src)
with open(dest_name, 'w') as dest:
dest.write(subsToMacros(dummy_subs, arg))
with open(arg) as src:
for line in src:
dest.write(line)
args[i] = dest_name
# If linking (not single file compilation) then add dummy definitions
if not ('-o' in args and
('-c' in args or '-S' in args or '-E' in args)):
args.append(subs_src)
for d in include_dirs:
args.append('-iquote {d}'.format(d=d))
if '-fno-inline' not in args:
args.append('-fno-inline')
err_code = 0
try:
shellcmd(args, echo=True)
except subprocess.CalledProcessError as e:
print e.output
err_code = e.returncode
if tmp_dir != '':
shutil.rmtree(tmp_dir)
exit(err_code)
def main():
"""Build the Subzero runtime support library for all architectures.
"""
nacl_root = FindBaseNaCl()
argparser = argparse.ArgumentParser(
description=' ' + main.__doc__,
formatter_class=argparse.RawTextHelpFormatter)
argparser.add_argument('--verbose', '-v', dest='verbose',
action='store_true',
help='Display some extra debugging output')
argparser.add_argument('--pnacl-root', dest='pnacl_root',
default=(
'{root}/toolchain/linux_x86/pnacl_newlib_raw'
).format(root=nacl_root),
help='Path to PNaCl toolchain binaries.')
argparser.add_argument('--exclude-target', dest='excluded_targets',
default=[], action='append',
help='Target whose runtime should not be built')
args = argparser.parse_args()
os.environ['PATH'] = ('{root}/bin{sep}{path}'
).format(root=args.pnacl_root, sep=os.pathsep, path=os.environ['PATH'])
srcdir = (
'{root}/toolchain_build/src/subzero/runtime'
).format(root=nacl_root)
rtdir = (
'{root}/toolchain_build/src/subzero/build/runtime'
).format(root=nacl_root)
try:
tempdir = tempfile.mkdtemp()
if os.path.exists(rtdir) and not os.path.isdir(rtdir):
os.remove(rtdir)
if not os.path.exists(rtdir):
os.makedirs(rtdir)
# Compile srcdir/szrt.c to tempdir/szrt.ll
shellcmd(['pnacl-clang',
'-O2',
'-c',
'{srcdir}/szrt.c'.format(srcdir=srcdir),
'-o', '{dir}/szrt.tmp.bc'.format(dir=tempdir)
], echo=args.verbose)
shellcmd(['pnacl-opt',
'-pnacl-abi-simplify-preopt',
'-pnacl-abi-simplify-postopt',
'-pnaclabi-allow-debug-metadata',
'{dir}/szrt.tmp.bc'.format(dir=tempdir),
'-S',
'-o', '{dir}/szrt.ll'.format(dir=tempdir)
], echo=args.verbose)
ll_files = ['{dir}/szrt.ll'.format(dir=tempdir),
'{srcdir}/szrt_ll.ll'.format(srcdir=srcdir)]
MakeRuntimesForTarget(targets.X8632Target, ll_files,
srcdir, tempdir, rtdir, args.verbose,
args.excluded_targets)
MakeRuntimesForTarget(targets.X8664Target, ll_files,
srcdir, tempdir, rtdir, args.verbose,
args.excluded_targets)
MakeRuntimesForTarget(targets.ARM32Target, ll_files,
srcdir, tempdir, rtdir, args.verbose,
args.excluded_targets)
MakeRuntimesForTarget(targets.MIPS32Target, ll_files,
srcdir, tempdir, rtdir, args.verbose,
args.excluded_targets)
finally:
try:
shutil.rmtree(tempdir)
except OSError as exc:
if exc.errno != errno.ENOENT: # ENOENT - no such file or directory
raise # re-raise exception
def PartialLink(obj_files, extra_args, lib, verbose):
"""Partially links a set of obj files into a final obj library."""
shellcmd(['le32-nacl-ld',
'-o', lib,
'-r',
] + extra_args + obj_files, echo=verbose)
def check_gsl(self):
""" This function simply generates a C program
that contains few GSL calls routine and then
checks if compilation, linking and execution are succesful"""
sys.stdout.flush()
code = """#include <gsl/gsl_rng.h>
const gsl_rng_type * T = gsl_rng_default;
gsl_rng * r = gsl_rng_alloc (T);
int main(void)
{
gsl_rng_env_setup();
gsl_rng_set (r,1);
return 0;
}
"""
writefile('tmpc.cc', code)
if self.config.gsl == "": # just trying default == -lgsl
self.config.gsl = '-lgsl'
self.config.gslinc = includefromlib(self.config.gsl)
ccomm = self.config.cxx + ' ' + self.config.gslinc + ' -o tmpc ' + 'tmpc.cc ' + self.config.gsl
print 'checking with:', ccomm
(output, error, retz) = shellcmd(ccomm)
print "Checking if provided GSL works...",
if (retz != 0):
if self.dmft.verbose:
print '\n\nlibgsl: provided GSL cannot be used! aborting...'
print 'error is:\n', '*' * 50, '\n', ccomm, '\n', error, '\n', '*' * 50
return -1
else:
print "no"
return -1
#sys.exit()
comm = './tmpc'
(output, error, retz) = shellcmd(comm)
if (retz != 0):
if self.dmft.verbose:
print '\n\nlibgsl: provided GSL cannot be used! aborting...'
print 'error is:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50
print retz
return -1
else:
print "no"
return -1
# sys.exit()
if self.config.gslinc == '':
# It worked, but we do not know the include files, hence figuring it out
ccomm = self.config.cc + ' -E tmpc.cc |grep gsl | grep include'
(output, error, retz) = shellcmd(ccomm)
#print 'output=', output
# compiler output in lines
lines = output.split('\n')
incl = {}
for i, line in enumerate(lines):
dat = line.split()
for d in dat:
m = re.search('gsl', d) # take out the directory
if m is not None:
incl[os.path.dirname(d[1:-1])] = True
for inc in incl.keys():
self.config.gslinc += ' -I' + inc[:
-4] # path has extra gsl. Take it out
delfiles(['tmpc.cc', 'tmpc'])
print 'yes'
return 0
return os.path.expandvars('$LLVM_BIN_PATH')
else:
return ''
if __name__ == '__main__':
argparser = argparse.ArgumentParser()
argparser.add_argument('cfile', nargs='+', type=str)
argparser.add_argument('--llvmbin', nargs='?', type=str)
args = argparser.parse_args()
if args.llvmbin:
llvm_bin_path = args.llvmbin
else:
llvm_bin_path = find_llvm_bin()
print('LLVM binary path set to: "%s"' % llvm_bin_path)
def toolpath(toolname):
return os.path.join(llvm_bin_path, toolname)
tempdir = tempfile.mkdtemp()
for cname in args.cfile:
basename = os.path.splitext(cname)[0]
llname = os.path.join(tempdir, basename + '.ll')
optllname = basename + '-opt.ll'
shellcmd(toolpath('clang') + ' -cc1 -O3 -emit-llvm {0} -o {1}'.format(
cname, llname))
shellcmd(toolpath('opt') + ' -O3 -S {0} > {1}'.format(
llname, optllname))
def main():
"""Build native gcc-style executables for one or all Spec2K components.
Afterwards, the executables can be run from the
native_client/tests/spec2k/ directory as:
'./run_all.sh RunBenchmarks SetupGccX8632Opt {train|ref} ...'
-- or --
'./run_all.sh RunBenchmarks SetupPnaclX8632Opt {train|ref} ...'
-- or --
'./run_all.sh RunBenchmarks SetupNonsfiX8632Opt {train|ref} ...'
"""
nacl_root = FindBaseNaCl()
# Use the same default ordering as spec2k/run_all.sh.
components = [ '177.mesa', '179.art', '183.equake', '188.ammp', '164.gzip',
'175.vpr', '176.gcc', '181.mcf', '186.crafty', '197.parser',
'253.perlbmk', '254.gap', '255.vortex', '256.bzip2',
'300.twolf', '252.eon' ]
argparser = argparse.ArgumentParser(description=main.__doc__)
szbuild.AddOptionalArgs(argparser)
argparser.add_argument('--run', dest='run', action='store_true',
help='Run after building')
argparser.add_argument('comps', nargs='*', default=components)
args = argparser.parse_args()
bad = set(args.comps) - set(components)
if bad:
print 'Unknown component{s}: '.format(s='s' if len(bad) > 1 else '') + \
' '.join(x for x in bad)
sys.exit(1)
# Fix up Subzero target strings for the run_all.sh script.
target_map = {
'arm32':'arm',
'x8632':'x8632',
'x8664':'x8664'
}
run_all_target = target_map[args.target] # fail if target not listed above
suffix = (
'pnacl.opt.{target}' if args.sandbox else
'nonsfi.opt.{target}' if args.nonsfi else
'gcc.opt.{target}').format(
target=run_all_target);
for comp in args.comps:
name = os.path.splitext(comp)[1] or comp
if name[0] == '.':
name = name[1:]
szbuild.ProcessPexe(args,
('{root}/tests/spec2k/{comp}/' +
'{name}.opt.stripped.pexe'
).format(root=nacl_root, comp=comp, name=name),
('{root}/tests/spec2k/{comp}/' +
'{name}.{suffix}'
).format(root=nacl_root, comp=comp, name=name,
suffix=suffix))
if args.run:
os.chdir('{root}/tests/spec2k'.format(root=FindBaseNaCl()))
setup = 'Setup' + ('Pnacl' if args.sandbox else
'Nonsfi' if args.nonsfi else
'Gcc') + {
'arm32': 'Arm',
'x8632': 'X8632',
'x8664': 'X8664'}[args.target] + 'Opt'
shellcmd(['./run_all.sh',
'RunTimedBenchmarks',
setup,
'train'] + args.comps)
def check_fftw(self):
""" This function simply generates a C program
that contains few FFTW calls routine and then
checks if compilation, linking and execution are succesful"""
sys.stdout.flush()
code = """
#include<fftw3.h>
#include<complex.h>
int main(void)
{
int N;
int i;
N=1;
fftw_complex *in, *out;
fftw_plan my_plan;
in = (fftw_complex*) fftw_malloc(sizeof(fftw_complex)*N);
out = (fftw_complex*) fftw_malloc(sizeof(fftw_complex)*N);
my_plan = fftw_plan_dft_1d(N, in, out, FFTW_FORWARD, FFTW_ESTIMATE);
fftw_execute(my_plan);
fftw_destroy_plan(my_plan);
fftw_free(in);
fftw_free(out);
return 0;
}
"""
writefile('tmpc.c', code)
if self.config.fftwlib == "":
self.config.fftwlib = '-lfftw3' # just trying default"
self.config.fftwinc = includefromlib(self.config.fftwlib)
ccomm = self.config.cc + ' ' + self.config.fftwinc + ' -o tmpc tmpc.c ' + self.config.fftwlib
print 'checking with:', ccomm
(output, error, retz) = shellcmd(ccomm)
print "Checking if provided FFTW works...",
if (retz != 0):
if self.dmft.verbose:
print '\n\nlibfftw: provided FFTW cannot be used! aborting...'
print 'error is:\n', '*' * 50, '\n', ccomm, '\n', error, '\n', '*' * 50
else:
print "no"
return 1
comm = './tmpc'
(output, error, retz) = shellcmd(comm)
if (retz != 0):
if self.dmft.verbose:
print '\n\nlibfftw: provided FFTW cannot be used! aborting...'
print 'error is:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50
print rez
else:
print "no"
return 1
if self.config.fftwinc == '':
# No include file given, hence checking which includes were used when compilation succeded
ccomm = self.config.cc + ' -E ' + self.config.fftwinc + ' tmpc.c |grep fftw | grep include'
(output, error, retz) = shellcmd(ccomm)
# compiler output in lines
lines = output.split('\n')
incl = {}
for i, line in enumerate(lines):
dat = line.split()
for d in dat:
m = re.search('fftw', d) # take out the directory
if m is not None:
incl[os.path.dirname(
d[1:-1]
)] = True # path has extra "xxx" so take them out
for inc in incl.keys():
self.config.fftwinc += ' -I' + inc
delfiles(['tmpc.c', 'tmpc'])
print 'yes'
return 0
if args.nacl_sdk_root:
nacl_sdk_root = os.path.expanduser(args.nacl_sdk_root)
if not nacl_sdk_root or not os.path.exists(nacl_sdk_root):
print '''\
Please set the NACL_SDK_ROOT environment variable or pass the path through
--nacl_sdk_root to point to a valid Native Client SDK installation.'''
sys.exit(1)
includes_path = os.path.join(nacl_sdk_root, 'include')
toolchain_path = os.path.join(nacl_sdk_root, 'toolchain', 'linux_pnacl')
clang_path = os.path.join(toolchain_path, 'bin64', 'pnacl-clang')
opt_path = os.path.join(toolchain_path, 'host_x86_64', 'bin', 'opt')
tempdir = tempfile.mkdtemp()
for cname in args.cfile:
basename = os.path.splitext(cname)[0]
llname = os.path.join(tempdir, basename + '.ll')
pnaclname = basename + '.pnacl.ll'
pnaclname = os.path.join(args.dir, pnaclname)
shellcmd(clang_path + ' -I{0} -c {1} -o {2}'.format(
includes_path, cname, llname))
shellcmd(opt_path +
' -O2 -pnacl-abi-simplify-preopt -pnacl-abi-simplify-postopt' +
('' if args.disable_verify else
' -verify-pnaclabi-module -verify-pnaclabi-functions') +
' -pnaclabi-allow-debug-metadata -disable-simplify-libcalls'
' {0} -S -o {1}'.format(llname, pnaclname))
args = argparser.parse_args()
nacl_root = FindBaseNaCl()
# Prepend bin to $PATH.
os.environ['PATH'] = (nacl_root +
'/toolchain/linux_x86/pnacl_newlib_raw/bin' +
os.pathsep + os.pathsep + os.environ['PATH'])
try:
tempdir = tempfile.mkdtemp()
for cname in args.cfile:
basename = os.path.splitext(cname)[0]
llname = os.path.join(tempdir, basename + '.ll')
pnaclname = basename + '.pnacl.ll'
pnaclname = os.path.join(args.dir, pnaclname)
shellcmd('pnacl-clang -O2 -c {0} -o {1}'.format(cname, llname))
shellcmd('pnacl-opt ' +
'-pnacl-abi-simplify-preopt -pnacl-abi-simplify-postopt' +
('' if args.disable_verify else
' -verify-pnaclabi-module -verify-pnaclabi-functions') +
' -pnaclabi-allow-debug-metadata'
' {0} -S -o {1}'.format(llname, pnaclname))
finally:
try:
shutil.rmtree(tempdir)
except OSError as exc:
if exc.errno != errno.ENOENT: # ENOENT - no such file or directory
raise # re-raise exception
def main():
"""Builds a cross-test binary for comparing Subzero and llc translation.
Each --test argument is compiled once by llc and once by Subzero. C/C++
tests are first compiled down to PNaCl bitcode using pnacl-clang and
pnacl-opt. The --prefix argument ensures that symbol names are different
between the two object files, to avoid linking errors.
There is also a --driver argument that specifies the C/C++ file that calls
the test functions with a variety of interesting inputs and compares their
results.
"""
# arch_map maps a Subzero target string to TargetInfo (e.g., triple).
arch_map = { 'x8632': targets.X8632Target,
'x8664': targets.X8664Target,
'arm32': targets.ARM32Target,
'mips32': targets.MIPS32Target}
arch_sz_flags = { 'x8632': [],
'x8664': [],
# For ARM, test a large stack offset as well. +/- 4095 is
# the limit, so test somewhere near that boundary.
'arm32': ['--test-stack-extra', '4084'],
'mips32': ['--test-stack-extra', '4084']
}
arch_llc_flags_extra = {
# Use sse2 instructions regardless of input -mattr
# argument to avoid differences in (undefined) behavior of
# converting NaN to int.
'x8632': ['-mattr=sse2'],
'x8664': ['-mattr=sse2'],
'arm32': [],
'mips32':[],
}
desc = 'Build a cross-test that compares Subzero and llc translation.'
argparser = argparse.ArgumentParser(description=desc)
argparser.add_argument('--test', required=True, action='append',
metavar='TESTFILE_LIST',
help='List of C/C++/.ll files with test functions')
argparser.add_argument('--driver', required=True,
metavar='DRIVER',
help='Driver program')
argparser.add_argument('--target', required=False, default='x8632',
choices=arch_map.keys(),
metavar='TARGET',
help='Translation target architecture.' +
' Default %(default)s.')
argparser.add_argument('-O', required=False, default='2', dest='optlevel',
choices=['m1', '-1', '0', '1', '2'],
metavar='OPTLEVEL',
help='Optimization level for llc and Subzero ' +
'(m1 and -1 are equivalent).' +
' Default %(default)s.')
argparser.add_argument('--clang-opt', required=False, default=True,
dest='clang_opt')
argparser.add_argument('--mattr', required=False, default='sse2',
dest='attr', choices=['sse2', 'sse4.1',
'neon', 'hwdiv-arm',
'base'],
metavar='ATTRIBUTE',
help='Target attribute. Default %(default)s.')
argparser.add_argument('--sandbox', required=False, default=0, type=int,
dest='sandbox',
help='Use sandboxing. Default "%(default)s".')
argparser.add_argument('--nonsfi', required=False, default=0, type=int,
dest='nonsfi',
help='Use Non-SFI mode. Default "%(default)s".')
argparser.add_argument('--prefix', required=True,
metavar='SZ_PREFIX',
help='String prepended to Subzero symbol names')
argparser.add_argument('--output', '-o', required=True,
metavar='EXECUTABLE',
help='Executable to produce')
argparser.add_argument('--dir', required=False, default='.',
metavar='OUTPUT_DIR',
help='Output directory for all files.' +
' Default "%(default)s".')
argparser.add_argument('--filetype', default='obj', dest='filetype',
choices=['obj', 'asm', 'iasm'],
help='Output file type. Default %(default)s.')
argparser.add_argument('--sz', dest='sz_args', action='append', default=[],
help='Extra arguments to pass to pnacl-sz.')
args = argparser.parse_args()
nacl_root = FindBaseNaCl()
bindir = ('{root}/toolchain/linux_x86/pnacl_newlib_raw/bin'
.format(root=nacl_root))
target_info = arch_map[args.target]
triple = target_info.triple
if args.sandbox:
triple = targets.ConvertTripleToNaCl(triple)
llc_flags = target_info.llc_flags + arch_llc_flags_extra[args.target]
if args.nonsfi:
llc_flags.extend(['-relocation-model=pic',
'-malign-double',
'-force-tls-non-pic',
'-mtls-use-call'])
mypath = os.path.abspath(os.path.dirname(sys.argv[0]))
# Construct a "unique key" for each test so that tests can be run in
# parallel without race conditions on temporary file creation.
key = '{sb}.O{opt}.{attr}.{target}'.format(
target=args.target,
sb=get_sfi_string(args, 'sb', 'nonsfi', 'nat'),
opt=args.optlevel, attr=args.attr)
objs = []
for arg in args.test:
base, ext = os.path.splitext(arg)
if ext == '.ll':
bitcode = arg
else:
# Use pnacl-clang and pnacl-opt to produce PNaCl bitcode.
bitcode_nonfinal = os.path.join(args.dir, base + '.' + key + '.bc')
bitcode = os.path.join(args.dir, base + '.' + key + '.pnacl.ll')
shellcmd(['{bin}/pnacl-clang'.format(bin=bindir),
('-O2' if args.clang_opt else '-O0'),
('-DARM32' if args.target == 'arm32' else ''), '-c', arg,
('-DMIPS32' if args.target == 'mips32' else ''),
'-o', bitcode_nonfinal])
shellcmd(['{bin}/pnacl-opt'.format(bin=bindir),
'-pnacl-abi-simplify-preopt',
'-pnacl-abi-simplify-postopt',
'-pnaclabi-allow-debug-metadata',
'-strip-metadata',
'-strip-module-flags',
'-strip-debug',
bitcode_nonfinal, '-S', '-o', bitcode])
base_sz = '{base}.{key}'.format(base=base, key=key)
asm_sz = os.path.join(args.dir, base_sz + '.sz.s')
obj_sz = os.path.join(args.dir, base_sz + '.sz.o')
obj_llc = os.path.join(args.dir, base_sz + '.llc.o')
shellcmd(['{path}/pnacl-sz'.format(path=os.path.dirname(mypath)),
] + args.sz_args + [
'-O' + args.optlevel,
'-mattr=' + args.attr,
'--target=' + args.target,
'--sandbox=' + str(args.sandbox),
'--nonsfi=' + str(args.nonsfi),
'--prefix=' + args.prefix,
'-allow-uninitialized-globals',
'-externalize',
'-filetype=' + args.filetype,
'-o=' + (obj_sz if args.filetype == 'obj' else asm_sz),
bitcode] + arch_sz_flags[args.target])
if args.filetype != 'obj':
shellcmd(['{bin}/llvm-mc'.format(bin=bindir),
'-triple=' + ('mipsel-linux-gnu'
if args.target == 'mips32' and args.sandbox
else triple),
'-filetype=obj',
'-o=' + obj_sz,
asm_sz])
# Each separately translated Subzero object file contains its own
# definition of the __Sz_block_profile_info profiling symbol. Avoid
# linker errors (multiply defined symbol) by making all copies weak.
# (This could also be done by Subzero if it supported weak symbol
# definitions.) This approach should be OK because cross tests are
# currently the only situation where multiple translated files are
# linked into the executable, but when PNaCl supports shared nexe
# libraries, this would need to change. (Note: the same issue applies
# to the __Sz_revision symbol.)
shellcmd(['{bin}/{objcopy}'.format(bin=bindir,
objcopy=GetObjcopyCmd(args.target)),
'--weaken-symbol=__Sz_block_profile_info',
'--weaken-symbol=__Sz_revision',
'--strip-symbol=nacl_tp_tdb_offset',
'--strip-symbol=nacl_tp_tls_offset',
obj_sz])
objs.append(obj_sz)
shellcmd(['{bin}/pnacl-llc'.format(bin=bindir),
'-mtriple=' + triple,
'-externalize',
'-filetype=obj',
'-bitcode-format=llvm',
'-o=' + obj_llc,
bitcode] + llc_flags)
strip_syms = [] if args.target == 'mips32' else ['nacl_tp_tdb_offset',
'nacl_tp_tls_offset']
shellcmd(['{bin}/{objcopy}'.format(bin=bindir,
objcopy=GetObjcopyCmd(args.target)),
obj_llc] +
[('--strip-symbol=' + sym) for sym in strip_syms])
objs.append(obj_llc)
# Add szrt_sb_${target}.o or szrt_native_${target}.o.
if not args.nonsfi:
objs.append((
'{root}/toolchain_build/src/subzero/build/runtime/' +
'szrt_{sb}_' + args.target + '.o'
).format(root=nacl_root,
sb=get_sfi_string(args, 'sb', 'nonsfi', 'native')))
target_params = []
if args.target == 'arm32':
target_params.append('-DARM32')
target_params.append('-static')
if args.target == 'mips32':
target_params.append('-DMIPS32')
pure_c = os.path.splitext(args.driver)[1] == '.c'
if not args.nonsfi:
# Set compiler to clang, clang++, pnacl-clang, or pnacl-clang++.
compiler = '{bin}/{prefix}{cc}'.format(
bin=bindir, prefix=get_sfi_string(args, 'pnacl-', '', ''),
cc='clang' if pure_c else 'clang++')
sb_native_args = (['-O0', '--pnacl-allow-native',
'-arch', target_info.compiler_arch,
'-Wn,-defsym=__Sz_AbsoluteZero=0']
if args.sandbox else
['-g', '-target=' + triple,
'-lm', '-lpthread',
'-Wl,--defsym=__Sz_AbsoluteZero=0'] +
target_info.cross_headers)
shellcmd([compiler] + target_params + [args.driver] + objs +
['-o', os.path.join(args.dir, args.output)] + sb_native_args)
return 0
base, ext = os.path.splitext(args.driver)
bitcode_nonfinal = os.path.join(args.dir, base + '.' + key + '.bc')
bitcode = os.path.join(args.dir, base + '.' + key + '.pnacl.ll')
asm_sz = os.path.join(args.dir, base + '.' + key + '.s')
obj_llc = os.path.join(args.dir, base + '.' + key + '.o')
compiler = '{bin}/{prefix}{cc}'.format(
bin=bindir, prefix='pnacl-',
cc='clang' if pure_c else 'clang++')
shellcmd([compiler] + target_params + [
args.driver,
'-O2',
'-o', bitcode_nonfinal,
'-Wl,-r'
])
shellcmd(['{bin}/pnacl-opt'.format(bin=bindir),
'-pnacl-abi-simplify-preopt',
'-pnacl-abi-simplify-postopt',
'-pnaclabi-allow-debug-metadata',
'-strip-metadata',
'-strip-module-flags',
'-strip-debug',
'-disable-opt',
bitcode_nonfinal, '-S', '-o', bitcode])
shellcmd(['{bin}/pnacl-llc'.format(bin=bindir),
'-mtriple=' + triple,
'-externalize',
'-filetype=obj',
'-O2',
'-bitcode-format=llvm',
'-o', obj_llc,
bitcode] + llc_flags)
if not args.sandbox and not args.nonsfi:
shellcmd(['{bin}/{objcopy}'.format(bin=bindir,
objcopy=GetObjcopyCmd(args.target)),
'--redefine-sym', '_start=_user_start',
obj_llc
])
objs.append(obj_llc)
if args.nonsfi:
LinkNonsfi(objs, os.path.join(args.dir, args.output), args.target)
elif args.sandbox:
LinkSandbox(objs, os.path.join(args.dir, args.output), args.target)
else:
LinkNative(objs, os.path.join(args.dir, args.output), args.target)
args = argparser.parse_args()
nacl_root = FindBaseNaCl()
# Prepend bin to $PATH.
os.environ['PATH'] = (
nacl_root + '/toolchain/linux_x86/pnacl_newlib_raw/bin' + os.pathsep +
os.pathsep + os.environ['PATH'])
try:
tempdir = tempfile.mkdtemp()
for cname in args.cfile:
basename = os.path.splitext(cname)[0]
llname = os.path.join(tempdir, basename + '.ll')
pnaclname = basename + '.pnacl.ll'
pnaclname = os.path.join(args.dir, pnaclname)
shellcmd('pnacl-clang -O2 -c {0} -o {1}'.format(cname, llname))
shellcmd('pnacl-opt ' +
'-pnacl-abi-simplify-preopt -pnacl-abi-simplify-postopt' +
('' if args.disable_verify else
' -verify-pnaclabi-module -verify-pnaclabi-functions') +
' -pnaclabi-allow-debug-metadata'
' {0} -S -o {1}'.format(llname, pnaclname))
finally:
try:
shutil.rmtree(tempdir)
except OSError as exc:
if exc.errno != errno.ENOENT: # ENOENT - no such file or directory
raise # re-raise exception
metavar='PATH',
help='Path to LLVM executables like llc ' +
'(defaults to $LLVM_BIN_PATH)')
args = argparser.parse_args()
objs = []
remove_internal = re.compile('^define internal ')
fix_target = re.compile('le32-unknown-nacl')
llvm_bin_path = args.llvm_bin_path
for arg in args.test:
base, ext = os.path.splitext(arg)
if ext == '.ll':
bitcode = arg
else:
bitcode = os.path.join(args.dir, base + '.pnacl.ll')
shellcmd(['../pydir/build-pnacl-ir.py', '--disable-verify',
'--dir', args.dir, arg])
# Read in the bitcode file, fix it up, and rewrite the file.
f = open(bitcode)
ll_lines = f.readlines()
f.close()
f = open(bitcode, 'w')
for line in ll_lines:
line = remove_internal.sub('define ', line)
line = fix_target.sub('i686-pc-linux-gnu', line)
f.write(line)
f.close()
asm_sz = os.path.join(args.dir, base + '.sz.s')
obj_sz = os.path.join(args.dir, base + '.sz.o')
obj_llc = os.path.join(args.dir, base + '.llc.o')
shellcmd(['../llvm2ice',
def down_install_gsl(self):
print "The GSL library is being installed."
sys.stdout.flush()
savecwd = os.getcwd()
# creating the build,lib and log dirs if don't exist
if not os.path.isdir(os.path.join(self.config.prefix, 'gsl')):
os.mkdir(os.path.join(self.config.prefix, 'gsl'))
if not os.path.isdir(os.path.join(os.getcwd(), 'log')):
os.mkdir(os.path.join(os.getcwd(), 'log'))
# Check if gsl.tgz is already present in the working dir
# otherwise download it
if not os.path.isfile(
os.path.join(os.getcwd(), getURLName(self.gslurl))):
print "Downloading GSL ...",
#downloader(self.lapackurl,self.downcmd)
#urllib.urlretrieve(self.gslurl, "gsl.tgz")
geturl(self.gslurl, "gsl.tgz")
print "done"
# unzip and untar
os.chdir('download')
print 'Unzip and untar GSL...',
comm = 'tar zxf gsl.tgz '
(output, error, retz) = shellcmd(comm)
if retz:
print '\n\nlibgsl: cannot unzip ' + self.gslversion + '.tgz'
print 'stderr:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50
sys.exit()
print 'done'
# change to GSL dir
os.chdir(os.path.join(os.getcwd(), self.gslversion))
# compile and generate library
print 'Configure GSL...',
sys.stdout.flush()
comm = './configure CC=' + self.config.cc + ' --prefix=' + os.path.join(
self.config.prefix, 'gsl')
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nlingsl: cannot configure GSL"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
log = output + error
# write log on a file
log = log + output + error
fulllog = os.path.join(savecwd, 'log/log.gsl')
writefile(fulllog, log)
print 'Configuration of GSL successful.'
print '(log is in ', fulllog, ')'
# compile and generate library
print 'Compile and generate GSL...',
sys.stdout.flush()
comm = self.make + ' -j4; ' + self.make + ' install'
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nlingsl: cannot compile GSL"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
log = output + error
# write the log on a file
log = log + output + error
fulllog = os.path.join(savecwd, 'log/log.gsl')
writefile(fulllog, log)
print 'Installation of GSL successful.'
print '(log is in ', fulllog, ')'
# move libcblas.a to the lib directory
#shutil.copy('libtmg.a',os.path.join(self.config.prefix,'gsl/libtmg.a'))
# set framework variables to point to the freshly installed GSL library
self.config.gsl = '-L' + os.path.join(self.config.prefix,
'gsl') + ' -lgsl '
os.chdir(savecwd)
# Check if the installation is successful
self.dmft.verbose = 1
# self.check_gsl()
self.dmft.verbose = 0
def main():
"""Build native gcc-style executables for one or all Spec2K components.
Afterwards, the executables can be run from the
native_client/tests/spec2k/ directory as:
'./run_all.sh RunBenchmarks SetupGccX8632Opt {train|ref} ...'
-- or --
'./run_all.sh RunBenchmarks SetupPnaclX8632Opt {train|ref} ...'
-- or --
'./run_all.sh RunBenchmarks SetupNonsfiX8632Opt {train|ref} ...'
"""
nacl_root = FindBaseNaCl()
# Use the same default ordering as spec2k/run_all.sh.
components = [
'177.mesa', '179.art', '183.equake', '188.ammp', '164.gzip', '175.vpr',
'176.gcc', '181.mcf', '186.crafty', '197.parser', '253.perlbmk',
'254.gap', '255.vortex', '256.bzip2', '300.twolf', '252.eon'
]
argparser = argparse.ArgumentParser(description=main.__doc__)
szbuild.AddOptionalArgs(argparser)
argparser.add_argument('--run',
dest='run',
action='store_true',
help='Run after building')
argparser.add_argument('comps', nargs='*', default=components)
args = argparser.parse_args()
bad = set(args.comps) - set(components)
if bad:
print 'Unknown component{s}: '.format(s='s' if len(bad) > 1 else '') + \
' '.join(x for x in bad)
sys.exit(1)
# Fix up Subzero target strings for the run_all.sh script.
target_map = {'arm32': 'arm', 'x8632': 'x8632', 'x8664': 'x8664'}
run_all_target = target_map[args.target] # fail if target not listed above
suffix = ('pnacl.opt.{target}' if args.sandbox else 'nonsfi.opt.{target}'
if args.nonsfi else 'gcc.opt.{target}').format(
target=run_all_target)
for comp in args.comps:
name = os.path.splitext(comp)[1] or comp
if name[0] == '.':
name = name[1:]
szbuild.ProcessPexe(args,
('{root}/tests/spec2k/{comp}/' +
'{name}.opt.stripped.pexe').format(root=nacl_root,
comp=comp,
name=name),
('{root}/tests/spec2k/{comp}/' +
'{name}.{suffix}').format(root=nacl_root,
comp=comp,
name=name,
suffix=suffix))
if args.run:
os.chdir('{root}/tests/spec2k'.format(root=FindBaseNaCl()))
setup = 'Setup' + ('Pnacl' if args.sandbox else
'Nonsfi' if args.nonsfi else 'Gcc') + {
'arm32': 'Arm',
'x8632': 'X8632',
'x8664': 'X8664'
}[args.target] + 'Opt'
shellcmd(['./run_all.sh', 'RunTimedBenchmarks', setup, 'train'] +
args.comps)
def check_linking(self):
""" Check if C main can be linked to Fortran subroutine """
# This one checks if the linking command works out of the box or
# if any specific flag is required. For example if the linker if the
# Intel FORTRAN compiler, then the "-nofor_main" is usually required.
# This function only checks if linker works but does not automatically
# detect the required flags
print 'Checking loader...',
sys.stdout.flush()
writefile(
'tmpf.f', """
subroutine fsub()
write(*,*)'success'
stop
end\n""")
writefile(
'tmpc.c', """
#if defined ADD_
#define fsub fsub_
#elif defined NOCHANGE
#define fsub fsub
#elif defined fcIsF2C
#define fsub fsub_
#elif defined UPCASE
#define fsub FSUB
#endif
void main(){
fsub();}\n""")
#ccomm = self.config.cc+' '+self.config.ccflags+' '+self.mangling+' -c -o tmpc.o tmpc.c'
#fcomm = self.config.fc+' '+self.config.fcflags+' -c -o tmpf.o tmpf.f'
#lcomm = self.config.fc+' '+self.config.ldflags_fc+' '+self.config.ld_fcmain+' -o lnk tmpf.o tmpc.o'
ccomm = self.config.cc + ' ' + self.config.cflags + ' -c -o tmpc.o tmpc.c'
fcomm = self.config.fc + ' ' + self.config.fflags + ' -c -o tmpf.o tmpf.f'
lcomm = self.config.fc + ' -o lnk tmpf.o tmpc.o'
(output, error, retz) = shellcmd(ccomm)
if retz:
print '\n\nCOMMON: in check_linking: cannot compile'
print 'command is: ', ccomm
print 'error is:\n', '*' * 50, '\n', error, '\n', '*' * 50
#sys.exit()
(output, error, retz) = shellcmd(fcomm)
if retz:
print '\n\nCOMMON: in check_linking: cannot compile'
print 'command is: ', fcomm
print 'error is:\n', '*' * 50, '\n', error, '\n', '*' * 50
#sys.exit()
(output, error, retz) = shellcmd(lcomm)
if retz:
print """\n\nCOMMON: in check_linking: cannot link
Cannot link a C main program to a Fortran77 subroutine
Make sure that the appropriate flags are passed to the linker."""
print 'command is: ', lcomm
print 'error is:\n', '*' * 50, '\n', error, '\n', '*' * 50
#sys.exit()
delfiles(['lnk', 'tmpf.f', 'tmpf.o', 'tmpc.c', 'tmpc.o'])
print 'works'
return 1
def down_install_blas(self):
print """
The netlib BLAS library is being installed.
Don't expect high performance from this netlib library!
If you want performance, you need to use an optimized BLAS library and,
to avoid unnecessary complications, if you need to compile this optimized BLAS
library, use the same compiler you're using here."""
sys.stdout.flush()
savecwd = os.getcwd()
# creating the build,lib and log dirs if don't exist
if not os.path.isdir(os.path.join(self.config.prefix, 'lib')):
os.mkdir(os.path.join(self.config.prefix, 'lib'))
if not os.path.isdir(os.path.join(os.getcwd(), 'log')):
os.mkdir(os.path.join(os.getcwd(), 'log'))
# Check if blas.tgz is already present in the working dir
# otherwise download it
if not os.path.isfile(
os.path.join(self.config.prefix, 'lib/librefblas.a')):
# if not os.path.isfile(os.path.join(os.getcwd(),getURLName(self.blasurl))):
# if not os.path.isfile(os.path.join(os.getcwd(),'BLAS')):
print "Downloading BLAS...",
# downloader(self.blasurl,self.downcmd)
# urllib.urlretrieve(self.blasurl, "blas.tgz")
geturl(self.blasurl, "blas.tgz")
print "Download is done"
else:
print "Netlib Blas library is already installed at " + os.path.join(
self.config.prefix, 'lib/librefblas.a')
self.config.blaslib = '-L' + os.path.join(self.config.prefix,
'lib') + ' -lrefblas '
return 0
# unzip and untar
os.chdir('download')
print 'Unzip and untar netlib BLAS...',
# comm = 'gunzip -f blas.tgz'
comm = 'mkdir BLAS; tar zx --strip-components=1 -C BLAS -f blas.tgz '
(output, error, retz) = shellcmd(comm)
if retz:
print '\n\nBLAS: cannot unzip blas.tgz'
print 'stderr:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50
sys.exit()
# comm = 'mkdir BLAS && tar x --strip-components=1 -C BLAS -f blas.tar'
# (output, error, retz) = shellcmd(comm)
# if retz:
# print '\n\nBLAS: cannot untar blas.tgz'
# print 'stderr:\n','*'*50,'\n',comm,'\n',error,'\n','*'*50
# sys.exit()
# os.remove('blas.tar')
print 'done'
# change to BLAS dir
os.chdir(os.path.join(os.getcwd(), 'BLAS'))
# compile and generate library
print 'Compile and generate netlib BLAS...',
sys.stdout.flush()
comm = self.config.fc + ' ' + self.config.fflags + " -c *.f"
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nBLAS: cannot compile blas"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
log = output + error
comm = "ar cr librefblas.a *.o"
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nBLAS: cannot create blas library"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
print "done"
log = log + output + error
comm = self.config.ranlib + " librefblas.a"
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nBLAS: cannot create table of contents for blas library"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
print "done"
# write the log on a file
log = log + output + error
fulllog = os.path.join(savecwd, 'log/log.blas')
writefile(fulllog, log)
print 'Installation of netlib BLAS successful.'
print '(log is in ', fulllog, ')'
# move librefblas.a to the lib directory
shutil.copy('librefblas.a',
os.path.join(self.config.prefix, 'lib/librefblas.a'))
# set framework variables to point to the freshly installed BLAS library
self.config.blaslib = '-L' + os.path.join(self.config.prefix,
'lib') + ' -lrefblas '
os.chdir(savecwd)
def main():
"""Framework for cross test generation and execution.
Builds and executes cross tests from the space of all possible attribute
combinations. The space can be restricted by providing subsets of attributes
to specifically include or exclude.
"""
# pypath is where to find other Subzero python scripts.
pypath = os.path.abspath(os.path.dirname(sys.argv[0]))
root = FindBaseNaCl()
# The rest of the attribute sets.
targets = [ 'x8632', 'x8664', 'arm32', 'mips32' ]
sandboxing = [ 'native', 'sandbox', 'nonsfi' ]
opt_levels = [ 'Om1', 'O2' ]
arch_attrs = { 'x8632': [ 'sse2', 'sse4.1' ],
'x8664': [ 'sse2', 'sse4.1' ],
'arm32': [ 'neon', 'hwdiv-arm' ],
'mips32': [ 'base' ]
}
flat_attrs = []
for v in arch_attrs.values():
flat_attrs += v
arch_flags = { 'x8632': [],
'x8664': [],
'arm32': [],
'mips32': []
}
# all_keys is only used in the help text.
all_keys = '; '.join([' '.join(targets), ' '.join(sandboxing),
' '.join(opt_levels), ' '.join(flat_attrs)])
argparser = argparse.ArgumentParser(
description=' ' + main.__doc__ +
'The set of attributes is the set of tests plus the following:\n' +
all_keys, formatter_class=argparse.RawTextHelpFormatter)
argparser.add_argument('--config', default='crosstest.cfg', dest='config',
metavar='FILE', help='Test configuration file')
argparser.add_argument('--print-tests', default=False, action='store_true',
help='Print the set of test names and exit')
argparser.add_argument('--include', '-i', default=[], dest='include',
action='append', metavar='ATTR_LIST',
help='Attributes to include (comma-separated). ' +
'Can be used multiple times.')
argparser.add_argument('--exclude', '-e', default=[], dest='exclude',
action='append', metavar='ATTR_LIST',
help='Attributes to include (comma-separated). ' +
'Can be used multiple times.')
argparser.add_argument('--verbose', '-v', default=False, action='store_true',
help='Use verbose output')
argparser.add_argument('--defer', default=False, action='store_true',
help='Defer execution until all executables are built')
argparser.add_argument('--no-compile', '-n', default=False,
action='store_true',
help="Don't build; reuse binaries from the last run")
argparser.add_argument('--dir', dest='dir', metavar='DIRECTORY',
default=('{root}/toolchain_build/src/subzero/' +
'crosstest/Output').format(root=root),
help='Output directory')
argparser.add_argument('--lit', default=False, action='store_true',
help='Generate files for lit testing')
argparser.add_argument('--toolchain-root', dest='toolchain_root',
default=(
'{root}/toolchain/linux_x86/pnacl_newlib_raw/bin'
).format(root=root),
help='Path to toolchain binaries.')
argparser.add_argument('--filetype', default=None, dest='filetype',
help='File type override, one of {asm, iasm, obj}.')
args = argparser.parse_args()
# Run from the crosstest directory to make it easy to grab inputs.
crosstest_dir = '{root}/toolchain_build/src/subzero/crosstest'.format(
root=root)
os.chdir(crosstest_dir)
tests = ConfigParser.RawConfigParser()
tests.read('crosstest.cfg')
if args.print_tests:
print 'Test name attributes: ' + ' '.join(sorted(tests.sections()))
sys.exit(0)
# includes and excludes are both lists of sets.
includes = [ set(item.split(',')) for item in args.include ]
excludes = [ set(item.split(',')) for item in args.exclude ]
# If any --include args are provided, the default is to not match.
default_match = not args.include
# Delete and recreate the output directory, unless --no-compile was specified.
if not args.no_compile:
if os.path.exists(args.dir):
if os.path.isdir(args.dir):
shutil.rmtree(args.dir)
else:
os.remove(args.dir)
if not os.path.exists(args.dir):
os.makedirs(args.dir)
# If --defer is specified, collect the run commands into deferred_cmds for
# later execution.
deferred_cmds = []
for test in sorted(tests.sections()):
for target in targets:
for sb in sandboxing:
for opt in opt_levels:
for attr in arch_attrs[target]:
desc = [ test, target, sb, opt, attr ]
if Match(set(desc), includes, excludes, default_match):
exe = '{test}_{target}_{sb}_{opt}_{attr}'.format(
test=test, target=target, sb=sb, opt=opt,
attr=attr)
extra = (tests.get(test, 'flags').split(' ')
if tests.has_option(test, 'flags') else [])
if args.filetype:
extra += ['--filetype={ftype}'.format(ftype=args.filetype)]
# Generate the compile command.
cmp_cmd = (
['{path}/crosstest.py'.format(path=pypath),
'-{opt}'.format(opt=opt),
'--mattr={attr}'.format(attr=attr),
'--prefix=Subzero_',
'--target={target}'.format(target=target),
'--nonsfi={nsfi}'.format(nsfi='1' if sb=='nonsfi' else '0'),
'--sandbox={sb}'.format(sb='1' if sb=='sandbox' else '0'),
'--dir={dir}'.format(dir=args.dir),
'--output={exe}'.format(exe=exe),
'--driver={drv}'.format(drv=tests.get(test, 'driver'))] +
extra +
['--test=' + t
for t in tests.get(test, 'test').split(' ')] +
arch_flags[target])
run_cmd_base = os.path.join(args.dir, exe)
# Generate the run command.
run_cmd = run_cmd_base
if sb == 'sandbox':
run_cmd = '{root}/run.py -q '.format(root=root) + run_cmd
elif sb == 'nonsfi':
run_cmd = (
'{root}/scons-out/opt-linux-{arch}/obj/src/nonsfi/' +
'loader/nonsfi_loader ').format(
root=root, arch=NonsfiLoaderArch(target)) + run_cmd
run_cmd = RunNativePrefix(args.toolchain_root, target, attr,
run_cmd)
else:
run_cmd = RunNativePrefix(args.toolchain_root, target, attr,
run_cmd)
if args.lit:
# Create a file to drive the lit test.
with open(run_cmd_base + '.xtest', 'w') as f:
f.write('# RUN: sh %s | FileCheck %s\n')
f.write('cd ' + crosstest_dir + ' && \\\n')
f.write(' '.join(cmp_cmd) + ' && \\\n')
f.write(run_cmd + '\n')
f.write('echo Recreate a failure using ' + __file__ +
' --toolchain-root=' + args.toolchain_root +
(' --filetype=' + args.filetype
if args.filetype else '') +
' --include=' + ','.join(desc) + '\n')
f.write('# CHECK: Failures=0\n')
else:
if not args.no_compile:
shellcmd(cmp_cmd,
echo=args.verbose)
if (args.defer):
deferred_cmds.append(run_cmd)
else:
shellcmd(run_cmd, echo=True)
for run_cmd in deferred_cmds:
shellcmd(run_cmd, echo=True)
def main():
"""Run the pnacl-sz compiler on an llvm file.
Takes an llvm input file, freezes it into a pexe file, converts
it to a Subzero program, and finally compiles it.
"""
argparser = argparse.ArgumentParser(
description=' ' + main.__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
argparser.add_argument('--input', '-i', required=True,
help='LLVM source file to compile')
argparser.add_argument('--output', '-o', required=False,
help='Output file to write')
argparser.add_argument('--insts', required=False,
action='store_true',
help='Stop after translating to ' +
'Subzero instructions')
argparser.add_argument('--no-local-syms', required=False,
action='store_true',
help="Don't keep local symbols in the pexe file")
argparser.add_argument('--llvm', required=False,
action='store_true',
help='Parse pexe into llvm IR first, then ' +
'convert to Subzero')
argparser.add_argument('--llvm-source', required=False,
action='store_true',
help='Parse source directly into llvm IR ' +
'(without generating a pexe), then ' +
'convert to Subzero')
argparser.add_argument(
'--pnacl-sz', required=False, default='./pnacl-sz', metavar='PNACL-SZ',
help="Subzero translator 'pnacl-sz'")
argparser.add_argument('--pnacl-bin-path', required=False,
default=(
'{root}/toolchain/linux_x86/pnacl_newlib_raw/bin'
).format(root=FindBaseNaCl()),
metavar='PNACL_BIN_PATH',
help='Path to LLVM & Binutils executables ' +
'(e.g. for building PEXE files)')
argparser.add_argument('--assemble', required=False,
action='store_true',
help='Assemble the output')
argparser.add_argument('--disassemble', required=False,
action='store_true',
help='Disassemble the assembled output')
argparser.add_argument('--dis-flags', required=False,
action='append', default=[],
help='Add a disassembler flag')
argparser.add_argument('--filetype', default='iasm', dest='filetype',
choices=['obj', 'asm', 'iasm'],
help='Output file type. Default %(default)s')
argparser.add_argument('--forceasm', required=False, action='store_true',
help='Force --filetype=asm')
argparser.add_argument('--target', default='x8632', dest='target',
choices=['x8632','x8664','arm32','mips32'],
help='Target architecture. Default %(default)s')
argparser.add_argument('--echo-cmd', required=False,
action='store_true',
help='Trace command that generates ICE instructions')
argparser.add_argument('--tbc', required=False, action='store_true',
help='Input is textual bitcode (not .ll)')
argparser.add_argument('--expect-fail', required=False, action='store_true',
help='Negate success of run by using LLVM not')
argparser.add_argument('--allow-pnacl-reader-error-recovery',
action='store_true',
help='Continue parsing after first error')
argparser.add_argument('--args', '-a', nargs=argparse.REMAINDER,
default=[],
help='Remaining arguments are passed to pnacl-sz')
argparser.add_argument('--sandbox', required=False, action='store_true',
help='Sandboxes the generated code')
args = argparser.parse_args()
pnacl_bin_path = args.pnacl_bin_path
llfile = args.input
if args.llvm and args.llvm_source:
raise RuntimeError("Can't specify both '--llvm' and '--llvm-source'")
if args.llvm_source and args.no_local_syms:
raise RuntimeError("Can't specify both '--llvm-source' and " +
"'--no-local-syms'")
if args.llvm_source and args.tbc:
raise RuntimeError("Can't specify both '--tbc' and '--llvm-source'")
if args.llvm and args.tbc:
raise RuntimeError("Can't specify both '--tbc' and '--llvm'")
if args.forceasm:
if args.expect_fail:
args.forceasm = False
elif args.filetype == 'asm':
pass
elif args.filetype == 'iasm':
# TODO(sehr) implement forceasm for iasm.
pass
elif args.filetype == 'obj':
args.filetype = 'asm'
args.assemble = True
cmd = []
if args.tbc:
cmd = [os.path.join(pnacl_bin_path, 'pnacl-bcfuzz'), llfile,
'-bitcode-as-text', '-output', '-', '|']
elif not args.llvm_source:
cmd = [os.path.join(pnacl_bin_path, 'llvm-as'), llfile, '-o', '-', '|',
os.path.join(pnacl_bin_path, 'pnacl-freeze')]
if not args.no_local_syms:
cmd += ['--allow-local-symbol-tables']
cmd += ['|']
if args.expect_fail:
cmd += [os.path.join(pnacl_bin_path, 'not')]
cmd += [args.pnacl_sz]
cmd += ['--target', args.target]
if args.sandbox:
cmd += ['-sandbox']
if args.insts:
# If the tests are based on '-verbose inst' output, force
# single-threaded translation because dump output does not get
# reassembled into order.
cmd += ['-verbose', 'inst,global_init', '-notranslate', '-threads=0']
elif args.allow_pnacl_reader_error_recovery:
cmd += ['-allow-pnacl-reader-error-recovery', '-threads=0']
if not args.llvm_source:
cmd += ['--bitcode-format=pnacl']
if not args.no_local_syms:
cmd += ['--allow-local-symbol-tables']
if args.llvm or args.llvm_source:
cmd += ['--build-on-read=0']
else:
cmd += ['--build-on-read=1']
cmd += ['--filetype=' + args.filetype]
cmd += ['--emit-revision=0']
script_name = os.path.basename(sys.argv[0])
for _, arg in enumerate(args.args):
# Redirecting the output file needs to be done through the script
# because forceasm may introduce a new temporary file between pnacl-sz
# and llvm-mc. Similar issues could occur when setting filetype, target,
# or sandbox through --args. Filter and report an error.
if re.search('^-?-(o|output|filetype|target|sandbox)(=.+)?$', arg):
preferred_option = '--output' if re.search('^-?-o(=.+)?$', arg) else arg
print 'Option should be set using:'
print ' %s ... %s ... --args' % (script_name, preferred_option)
print 'rather than:'
print ' %s ... --args %s ...' % (script_name, arg)
exit(1)
asm_temp = None
output_file_name = None
keep_output_file = False
if args.output:
output_file_name = args.output
keep_output_file = True
cmd += args.args
if args.llvm_source:
cmd += [llfile]
if args.assemble or args.disassemble:
if not output_file_name:
# On windows we may need to close the file first before it can be
# re-opened by the other tools, so don't do delete-on-close,
# and instead manually delete.
asm_temp = tempfile.NamedTemporaryFile(delete=False)
asm_temp.close()
output_file_name = asm_temp.name
if args.assemble and args.filetype != 'obj':
cmd += (['|', os.path.join(pnacl_bin_path, 'llvm-mc')] +
TargetAssemblerFlags(args.target, args.sandbox) +
['-filetype=obj', '-o', output_file_name])
elif output_file_name:
cmd += ['-o', output_file_name]
if args.disassemble:
# Show wide instruction encodings, diassemble, show relocs and
# dissasemble zeros.
cmd += (['&&', os.path.join(pnacl_bin_path, GetObjdumpCmd(args.target))] +
args.dis_flags +
['-w', '-d', '-r', '-z'] + TargetDisassemblerFlags(args.target) +
[output_file_name])
stdout_result = shellcmd(cmd, echo=args.echo_cmd)
if not args.echo_cmd:
sys.stdout.write(stdout_result)
if asm_temp and not keep_output_file:
os.remove(output_file_name)
def down_install_lapack(self):
print """
The LAPACK library is being installed.
"""
sys.stdout.flush()
savecwd = os.getcwd()
# creating the build,lib and log dirs if don't exist
if not os.path.isdir(os.path.join(self.config.prefix, 'lib')):
os.mkdir(os.path.join(self.config.prefix, 'lib'))
if not os.path.isdir(os.path.join(os.getcwd(), 'log')):
os.mkdir(os.path.join(os.getcwd(), 'log'))
# Check if lapack.tgz is already present in the working dir
# otherwise download it
if not os.path.isfile(
os.path.join(self.config.prefix, 'lib/liblapack.a')):
# if not os.path.isfile(os.path.join(os.getcwd(),getURLName(self.lapackurl))):
print "Downloading LAPACK...",
# downloader(self.lapackurl,self.downcmd)
# urllib.urlretrieve(self.lapackurl, "lapack.tgz")
geturl(self.lapackurl, "lapack.tgz")
print "Download is done"
else:
print "Netlib Lapack library is already installed at " + os.path.join(
self.config.prefix, 'lib/liblapack.a')
self.config.lapacklib = '-L' + os.path.join(
self.config.prefix, 'lib') + ' -ltmg -llapack'
return 0
# unzip and untar
os.chdir('download')
print 'Unzip and untar Lapack...',
#comm = 'gunzip -f '+self.lapversion+'.tgz'
comm = 'tar zxf lapack.tgz'
(output, error, retz) = shellcmd(comm)
if retz:
print '\n\nLAPACK: cannot unzip ' + self.lapversion + '.tgz'
print 'stderr:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50
sys.exit()
#comm = 'tar xf '+self.lapversion+'.tar'
#(output, error, retz) = shellcmd(comm)
#if retz:
# print '\n\nLAPACK: cannot untar '+self.lapversion+'.tar'
# print 'stderr:\n','*'*50,'\n',comm,'\n',error,'\n','*'*50
# sys.exit()
#os.remove(self.lapversion+'.tar')
print 'done'
##Apply the patch to correct [sd]lantr
#print 'Apply patch on lapacke...',
#comm = '(cd '+self.lapversion+' && patch -p 0 < '+(os.path.join(savecwd,'../script/patch_lantr'))+')'
#(output, error, retz) = shellcmd(comm)
#print 'done'
# # Overwrite [sd]lamch.f
# shutil.copy(os.path.join(self.dmft.installerdir,'src/dlamch.f'),
# os.path.join(os.getcwd(),'lapack-3.3.1/INSTALL'))
# shutil.copy(os.path.join(self.dmft.installerdir,'src/slamch.f'),
# os.path.join(os.getcwd(),'lapack-3.3.1/INSTALL'))
# change to BLAS dir
os.chdir(os.path.join(os.getcwd(), self.lapversion))
# Write Makefile.in
writefile(
'make.inc', """
# -*- Makefile generated by DMFT installer -*-
####################################################################
# LAPACK make include file. #
# LAPACK, Version """ + self.lapversion +
"""" #
# April 2012 #
####################################################################
#
SHELL = /bin/sh
#
# Modify the FORTRAN and OPTS definitions to refer to the
# compiler and desired compiler options for your machine. NOOPT
# refers to the compiler options desired when NO OPTIMIZATION is
# selected. Define LOADER and LOADOPTS to refer to the loader and
# desired load options for your machine.
#
FORTRAN = """ + self.config.fc + """
OPTS = """ + self.config.fflags + """
DRVOPTS = $(OPTS)
NOOPT = -O0
LOADER = """ + self.config.fc + """
LOADOPTS =
MAKE = make -j 8
#
# Timer for the SECOND and DSECND routines
#
# Default : SECOND and DSECND will use a call to the EXTERNAL FUNCTION ETIME
# TIMER = EXT_ETIME
# For RS6K : SECOND and DSECND will use a call to the EXTERNAL FUNCTION ETIME_
# TIMER = EXT_ETIME_
# For gfortran compiler: SECOND and DSECND will use a call to the INTERNAL FUNCTION ETIME
# TIMER = INT_ETIME
# If your Fortran compiler does not provide etime (like Nag Fortran Compiler, etc...)
# SECOND and DSECND will use a call to the INTERNAL FUNCTION CPU_TIME
TIMER = INT_CPU_TIME
# If neither of this works...you can use the NONE value... In that case, SECOND and DSECND will always return 0
# TIMER = NONE
#
# Configuration LAPACKE: Native C interface to LAPACK
# To generate LAPACKE library: type 'make lapackelib'
# Configuration file: turned off (default)
# Complex types: C99 (default)
# Name pattern: mixed case (default)
# (64-bit) Data model: LP64 (default)
#
# CC is the C compiler, normally invoked with options CFLAGS.
#
CC = """ + self.config.cc + """
CFLAGS = """ + self.config.cflags + """
#
# The archiver and the flag(s) to use when building archive (library)
# If you system has no ranlib, set RANLIB = echo.
#
ARCH = ar
ARCHFLAGS= """ + self.config.arflags + """
RANLIB = """ + self.config.ranlib + """
#
# The location of BLAS library for linking the testing programs.
# The target's machine-specific, optimized BLAS library should be
# used whenever possible.
#
BLASLIB = """ + self.config.blaslib + """
#
# Location of the extended-precision BLAS (XBLAS) Fortran library
# used for building and testing extended-precision routines. The
# relevant routines will be compiled and XBLAS will be linked only if
# USEXBLAS is defined.
#
# USEXBLAS = Yes
XBLASLIB =
# XBLASLIB = -lxblas
#
# Names of generated libraries.
#
LAPACKLIB = liblapack.a
TMGLIB = libtmg.a
EIGSRCLIB = libeigsrc.a
LINSRCLIB = liblinsrc.a
LAPACKELIB = liblapacke.a
""")
# compile and generate library
print 'Compile and generate LAPACK...',
sys.stdout.flush()
comm = self.make + ' lapacklib tmglib'
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nLAPACK: cannot compile LAPACK"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
log = output + error
# write the log on a file
log = log + output + error
fulllog = os.path.join(savecwd, 'log/log.lapack')
writefile(fulllog, log)
print 'Installation of liblapack.a successful.'
print '(log is in ', fulllog, ')'
# move libcblas.a to the lib directory
shutil.copy('liblapack.a',
os.path.join(self.config.prefix, 'lib/liblapack.a'))
shutil.copy('libtmg.a', os.path.join(self.config.prefix,
'lib/libtmg.a'))
# set framework variables to point to the freshly installed BLAS library
self.config.lapacklib = '-L' + os.path.join(self.config.prefix,
'lib') + ' -ltmg -llapack'
os.chdir(savecwd)
self.config.lapinstalled = 1
# Check if the installation is successful
self.dmft.verbose = 1
ret = self.check_lapack()
self.dmft.verbose = 0
if ret != 0:
sys.exit()
def main():
"""Run the pnacl-sz compiler on an llvm file.
Takes an llvm input file, freezes it into a pexe file, converts
it to a Subzero program, and finally compiles it.
"""
argparser = argparse.ArgumentParser(
description=' ' + main.__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
argparser.add_argument('--input',
'-i',
required=True,
help='LLVM source file to compile')
argparser.add_argument('--output',
'-o',
required=False,
help='Output file to write')
argparser.add_argument('--insts',
required=False,
action='store_true',
help='Stop after translating to ' +
'Subzero instructions')
argparser.add_argument('--no-local-syms',
required=False,
action='store_true',
help="Don't keep local symbols in the pexe file")
argparser.add_argument('--llvm',
required=False,
action='store_true',
help='Parse pexe into llvm IR first, then ' +
'convert to Subzero')
argparser.add_argument('--llvm-source',
required=False,
action='store_true',
help='Parse source directly into llvm IR ' +
'(without generating a pexe), then ' +
'convert to Subzero')
argparser.add_argument('--pnacl-sz',
required=False,
default='./pnacl-sz',
metavar='PNACL-SZ',
help="Subzero translator 'pnacl-sz'")
argparser.add_argument(
'--pnacl-bin-path',
required=False,
default=('{root}/toolchain/linux_x86/pnacl_newlib_raw/bin').format(
root=FindBaseNaCl()),
metavar='PNACL_BIN_PATH',
help='Path to LLVM & Binutils executables ' +
'(e.g. for building PEXE files)')
argparser.add_argument('--assemble',
required=False,
action='store_true',
help='Assemble the output')
argparser.add_argument('--disassemble',
required=False,
action='store_true',
help='Disassemble the assembled output')
argparser.add_argument('--dis-flags',
required=False,
action='append',
default=[],
help='Add a disassembler flag')
argparser.add_argument('--filetype',
default='iasm',
dest='filetype',
choices=['obj', 'asm', 'iasm'],
help='Output file type. Default %(default)s')
argparser.add_argument('--forceasm',
required=False,
action='store_true',
help='Force --filetype=asm')
argparser.add_argument('--target',
default='x8632',
dest='target',
choices=['x8632', 'x8664', 'arm32', 'mips32'],
help='Target architecture. Default %(default)s')
argparser.add_argument(
'--echo-cmd',
required=False,
action='store_true',
help='Trace command that generates ICE instructions')
argparser.add_argument('--tbc',
required=False,
action='store_true',
help='Input is textual bitcode (not .ll)')
argparser.add_argument('--expect-fail',
required=False,
action='store_true',
help='Negate success of run by using LLVM not')
argparser.add_argument('--allow-pnacl-reader-error-recovery',
action='store_true',
help='Continue parsing after first error')
argparser.add_argument('--args',
'-a',
nargs=argparse.REMAINDER,
default=[],
help='Remaining arguments are passed to pnacl-sz')
argparser.add_argument('--sandbox',
required=False,
action='store_true',
help='Sandboxes the generated code')
args = argparser.parse_args()
pnacl_bin_path = args.pnacl_bin_path
llfile = args.input
if args.llvm and args.llvm_source:
raise RuntimeError("Can't specify both '--llvm' and '--llvm-source'")
if args.llvm_source and args.no_local_syms:
raise RuntimeError("Can't specify both '--llvm-source' and " +
"'--no-local-syms'")
if args.llvm_source and args.tbc:
raise RuntimeError("Can't specify both '--tbc' and '--llvm-source'")
if args.llvm and args.tbc:
raise RuntimeError("Can't specify both '--tbc' and '--llvm'")
if args.forceasm:
if args.expect_fail:
args.forceasm = False
elif args.filetype == 'asm':
pass
elif args.filetype == 'iasm':
# TODO(sehr) implement forceasm for iasm.
pass
elif args.filetype == 'obj':
args.filetype = 'asm'
args.assemble = True
cmd = []
if args.tbc:
cmd = [
os.path.join(pnacl_bin_path, 'pnacl-bcfuzz'), llfile,
'-bitcode-as-text', '-output', '-', '|'
]
elif not args.llvm_source:
cmd = [
os.path.join(pnacl_bin_path, 'llvm-as'), llfile, '-o', '-', '|',
os.path.join(pnacl_bin_path, 'pnacl-freeze')
]
if not args.no_local_syms:
cmd += ['--allow-local-symbol-tables']
cmd += ['|']
if args.expect_fail:
cmd += [os.path.join(pnacl_bin_path, 'not')]
cmd += [args.pnacl_sz]
cmd += ['--target', args.target]
if args.sandbox:
cmd += ['-sandbox']
if args.insts:
# If the tests are based on '-verbose inst' output, force
# single-threaded translation because dump output does not get
# reassembled into order.
cmd += ['-verbose', 'inst,global_init', '-notranslate', '-threads=0']
elif args.allow_pnacl_reader_error_recovery:
cmd += ['-allow-pnacl-reader-error-recovery', '-threads=0']
if not args.llvm_source:
cmd += ['--bitcode-format=pnacl']
if not args.no_local_syms:
cmd += ['--allow-local-symbol-tables']
if args.llvm or args.llvm_source:
cmd += ['--build-on-read=0']
else:
cmd += ['--build-on-read=1']
cmd += ['--filetype=' + args.filetype]
cmd += ['--emit-revision=0']
script_name = os.path.basename(sys.argv[0])
for _, arg in enumerate(args.args):
# Redirecting the output file needs to be done through the script
# because forceasm may introduce a new temporary file between pnacl-sz
# and llvm-mc. Similar issues could occur when setting filetype, target,
# or sandbox through --args. Filter and report an error.
if re.search('^-?-(o|output|filetype|target|sandbox)(=.+)?$', arg):
preferred_option = '--output' if re.search('^-?-o(=.+)?$',
arg) else arg
print 'Option should be set using:'
print ' %s ... %s ... --args' % (script_name, preferred_option)
print 'rather than:'
print ' %s ... --args %s ...' % (script_name, arg)
exit(1)
asm_temp = None
output_file_name = None
keep_output_file = False
if args.output:
output_file_name = args.output
keep_output_file = True
cmd += args.args
if args.llvm_source:
cmd += [llfile]
if args.assemble or args.disassemble:
if not output_file_name:
# On windows we may need to close the file first before it can be
# re-opened by the other tools, so don't do delete-on-close,
# and instead manually delete.
asm_temp = tempfile.NamedTemporaryFile(delete=False)
asm_temp.close()
output_file_name = asm_temp.name
if args.assemble and args.filetype != 'obj':
cmd += (['|', os.path.join(pnacl_bin_path, 'llvm-mc')] +
TargetAssemblerFlags(args.target, args.sandbox) +
['-filetype=obj', '-o', output_file_name])
elif output_file_name:
cmd += ['-o', output_file_name]
if args.disassemble:
# Show wide instruction encodings, diassemble, show relocs and
# dissasemble zeros.
cmd += (
['&&',
os.path.join(pnacl_bin_path, GetObjdumpCmd(args.target))] +
args.dis_flags + ['-w', '-d', '-r', '-z'] +
TargetDisassemblerFlags(args.target) + [output_file_name])
stdout_result = shellcmd(cmd, echo=args.echo_cmd)
if not args.echo_cmd:
sys.stdout.write(stdout_result)
if asm_temp and not keep_output_file:
os.remove(output_file_name)
def check_lapack(self):
print "Checking if provided LAPACK works...",
# This function simply generates a C program
# that contains few calls to LAPACK routine and then
# checks if compilation, linking and execution are succesful
sys.stdout.flush()
writefile(
'tmpf.f', """
program ftest
integer N
parameter (N = 1)
double precision A(N, N), B(N)
integer I(N)
integer INFO
B(:) = 1
A(:,:) = 2
I(:) = 0
call cheevd( 'N', 'U', N, A, N, B, B, -1,
$ B, -1, I, -1, INFO)
stop
end\n""")
ldflg = self.config.lapacklib + ' ' + self.config.blaslib + ' -lm'
ccomm = self.config.fc + ' -o tmpf ' + 'tmpf.f ' + ldflg
(output, error, retz) = shellcmd(ccomm)
if (retz != 0):
if self.dmft.verbose:
print '\n\nLAPACK: provided LAPACK cannot be used! aborting...'
print 'error is:\n', '*' * 50, '\n', ccomm, '\n', error, '\n', '*' * 50
else:
print "no"
return -1
comm = './tmpf'
(output, error, retz) = shellcmd(comm)
if (retz != 0):
if self.dmft.verbose:
print '\n\nLAPACK: provided LAPACK cannot be used! aborting...'
print 'error is:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50
else:
print "no"
return -1
delfiles(['tmpf.f', 'tmpf'])
print 'yes'
# print "Checking if provided LAPACK contains functions for test works...",
# This function simply generates a C program
# that contains few calls to LAPACK routine and then
# checks if compilation, linking and execution are succesful
sys.stdout.flush()
writefile(
'tmpf.f', """
program ftest
double precision D(1), A(1:1), B(2)
integer ISEED( 4 )
integer INFO
B(1) = 1
do I = 1, 4
ISEED( I ) = 1
enddo
call dlarnv( 1, ISEED, 1, D )
call dlagsy( 1, 0, D, A, 1, ISEED, B, INFO )
stop
end\n""")
ccomm = self.config.fc + ' -o tmpf ' + 'tmpf.f ' + ldflg
(output, error, retz) = shellcmd(ccomm)
if (retz != 0):
print 'no'
self.dmft.needtmg = 1
else:
comm = './tmpf'
(output, error, retz) = shellcmd(comm)
if (retz != 0):
print 'no'
self.dmft.needtmg = 1
else:
self.dmft.needtmg = 0
# print 'yes'
delfiles(['tmpf.f', 'tmpf'])
return 0
'(for building PNaCl files)')
argparser.add_argument('--pnacl', required=False,
action='store_true',
help='Convert llvm source to PNaCl bitcode ' +
'file first')
argparser.add_argument('--echo-cmd', required=False,
action='store_true',
help='Trace command that generates ICE instructions')
argparser.add_argument('llfile', nargs=1,
metavar='LLVM_FILE',
help='Llvm source file')
args = argparser.parse_args()
llvm_bin_path = args.llvm_bin_path
llfile = args.llfile[0]
cmd = []
if args.pnacl:
cmd = [os.path.join(llvm_bin_path, 'llvm-as'), llfile, '-o', '-', '|',
os.path.join(llvm_bin_path, 'pnacl-freeze'),
'--allow-local-symbol-tables', '|']
cmd += [args.llvm2ice, '-verbose', 'inst', '-notranslate']
if args.pnacl:
cmd += ['--allow-local-symbol-tables', '--bitcode-format=pnacl']
else:
cmd.append(llfile)
stdout_result = shellcmd(cmd, echo=args.echo_cmd)
if not args.echo_cmd:
sys.stdout.write(stdout_result)
def down_install_fftw(self):
print """
The FFTW library is being installed.
"""
sys.stdout.flush()
savecwd = os.getcwd()
# creating the build,lib and log dirs if don't exist
if not os.path.isdir(os.path.join(self.config.prefix, 'fftw')):
os.mkdir(os.path.join(self.config.prefix, 'fftw'))
if not os.path.isdir(os.path.join(os.getcwd(), 'log')):
os.mkdir(os.path.join(os.getcwd(), 'log'))
# Check if fftw.tgz is already present in the working dir
# otherwise download it
if not os.path.isfile(os.path.join(os.getcwd(), "fftw.tgz")):
print "Downloading FFTW ...",
#downloader(self.lapackurl,self.downcmd)
#urllib.urlretrieve(self.fftwurl, "fftw.tgz")
geturl(self.fftwurl, "fftw.tgz")
print "done"
# unzip and untar
os.chdir('download')
print 'Unzip and untar FFTW...',
comm = 'tar zxf fftw.tgz '
(output, error, retz) = shellcmd(comm)
if retz:
print '\n\nlibfftw: cannot unzip ' + self.fftwversion + '.tgz'
print 'stderr:\n', '*' * 50, '\n', comm, '\n', error, '\n', '*' * 50
sys.exit()
print 'done'
# change to FFTW dir
os.chdir(os.path.join(os.getcwd(), self.fftwversion))
# compile and generate library
print 'Configure FFTW...',
sys.stdout.flush()
if (self.config.ompflag == ""):
comm = './configure MPICC='+self.config.pcc+' CC='+self.config.cc+' F77='+self.config.fc+\
' --enable-mpi --enable-threads --enable-shared --prefix='+os.path.join(self.config.prefix,'fftw')
else:
comm = './configure MPICC='+self.config.pcc+' CC='+self.config.cc+' F77='+self.config.fc+\
' --enable-mpi --enable-threads --enable-shared -enable-openmp --prefix='+os.path.join(self.config.prefix,'fftw')
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nlinfftw: cannot configure FFTW"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
log = output + error
# write log on a file
log = log + output + error
fulllog = os.path.join(savecwd, 'log/log.fftw')
writefile(fulllog, log)
print 'Configuration of FFTW successful.'
print '(log is in ', fulllog, ')'
# compile and generate library
print 'Compile and generate FFTW...',
sys.stdout.flush()
comm = self.make + ' -j4; ' + self.make + ' install'
(output, error, retz) = shellcmd(comm)
if retz:
print "\n\nlinfftw: cannot compile FFTW"
print "stderr:\n", "*" * 50, "\n", comm, '\n', error, "\n", "*" * 50
sys.exit()
log = output + error
# write the log on a file
log = log + output + error
fulllog = os.path.join(savecwd, 'log/log.fftw')
writefile(fulllog, log)
print 'Installation of FFTW successful.'
print '(log is in ', fulllog, ')'
# move libcblas.a to the lib directory
#shutil.copy('libtmg.a',os.path.join(self.config.prefix,'fftw/libtmg.a'))
self.set_fftlib()
os.chdir(savecwd)
# Check if the installation is successful
self.dmft.verbose = 1
self.check_fftw()
self.dmft.verbose = 0
