def __init__(self, header, filename):
pack_format = ''.join(fmt for fmt, _ in self.ELF_HEADER_FORMAT)
e_class = ord(header[4])
if e_class == ELFHeader.ELFCLASS32:
pack_format = pack_format.replace('P', 'I')
elif e_class == ELFHeader.ELFCLASS64:
pack_format = pack_format.replace('P', 'Q')
else:
Log.Fatal('%s: ELF file has unknown class (%d)', filename, e_class)
ehdr = self.Ehdr(*struct.unpack_from(pack_format, header))
e_osabi = ord(header[7])
e_abiver = ord(header[8])
if e_osabi not in ELFHeader.ELF_OSABI:
Log.Fatal('%s: ELF file has unknown OS ABI (%d)', filename,
e_osabi)
if e_abiver not in ELFHeader.ELF_ABI_VER:
Log.Fatal('%s: ELF file has unknown ABI version (%d)', filename,
e_abiver)
if ehdr.e_type not in ELFHeader.ELF_TYPES:
Log.Fatal('%s: ELF file has unknown type (%d)', filename,
ehdr.e_type)
if ehdr.e_machine not in ELFHeader.ELF_MACHINES:
Log.Fatal('%s: ELF file has unknown machine type (%d)', filename,
ehdr.e_machine)
self.type = self.ELF_TYPES[ehdr.e_type]
self.machine = self.ELF_MACHINES[ehdr.e_machine]
self.osabi = self.ELF_OSABI[e_osabi]
self.abiver = self.ELF_ABI_VER[e_abiver]
self.arch = FixArch(self.machine) # For convenience
self.phoff = ehdr.e_phoff
self.phnum = ehdr.e_phnum
self.phentsize = ehdr.e_phentsize
def ReadConfig():
# Mock out ReadConfig if running unittests. Settings are applied directly
# by DriverTestEnv rather than reading this configuration file.
if env.has('PNACL_RUNNING_UNITTESTS'):
return
driver_bin = env.getone('DRIVER_BIN')
driver_conf = pathtools.join(driver_bin, 'driver.conf')
fp = DriverOpen(driver_conf, 'r')
linecount = 0
for line in fp:
linecount += 1
line = line.strip()
if line == '' or line.startswith('#'):
continue
sep = line.find('=')
if sep < 0:
Log.Fatal("%s: Parse error, missing '=' on line %d",
pathtools.touser(driver_conf), linecount)
keyname = line[:sep].strip()
value = line[sep + 1:].strip()
env.setraw(keyname, value)
DriverClose(fp)
if env.getone('LIBMODE') not in ('newlib', 'glibc'):
Log.Fatal('Invalid LIBMODE in %s', pathtools.touser(driver_conf))
def main(argv):
env.update(EXTRA_ENV)
driver_tools.ParseArgs(argv, PATTERNS)
inputs = env.get('INPUTS')
if len(inputs) == 0:
Log.Fatal("No input files given")
for infile in inputs:
driver_tools.CheckPathLength(infile)
env.push()
env.set('input', infile)
# For frozen PNaCl bitcode, use 'llvm-nm -bitcode-format=pnacl'. For all
# other formats, use the binutils nm with our gold plugin.
# Update: llvm-nm -bitcode-format=pnacl is currently disabled.
if filetype.IsPNaClBitcode(infile):
Log.Fatal(
'nm on finalized bitcode is currently disabled.\n'
'See: https://code.google.com/p/nativeclient/issues/detail?id=3993'
)
else:
env.set('TOOLNAME', '${NM}')
env.append('FLAGS', '--plugin=${GOLD_PLUGIN_SO}')
driver_tools.Run('"${TOOLNAME}" ${FLAGS} ${input}')
env.pop()
# only reached in case of no errors
return 0
def ArchMerge(filename, must_match):
file_type = filetype.FileType(filename)
if file_type in ('o','so'):
elfheader = elftools.GetELFHeader(filename)
if not elfheader:
Log.Fatal("%s: Cannot read ELF header", filename)
new_arch = elfheader.arch
elif filetype.IsNativeArchive(filename):
new_arch = file_type[len('archive-'):]
else:
Log.Fatal('%s: Unexpected file type in ArchMerge', filename)
existing_arch = GetArch()
if not existing_arch:
SetArch(new_arch)
return True
elif new_arch != existing_arch:
if must_match:
msg = "%s: Incompatible object file (%s != %s)"
logfunc = Log.Fatal
else:
msg = "%s: Skipping incompatible object file (%s != %s)"
logfunc = Log.Warning
logfunc(msg, filename, new_arch, existing_arch)
return False
else: # existing_arch and new_arch == existing_arch
return True
def main(argv):
env.update(EXTRA_ENV)
ParseArgs(argv, PATTERNS)
inputs = env.get('INPUTS')
if len(inputs) == 0:
Log.Fatal("No input files given")
for infile in inputs:
env.push()
env.set('input', infile)
if filetype.IsLLVMBitcode(infile):
# Hack to support newlib build.
# Newlib determines whether the toolchain supports .init_array, etc., by
# compiling a small test and looking for a specific section tidbit using
# "readelf -S". Since pnacl compiles to bitcode, readelf isn't available.
# (there is a line: "if ${READELF} -S conftest | grep -e INIT_ARRAY"
# in newlib's configure file).
# TODO(sehr): we may want to implement a whole readelf on bitcode.
flags = env.get('FLAGS')
if len(flags) == 1 and flags[0] == '-S':
print 'INIT_ARRAY'
return 0
Log.Fatal('Cannot handle pnacl-readelf %s' % str(argv))
return 1
Run('"${READELF}" ${FLAGS} ${input}')
env.pop()
# only reached in case of no errors
return 0
def main(argv):
env.update(EXTRA_ENV)
driver_tools.ParseArgs(argv, StripPatterns)
inputs = env.get('INPUTS')
output = env.getone('OUTPUT')
if len(inputs) > 1 and output != '':
Log.Fatal('Cannot have -o with multiple inputs')
if '--info' in env.get('STRIP_FLAGS'):
code, _, _ = driver_tools.Run('${STRIP} ${STRIP_FLAGS}')
return code
for f in inputs:
if output != '':
f_output = output
else:
f_output = f
if driver_tools.IsLLVMBitcode(f):
driver_tools.RunWithEnv('${RUN_OPT}', input=f, output=f_output)
elif driver_tools.IsELF(f) or driver_tools.IsNativeArchive(f):
driver_tools.RunWithEnv('${RUN_STRIP}', input=f, output=f_output)
elif driver_tools.IsBitcodeArchive(f):
# The strip tool supports native archives, but it does not support the
# LLVM gold plugin so cannot handle bitcode. There is also no bitcode
# tool like opt that support archives.
Log.Fatal('%s: strip does not support bitcode archives',
pathtools.touser(f))
else:
Log.Fatal('%s: File is neither ELF, nor bitcode',
pathtools.touser(f))
return 0
def main(argv):
env.update(EXTRA_ENV)
driver_tools.ParseArgs(argv, DISPatterns)
inputs = env.get('INPUTS')
output = env.getone('OUTPUT')
if len(inputs) == 0:
Log.Fatal("No input files given")
if len(inputs) > 1 and output != '':
Log.Fatal("Cannot have -o with multiple inputs")
for infile in inputs:
env.push()
env.set('input', infile)
env.set('output', output)
# When we output to stdout, set redirect_stdout and set log_stdout
# to False to bypass the driver's line-by-line handling of stdout
# which is extremely slow when you have a lot of output
if (filetype.IsLLVMBitcode(infile) or filetype.IsPNaClBitcode(infile)):
bitcodetype = 'PNaCl' if filetype.IsPNaClBitcode(
infile) else 'LLVM'
format = bitcodetype.lower()
if env.has('FILE_TYPE'):
sys.stdout.write('%s: %s bitcode\n' % (infile, bitcodetype))
continue
env.append('FLAGS', '-bitcode-format=' + format)
if output == '':
# LLVM by default outputs to a file if -o is missing
# Let's instead output to stdout
env.set('output', '-')
env.append('FLAGS', '-f')
driver_tools.Run('${LLVM_DIS} ${FLAGS} ${input} -o ${output}')
elif filetype.IsELF(infile):
if env.has('FILE_TYPE'):
sys.stdout.write('%s: ELF\n' % infile)
continue
flags = env.get('FLAGS')
if len(flags) == 0:
env.append('FLAGS', '-d')
if output == '':
# objdump to stdout
driver_tools.Run('"${OBJDUMP}" ${FLAGS} ${input}')
else:
# objdump always outputs to stdout, and doesn't recognize -o
# Let's add this feature to be consistent.
fp = DriverOpen(output, 'w')
driver_tools.Run('${OBJDUMP} ${FLAGS} ${input}',
redirect_stdout=fp)
DriverClose(fp)
else:
Log.Fatal('Unknown file type')
env.pop()
# only reached in case of no errors
return 0
def main(argv):
env.update(EXTRA_ENV)
driver_tools.ParseArgs(argv, PrepPatterns)
inputs = env.get('INPUTS')
output = env.getone('OUTPUT')
if len(inputs) != 1:
Log.Fatal('Can only have one input')
f_input = inputs[0]
# Allow in-place file changes if output isn't specified..
if output != '':
f_output = output
else:
f_output = f_input
if env.getbool('DISABLE_FINALIZE') or filetype.IsPNaClBitcode(f_input):
# Just copy the input file to the output file.
if f_input != f_output:
shutil.copyfile(f_input, f_output)
return 0
opt_flags = ['-disable-opt', '-strip', '-strip-metadata',
'--bitcode-format=pnacl', f_input, '-o', f_output]
# Transform the file, and convert it to a PNaCl bitcode file.
driver_tools.RunDriver('opt', opt_flags)
return 0
def FileType(filename):
# Auto-detect bitcode files, since we can't rely on extensions
ext = filename.split('.')[-1]
# TODO(pdox): We open and read the the first few bytes of each file
# up to 4 times, when we only need to do it once. The
# OS cache prevents us from hitting the disk, but this
# is still slower than it needs to be.
if IsArchive(filename):
return artools.GetArchiveType(filename)
if IsELF(filename):
return GetELFType(filename)
if IsLLVMBitcode(filename) or IsPNaClBitcode(filename):
return GetBitcodeType(filename, False)
if (ext in ('o', 'so', 'a', 'po', 'pso', 'pa', 'x')
and ldtools.IsLinkerScript(filename)):
return 'ldscript'
# Use the file extension if it is recognized
if ext in ExtensionMap:
return ExtensionMap[ext]
Log.Fatal('%s: Unrecognized file type', filename)
def ParseArgsBase(argv, patternlist):
""" Parse argv using the patterns in patternlist
Returns: (matched, unmatched)
"""
matched = []
unmatched = []
i = 0
while i < len(argv):
if ShouldExpandCommandFile(argv[i]):
argv = DoExpandCommandFile(argv, i)
num_matched, action, groups = MatchOne(argv, i, patternlist)
if num_matched == 0:
unmatched.append(argv[i])
i += 1
continue
matched += argv[i:i+num_matched]
if isinstance(action, str):
# Perform $N substitution
for g in xrange(0, len(groups)):
action = action.replace('$%d' % g, 'groups[%d]' % g)
try:
if isinstance(action, str):
# NOTE: this is essentially an eval for python expressions
# which does rely on the current environment for unbound vars
# Log.Info('about to exec [%s]', str(action))
exec(action)
else:
action(*groups)
except Exception, err:
Log.Fatal('ParseArgs action [%s] failed with: %s', action, err)
i += num_matched
def GenerateDynamicNMF(pexe_file, baseline_nmf, needed, output_file):
nmf_json = baseline_nmf
# Set runnable-ld.so as the program interpreter.
nmf_json['program'] = {}
AddUrlForAllArches(nmf_json['program'], 'runnable-ld.so')
# Set the pexe as the main program.
nmf_json['files'] = baseline_nmf.get('files', {})
nmf_json['files']['main.nexe'] = {}
SetPortableUrl(nmf_json['files']['main.nexe'],
pathtools.basename(pexe_file), pexe_file)
# Get transitive closure of needed libraries.
transitive_needed = GetTransitiveClosureOfNeeded(needed)
# Make urls for libraries.
for lib in transitive_needed:
nmf_json['files'][lib] = {}
actual_lib_path, file_type = GetActualFilePathAndType(lib)
if file_type == 'so':
# Assume a native version exists for every known arch.
AddUrlForAllArches(nmf_json['files'][lib], lib)
elif file_type == 'pso':
SetPortableUrl(nmf_json['files'][lib], lib, actual_lib_path)
else:
Log.Fatal('Needed library is not a .so nor a .pso')
json.dump(nmf_json, output_file, sort_keys=True, indent=2)
def RunSandboxedCompiler(use_sz):
driver_tools.CheckTranslatorPrerequisites()
infile = env.getone('input')
is_pnacl = filetype.IsPNaClBitcode(infile)
if not is_pnacl and not env.getbool('ALLOW_LLVM_BITCODE_INPUT'):
Log.Fatal('Translator expects finalized PNaCl bitcode. '
'Pass --allow-llvm-bitcode-input to override.')
threads = int(env.getone('SPLIT_MODULE'))
command = [driver_tools.SelLdrCommand(),
'-a', # Allow file access
'-E NACL_IRT_PNACL_TRANSLATOR_COMPILE_INPUT=%s' % infile]
driver_tools.AddListToEnv(command, 'NACL_IRT_PNACL_TRANSLATOR_COMPILE_OUTPUT',
GetObjectFiles(use_sz))
driver_tools.AddListToEnv(command, 'NACL_IRT_PNACL_TRANSLATOR_COMPILE_ARG',
BuildOverrideCompilerCommandLine(is_pnacl, use_sz))
command.extend(['-E NACL_IRT_PNACL_TRANSLATOR_COMPILE_THREADS=%d' % threads,
'--'])
if use_sz:
command.append('${PNACL_SZ_SB}')
else:
command.append('${LLC_SB}')
driver_tools.Run(' '.join(command),
# stdout/stderr will be automatically dumped
# upon failure
redirect_stderr=subprocess.PIPE,
redirect_stdout=subprocess.PIPE)
def CheckInputsArch(inputs):
count = 0
for f in inputs:
if ldtools.IsFlag(f):
continue
elif filetype.IsLLVMBitcode(f) or filetype.IsBitcodeArchive(f):
pass
elif filetype.IsNative(f):
ArchMerge(f, True)
else:
Log.Fatal("%s: Unexpected type of file for linking (%s)",
pathtools.touser(f), filetype.FileType(f))
count += 1
if count == 0:
Log.Fatal("no input files")
def RunLDSandboxed():
CheckTranslatorPrerequisites()
# The "main" input file is the application's combined object file.
all_inputs = env.get('inputs')
main_input = env.getone('LLC_TRANSLATED_FILE')
if not main_input:
Log.Fatal("Sandboxed LD requires one shm input file")
outfile = env.getone('output')
files = LinkerFiles(all_inputs)
ld_flags = env.get('LD_FLAGS')
script = MakeSelUniversalScriptForLD(ld_flags,
main_input,
files,
outfile)
Run('${SEL_UNIVERSAL_PREFIX} ${SEL_UNIVERSAL} ' +
'${SEL_UNIVERSAL_FLAGS} -- ${LD_SB}',
stdin_contents=script,
# stdout/stderr will be automatically dumped
# upon failure
redirect_stderr=subprocess.PIPE,
redirect_stdout=subprocess.PIPE)
def RunLLCSandboxed():
driver_tools.CheckTranslatorPrerequisites()
infile = env.getone('input')
outfile = env.getone('output')
if not driver_tools.IsPNaClBitcode(infile):
Log.Fatal('Input to sandboxed translator must be PNaCl bitcode')
script = MakeSelUniversalScriptForLLC(infile, outfile)
command = (
'${SEL_UNIVERSAL_PREFIX} ${SEL_UNIVERSAL} ${SEL_UNIVERSAL_FLAGS} '
'-- ${LLC_SB}')
_, stdout, _ = driver_tools.Run(
command,
stdin_contents=script,
# stdout/stderr will be automatically dumped
# upon failure
redirect_stderr=subprocess.PIPE,
redirect_stdout=subprocess.PIPE)
# Get the values returned from the llc RPC to use in input to ld
is_shared = re.search(r'output\s+0:\s+i\(([0|1])\)', stdout).group(1)
is_shared = (is_shared == '1')
soname = re.search(r'output\s+1:\s+s\("(.*)"\)', stdout).group(1)
needed_str = re.search(r'output\s+2:\s+s\("(.*)"\)', stdout).group(1)
# If the delimiter changes, this line needs to change
needed_libs = [lib for lib in needed_str.split(r'\n') if lib]
return is_shared, soname, needed_libs
def DriverMain(module, argv):
# TODO(robertm): this is ugly - try to get rid of this
if '--pnacl-driver-verbose' in argv:
Log.IncreaseVerbosity()
env.set('LOG_VERBOSE', '1')
# driver_path has the form: /foo/bar/pnacl_root/newlib/bin/pnacl-clang
driver_path = pathtools.abspath(pathtools.normalize(argv[0]))
driver_bin = pathtools.dirname(driver_path)
script_name = pathtools.basename(driver_path)
env.set('SCRIPT_NAME', script_name)
env.set('DRIVER_PATH', driver_path)
env.set('DRIVER_BIN', driver_bin)
Log.SetScriptName(script_name)
ReadConfig()
if IsWindowsPython():
SetupCygwinLibs()
# skip tool name
argv = argv[1:]
# Handle help info
if ('--help' in argv or '-h' in argv or '-help' in argv
or '--help-full' in argv):
help_func = getattr(module, 'get_help', None)
if not help_func:
Log.Fatal(HelpNotAvailable())
helpstr = help_func(argv)
print helpstr
return 0
return module.main(argv)
def main(argv):
env.update(EXTRA_ENV)
driver_tools.ParseArgs(argv, PATTERNS)
inputs = env.get('INPUTS')
if len(inputs) == 0:
Log.Fatal("No input files given")
for infile in inputs:
env.push()
env.set('input', infile)
# For frozen PNaCl bitcode, use 'llvm-nm -bitcode-format=pnacl'. For all
# other formats, use the binutils nm with our gold plugin.
if filetype.IsPNaClBitcode(infile):
env.set('TOOLNAME', '${LLVM_NM}')
env.append('FLAGS', '-bitcode-format=pnacl')
else:
env.set('TOOLNAME', '${NM}')
env.append('FLAGS', '--plugin=${GOLD_PLUGIN_SO}')
driver_tools.Run('"${TOOLNAME}" ${FLAGS} ${input}')
env.pop()
# only reached in case of no errors
return 0
def GetBuildOS():
name = platform.system().lower()
if name.startswith('cygwin_nt') or 'windows' in name:
name = 'windows'
if name not in ('linux', 'darwin', 'windows'):
Log.Fatal("Unsupported platform '%s'", name)
return name
def CompileHeaders(header_inputs, output):
if output != '' and len(header_inputs) > 1:
Log.Fatal('Cannot have -o <out> and compile multiple header files: %s',
repr(header_inputs))
for f in header_inputs:
f_output = output if output else DefaultPCHOutputName(f)
RunCC(f, f_output, mode='', emit_llvm_flag='')
def main(argv):
env.update(EXTRA_ENV)
driver_tools.ParseArgs(argv, PrepPatterns)
inputs = env.get('INPUTS')
output = env.getone('OUTPUT')
if len(inputs) != 1:
Log.Fatal('Can only have one input')
f_input = inputs[0]
# Allow in-place file changes if output isn't specified..
if output != '':
f_output = output
else:
f_output = f_input
if env.getbool('DISABLE_FINALIZE'):
# Just copy the input file to the output file.
if f_input != f_output:
shutil.copyfile(f_input, f_output)
return 0
# Transform the file, and convert it to a PNaCl bitcode file.
driver_tools.RunWithEnv(' '.join(['${RUN_OPT}', '--bitcode-format=pnacl']),
input=inputs[0],
output=f_output)
return 0
def setmany(self, **kwargs):
for k,v in kwargs.iteritems():
if isinstance(v, types.StringTypes):
self.set(k, v)
elif isinstance(v, types.ListType):
self.set(k, *v)
else:
Log.Fatal('env.setmany given a non-string and non-list value')
def FindBaseToolchain():
""" Find toolchain/OS_ARCH directory """
base_dir = FindBaseDir(lambda cur: pathtools.basename(cur) == 'toolchain')
if base_dir is None:
Log.Fatal("Unable to find 'toolchain' directory")
toolchain_dir = os.path.join(base_dir,
'%s_%s' % (GetOSName(), GetArchNameShort()))
return shell.escape(toolchain_dir)
def GetArch(required = False):
arch = env.getone('ARCH')
if arch == '':
arch = None
if required and not arch:
Log.Fatal('Missing -arch!')
return arch
def LinkerFiles(args):
ret = []
for f in args:
if IsFlag(f):
continue
else:
if not pathtools.exists(f):
Log.Fatal("Unable to open '%s'", pathtools.touser(f))
ret.append(f)
return ret
def RunTranslate(infile, output, mode):
if not env.getbool('ALLOW_TRANSLATE'):
Log.Fatal('%s: Trying to convert bitcode to an object file before '
'bitcode linking. This is supposed to wait until '
'translation. Use --pnacl-allow-translate to override.',
pathtools.touser(infile))
args = env.get('TRANSLATE_FLAGS') + [mode, infile, '-o', output]
if env.getbool('PIC'):
args += ['-fPIC']
RunDriver('translate', args)
def GetNativeLibsDirname(other_inputs):
"""Check that native libs have a common directory and return the directory."""
dirname = None
for f in other_inputs:
if IsFlag(f):
continue
else:
if not pathtools.exists(f):
Log.Fatal("Unable to open '%s'", pathtools.touser(f))
if dirname is None:
dirname = pathtools.dirname(f)
else:
if dirname != pathtools.dirname(f):
Log.Fatal(
'Need a common directory for native libs: %s != %s',
dirname, pathtools.dirname(f))
if not dirname:
Log.Fatal('No native libraries found')
return dirname + '/'
def GetBuildArch():
m = platform.machine()
# Windows is special
if m == 'x86':
m = 'i686'
if m not in ('i386', 'i686', 'x86_64'):
Log.Fatal("Unsupported architecture '%s'", m)
return m
def Run(args,
errexit=True,
redirect_stdout=None,
redirect_stderr=None):
""" Run: Run a command.
Returns: return_code, stdout, stderr
Run() is used to invoke "other" tools, e.g.
those NOT prefixed with "pnacl-"
stdout and stderr only contain meaningful data if
redirect_{stdout,stderr} == subprocess.PIPE
Run will terminate the program upon failure unless errexit == False
TODO(robertm): errexit == True has not been tested and needs more work
redirect_stdout and redirect_stderr are passed straight
to subprocess.Popen
"""
result_stdout = None
result_stderr = None
if isinstance(args, str):
args = shell.split(env.eval(args))
args = [pathtools.tosys(args[0])] + args[1:]
Log.Info('Running: ' + StringifyCommand(args))
if env.getbool('DRY_RUN'):
if redirect_stderr or redirect_stdout:
# TODO(pdox): Prevent this from happening, so that
# dry-run is more useful.
Log.Fatal("Unhandled dry-run case.")
return 0, None, None
try:
# If we have too long of a cmdline on windows, running it would fail.
# Attempt to use a file with the command line options instead in that case.
if ArgsTooLongForWindows(args):
actual_args = ConvertArgsToFile(args)
Log.Info('Wrote long commandline to file for Windows: ' +
StringifyCommand(actual_args))
else:
actual_args = args
p = subprocess.Popen(actual_args,
stdout=redirect_stdout,
stderr=redirect_stderr)
result_stdout, result_stderr = p.communicate()
except Exception, e:
msg = '%s\nCommand was: %s' % (str(e), StringifyCommand(args))
print(msg)
DriverExit(1)
def CheckTranslatorPrerequisites():
""" Assert that the scons artifacts for running the sandboxed translator
exist: sel_universal, and sel_ldr. """
reqs = ['SEL_UNIVERSAL', 'SEL_LDR']
# Linux also requires the nacl bootstrap helper.
if GetBuildOS() == 'linux':
reqs.append('BOOTSTRAP_LDR')
for var in reqs:
needed_file = env.getone(var)
if not pathtools.exists(needed_file):
Log.Fatal('Could not find %s [%s]', var, needed_file)
def ReadDriverRevision():
rev_file = env.getone('DRIVER_REV_FILE')
nacl_ver = DriverOpen(rev_file, 'rb').readlines()[0]
m = re.search(r'\[GIT\].*/native_client(?:\.git)?:\s*([0-9a-f]{40})',
nacl_ver)
if m:
return m.group(1)
# fail-fast: if the REV file exists but regex search failed,
# we need to fix the regex to get nacl-version.
if not m:
Log.Fatal('Failed to parse REV file to get nacl-version.')