def ConvertArgsToFile(args):
fd, outfile = tempfile.mkstemp()
# Remember to delete this file afterwards.
TempFiles.add(outfile)
cmd = args[0]
other_args = args[1:]
os.write(fd, ' '.join(other_args))
os.close(fd)
return [cmd, '@' + outfile]
def ConvertArgsToFile(args): fd, outfile = tempfile.mkstemp() # Remember to delete this file afterwards. TempFiles.add(outfile) cmd = args[0] other_args = args[1:] os.write(fd, ' '.join(other_args)) os.close(fd) return [cmd, '@' + outfile]
def TempNameForInput(self, input, imtype):
fullpath = pathtools.abspath(input)
# If input is already a temporary name, just add an extension
if fullpath.startswith(self.TempBase):
temp = fullpath + '.' + imtype
else:
# Source file
temp = self.TempMap[fullpath] + '.' + imtype
TempFiles.add(temp)
return temp
def TempNameForInput(self, input, imtype):
fullpath = pathtools.abspath(input)
# If input is already a temporary name, just add an extension
if fullpath.startswith(self.TempBase):
temp = fullpath + '.' + imtype
else:
# Source file
temp = self.TempMap[fullpath] + '.' + imtype
TempFiles.add(temp)
return temp
def TempNameForInput(self, input, imtype):
# If input is already a temporary name, just change the extension
if input.startswith(self.TempBase):
temp = self.TempBase + '.' + imtype
else:
# Source file
temp = self.TempMap[input] + '.' + imtype
temp = self.ValidatePathLength(temp, imtype)
TempFiles.add(temp)
return temp
def TempNameForInput(self, input, imtype):
fullpath = pathtools.abspath(input)
# If input is already a temporary name, just change the extension
if fullpath.startswith(self.TempBase):
temp = self.TempBase + '.' + imtype
else:
# Source file
temp = self.TempMap[fullpath] + '.' + imtype
temp = self.ValidatePathLength(temp, imtype)
TempFiles.add(temp)
return temp
def __init__(self, inputs, output):
self.TempBase = tempfile.mkdtemp()
inputs = [ pathtools.abspath(i) for i in inputs ]
output = pathtools.basename(output)
TempFiles.add(self.TempBase)
self.Output = output + '---linked'
# Build the initial mapping
self.TempMap = dict()
for f in inputs:
if f.startswith('-'):
continue
path = PathSplit(f)
self.TempMap[f] = [1, path]
def MangledName(path):
return output + '---' + '_'.join(path[-n:]) + '---'
while True:
# Find conflicts
ConflictMap = dict()
Conflicts = set()
for (f, [n, path]) in self.TempMap.iteritems():
candidate = pathtools.abspath(MangledName(path))
if candidate in ConflictMap:
Conflicts.add(ConflictMap[candidate])
Conflicts.add(f)
else:
ConflictMap[candidate] = f
if len(Conflicts) == 0:
break
# Resolve conflicts
for f in Conflicts:
n = self.TempMap[f][0]
if n+1 > len(self.TempMap[f][1]):
Log.Fatal('Unable to resolve naming conflicts')
self.TempMap[f][0] = n+1
# Clean up the map and put the paths in tempdir
NewMap = dict()
for (f, [n, path]) in self.TempMap.iteritems():
NewMap[f] = os.path.join(self.TempBase, MangledName(path))
self.TempMap = NewMap
return
def __init__(self, inputs, output):
self.TempBase = tempfile.mkdtemp()
inputs = [pathtools.abspath(i) for i in inputs]
output = pathtools.basename(output)
TempFiles.add(self.TempBase)
self.Output = output + '---linked'
# Build the initial mapping
self.TempMap = dict()
for f in inputs:
if f.startswith('-'):
continue
path = PathSplit(f)
self.TempMap[f] = [1, path]
def MangledName(path):
return output + '---' + '_'.join(path[-n:]) + '---'
while True:
# Find conflicts
ConflictMap = dict()
Conflicts = set()
for (f, [n, path]) in self.TempMap.iteritems():
candidate = pathtools.abspath(MangledName(path))
if candidate in ConflictMap:
Conflicts.add(ConflictMap[candidate])
Conflicts.add(f)
else:
ConflictMap[candidate] = f
if len(Conflicts) == 0:
break
# Resolve conflicts
for f in Conflicts:
n = self.TempMap[f][0]
if n + 1 > len(self.TempMap[f][1]):
Log.Fatal('Unable to resolve naming conflicts')
self.TempMap[f][0] = n + 1
# Clean up the map and put the paths in tempdir
NewMap = dict()
for (f, [n, path]) in self.TempMap.iteritems():
NewMap[f] = os.path.join(self.TempBase, MangledName(path))
self.TempMap = NewMap
return
def TempNameForOutput(self, imtype):
temp = self.TempBase + '.' + imtype
TempFiles.add(temp)
return temp
def main(argv):
env.update(EXTRA_ENV)
driver_tools.ParseArgs(argv, TranslatorPatterns)
driver_tools.GetArch(required = True)
SetUpArch()
SetUpLinkOptions()
# Now commit to whether or not Subzero is used.
use_sz = env.getbool('USE_SZ') and not env.getbool('SZ_UNSUPPORTED')
inputs = env.get('INPUTS')
output = env.getone('OUTPUT')
if len(inputs) == 0:
Log.Fatal("No input files")
for path in inputs:
driver_tools.CheckPathLength(path)
if output == '':
Log.Fatal("Please specify output file with -o")
# Find the bitcode file on the command line.
bcfiles = [f for f in inputs
if not ldtools.IsFlag(f) and
(filetype.IsPNaClBitcode(f)
or filetype.IsLLVMBitcode(f)
or filetype.FileType(f) == 'll')]
if len(bcfiles) > 1:
Log.Fatal('Expecting at most 1 bitcode file')
elif len(bcfiles) == 1:
bcfile = bcfiles[0]
else:
bcfile = None
if ((env.getbool('ALLOW_LLVM_BITCODE_INPUT') or
env.getone('TARGET_OS') != 'nacl' or
env.getbool('USE_EMULATOR')) and
env.getone('SPLIT_MODULE') == 'auto'):
# When llvm input is allowed, the pexe may not be ABI-stable, so do not
# split it. Non-ABI-stable pexes may have symbol naming and visibility
# issues that the current splitting scheme doesn't account for.
#
# For now, also do not enable multi-threaded translation when TARGET_OS !=
# 'nacl', since in these cases we will be using the host toolchain's
# linker.
#
# The x86->arm emulator is very flaky when threading is used, so don't
# do module splitting when using it.
env.set('SPLIT_MODULE', 'seq')
# Do not set -streaming-bitcode for sandboxed mode, because it is already
# in the default command line.
if not env.getbool('SANDBOXED') and env.getbool('STREAM_BITCODE'):
env.append('LLC_FLAGS_EXTRA', '-streaming-bitcode')
if env.getone('SPLIT_MODULE') == 'seq':
env.set('SPLIT_MODULE', '1')
env.set('SZ_THREADS', '0')
elif env.getone('SPLIT_MODULE') == 'auto':
try:
num_modules = min(4, multiprocessing.cpu_count())
except NotImplementedError:
num_modules = 2
env.set('SPLIT_MODULE', str(num_modules))
env.set('SZ_THREADS', str(num_modules))
else:
num_modules = int(env.getone('SPLIT_MODULE'))
if num_modules < 1:
Log.Fatal('Value given for -split-module must be > 0')
env.set('SPLIT_MODULE', str(num_modules))
env.set('SZ_THREADS', str(num_modules))
modules = env.getone('SPLIT_MODULE')
module_sched = env.getone('SPLIT_MODULE_SCHED')
sz_threads = env.getone('SZ_THREADS')
# TODO(dschuff,jvoung): No need to specify -split-module=X since the IPC
# already has a parameter for the number of threads and modules.
env.append('LLC_FLAGS_EXTRA', '-split-module=' + modules)
env.append('LD_FLAGS', '-split-module=' + ('1' if use_sz else modules))
env.append('LLC_FLAGS_EXTRA', '-split-module-sched=' + module_sched)
# In sandboxed mode, the IPC already has a parameter for the number
# of threads, so no need to specify that again via argv[].
if not env.getbool('SANDBOXED'):
env.append('SZ_FLAGS_EXTRA', '--threads=' + sz_threads)
# If there's a bitcode file, translate it now.
tng = driver_tools.TempNameGen(inputs + bcfiles, output)
output_type = env.getone('OUTPUT_TYPE')
if bcfile:
sfile = None
if output_type == 's':
sfile = output
ofile = None
if output_type == 'o':
ofile = output
elif output_type != 's':
ofile = tng.TempNameForInput(bcfile, 'o')
if sfile:
RunCompiler(bcfile, sfile, outfiletype='asm', use_sz=use_sz)
if ofile:
RunAS(sfile, ofile)
else:
RunCompiler(bcfile, ofile, outfiletype='obj', use_sz=use_sz)
else:
ofile = None
# If we've been told to stop after translation, stop now.
if output_type in ('o','s'):
return 0
if use_sz:
# Reset SPLIT_MODULE to 1 to fall back to normal linking behavior.
env.set('SPLIT_MODULE', '1')
# Replace the bitcode file with __BITCODE__ in the input list
if bcfile:
inputs = ListReplace(inputs, bcfile, '__BITCODE__')
env.set('INPUTS', *inputs)
if int(env.getone('SPLIT_MODULE')) > 1:
modules = int(env.getone('SPLIT_MODULE'))
for i in range(1, modules):
filename = ofile + '.module%d' % i
TempFiles.add(filename)
env.append('INPUTS', filename)
if env.getone('TARGET_OS') != 'nacl':
RunHostLD(ofile, output)
else:
RunLD(ofile, output)
return 0
def TempNameForOutput(self, imtype): temp = self.ValidatePathLength(self.TempBase + '.' + imtype, imtype) TempFiles.add(temp) return temp
def TempNameForOutput(self, imtype): temp = self.TempBase + '.' + imtype TempFiles.add(temp) return temp
def main(argv):
env.update(EXTRA_ENV)
driver_tools.ParseArgs(argv, TranslatorPatterns)
driver_tools.GetArch(required = True)
SetUpArch()
SetUpLinkOptions()
# Now commit to whether or not Subzero is used.
use_sz = env.getbool('USE_SZ') and not env.getbool('SZ_UNSUPPORTED')
inputs = env.get('INPUTS')
output = env.getone('OUTPUT')
if len(inputs) == 0:
Log.Fatal("No input files")
for path in inputs:
driver_tools.CheckPathLength(path)
if output == '':
Log.Fatal("Please specify output file with -o")
# Find the bitcode file on the command line.
bcfiles = [f for f in inputs
if not ldtools.IsFlag(f) and
(filetype.IsPNaClBitcode(f)
or filetype.IsLLVMBitcode(f)
or filetype.FileType(f) == 'll')]
if len(bcfiles) > 1:
Log.Fatal('Expecting at most 1 bitcode file')
elif len(bcfiles) == 1:
bcfile = bcfiles[0]
else:
bcfile = None
if ((env.getbool('ALLOW_LLVM_BITCODE_INPUT') or
env.getone('TARGET_OS') != 'nacl' or
env.getbool('USE_EMULATOR')) and
env.getone('SPLIT_MODULE') == 'auto'):
# When llvm input is allowed, the pexe may not be ABI-stable, so do not
# split it. Non-ABI-stable pexes may have symbol naming and visibility
# issues that the current splitting scheme doesn't account for.
#
# For now, also do not enable multi-threaded translation when TARGET_OS !=
# 'nacl', since in these cases we will be using the host toolchain's
# linker.
#
# The x86->arm emulator is very flaky when threading is used, so don't
# do module splitting when using it.
env.set('SPLIT_MODULE', 'seq')
# Do not set -streaming-bitcode for sandboxed mode, because it is already
# in the default command line.
if not env.getbool('SANDBOXED') and env.getbool('STREAM_BITCODE'):
env.append('LLC_FLAGS_EXTRA', '-streaming-bitcode')
if env.getone('SPLIT_MODULE') == 'seq':
env.set('SPLIT_MODULE', '1')
env.set('SZ_THREADS', '0')
elif env.getone('SPLIT_MODULE') == 'auto':
try:
num_modules = min(4, multiprocessing.cpu_count())
except NotImplementedError:
num_modules = 2
env.set('SPLIT_MODULE', str(num_modules))
env.set('SZ_THREADS', str(num_modules))
else:
num_modules = int(env.getone('SPLIT_MODULE'))
if num_modules < 1:
Log.Fatal('Value given for -split-module must be > 0')
env.set('SPLIT_MODULE', str(num_modules))
env.set('SZ_THREADS', str(num_modules))
modules = env.getone('SPLIT_MODULE')
module_sched = env.getone('SPLIT_MODULE_SCHED')
sz_threads = env.getone('SZ_THREADS')
# TODO(dschuff,jvoung): No need to specify -split-module=X since the IPC
# already has a parameter for the number of threads and modules.
env.append('LLC_FLAGS_EXTRA', '-split-module=' + modules)
env.append('LD_FLAGS', '-split-module=' + ('1' if use_sz else modules))
env.append('LLC_FLAGS_EXTRA', '-split-module-sched=' + module_sched)
# In sandboxed mode, the IPC already has a parameter for the number
# of threads, so no need to specify that again via argv[].
if not env.getbool('SANDBOXED'):
env.append('SZ_FLAGS_EXTRA', '--threads=' + sz_threads)
# If there's a bitcode file, translate it now.
tng = driver_tools.TempNameGen(inputs + bcfiles, output)
output_type = env.getone('OUTPUT_TYPE')
if bcfile:
sfile = None
if output_type == 's':
sfile = output
ofile = None
if output_type == 'o':
ofile = output
elif output_type != 's':
ofile = tng.TempNameForInput(bcfile, 'o')
if sfile:
RunCompiler(bcfile, sfile, outfiletype='asm', use_sz=use_sz)
if ofile:
RunAS(sfile, ofile)
else:
RunCompiler(bcfile, ofile, outfiletype='obj', use_sz=use_sz)
else:
ofile = None
# If we've been told to stop after translation, stop now.
if output_type in ('o','s'):
return 0
if use_sz:
# Reset SPLIT_MODULE to 1 to fall back to normal linking behavior.
env.set('SPLIT_MODULE', '1')
# Replace the bitcode file with __BITCODE__ in the input list
if bcfile:
inputs = ListReplace(inputs, bcfile, '__BITCODE__')
env.set('INPUTS', *inputs)
if int(env.getone('SPLIT_MODULE')) > 1:
modules = int(env.getone('SPLIT_MODULE'))
for i in range(1, modules):
filename = ofile + '.module%d' % i
TempFiles.add(filename)
env.append('INPUTS', filename)
if env.getone('TARGET_OS') != 'nacl':
RunHostLD(ofile, output)
else:
RunLD(ofile, output)
return 0
def main(argv):
env.update(EXTRA_ENV)
driver_tools.ParseArgs(argv, TranslatorPatterns)
driver_tools.GetArch(required = True)
SetUpArch()
inputs = env.get('INPUTS')
output = env.getone('OUTPUT')
if len(inputs) == 0:
Log.Fatal("No input files")
if output == '':
Log.Fatal("Please specify output file with -o")
# Find the bitcode file on the command line.
bcfiles = [f for f in inputs
if not ldtools.IsFlag(f) and
(filetype.IsPNaClBitcode(f)
or filetype.IsLLVMBitcode(f)
or filetype.FileType(f) == 'll')]
if len(bcfiles) > 1:
Log.Fatal('Expecting at most 1 bitcode file')
elif len(bcfiles) == 1:
bcfile = bcfiles[0]
else:
bcfile = None
if not env.getbool('SPLIT_MODULE'):
try:
env.set('SPLIT_MODULE', str(min(4, multiprocessing.cpu_count())))
except NotImplementedError:
env.set('SPLIT_MODULE', '2')
elif int(env.getone('SPLIT_MODULE')) < 1:
Log.Fatal('Value given for -split-module must be > 0')
if (env.getbool('ALLOW_LLVM_BITCODE_INPUT') or
env.getone('TARGET_OS') != 'nacl' or
env.getbool('USE_EMULATOR')):
# When llvm input is allowed, the pexe may not be ABI-stable, so do not
# split it. Non-ABI-stable pexes may have symbol naming and visibility
# issues that the current splitting scheme doesn't account for.
#
# For now, also do not enable multi-threaded translation when TARGET_OS !=
# 'nacl', since in these cases we will be using the host toolchain's
# linker.
#
# The x86->arm emulator is very flaky when threading is used, so don't
# do module splitting when using it.
env.set('SPLIT_MODULE', '1')
else:
modules = env.getone('SPLIT_MODULE')
if modules != '1':
env.append('LLC_FLAGS_EXTRA', '-split-module=' + modules)
env.append('LD_FLAGS', '-split-module=' + modules)
if not env.getbool('SANDBOXED') and env.getbool('STREAM_BITCODE'):
# Do not set -streaming-bitcode for sandboxed mode, because it is already
# in the default command line.
env.append('LLC_FLAGS_EXTRA', '-streaming-bitcode')
# If there's a bitcode file, translate it now.
tng = driver_tools.TempNameGen(inputs + bcfiles, output)
output_type = env.getone('OUTPUT_TYPE')
if bcfile:
sfile = None
if output_type == 's':
sfile = output
ofile = None
if output_type == 'o':
ofile = output
elif output_type != 's':
ofile = tng.TempNameForInput(bcfile, 'o')
if sfile:
RunLLC(bcfile, sfile, outfiletype='asm')
if ofile:
RunAS(sfile, ofile)
else:
RunLLC(bcfile, ofile, outfiletype='obj')
else:
ofile = None
# If we've been told to stop after translation, stop now.
if output_type in ('o','s'):
return 0
# Replace the bitcode file with __BITCODE__ in the input list
if bcfile:
inputs = ListReplace(inputs, bcfile, '__BITCODE__')
env.set('INPUTS', *inputs)
if int(env.getone('SPLIT_MODULE')) > 1:
modules = int(env.getone('SPLIT_MODULE'))
for i in range(1, modules):
filename = ofile + '.module%d' % i
TempFiles.add(filename)
env.append('INPUTS', filename)
if env.getone('TARGET_OS') != 'nacl':
RunHostLD(ofile, output)
else:
RunLD(ofile, output)
return 0
def TempNameForOutput(self, imtype): temp = os.path.join(self.TempBase, self.Output + '.' + imtype) TempFiles.add(temp) return temp
def TempNameForOutput(self, imtype):
temp = os.path.join(self.TempBase, self.Output + '.' + imtype)
TempFiles.add(temp)
return temp
def main(argv):
env.update(EXTRA_ENV)
driver_tools.ParseArgs(argv, TranslatorPatterns)
driver_tools.GetArch(required=True)
inputs = env.get('INPUTS')
output = env.getone('OUTPUT')
if len(inputs) == 0:
Log.Fatal("No input files")
if output == '':
Log.Fatal("Please specify output file with -o")
# Find the bitcode file on the command line.
bcfiles = [
f for f in inputs
if not ldtools.IsFlag(f) and (filetype.IsPNaClBitcode(
f) or filetype.IsLLVMBitcode(f) or filetype.FileType(f) == 'll')
]
if len(bcfiles) > 1:
Log.Fatal('Expecting at most 1 bitcode file')
elif len(bcfiles) == 1:
bcfile = bcfiles[0]
else:
bcfile = None
if not env.getbool('SPLIT_MODULE'):
try:
env.set('SPLIT_MODULE', str(min(4, multiprocessing.cpu_count())))
except NotImplementedError:
env.set('SPLIT_MODULE', '2')
elif int(env.getone('SPLIT_MODULE')) < 1:
Log.Fatal('Value given for -split-module must be > 0')
if (env.getbool('ALLOW_LLVM_BITCODE_INPUT')
or env.getone('ARCH') == 'X8632_LINUX'
or env.getbool('USE_EMULATOR')):
# When llvm input is allowed, the pexe may not be ABI-stable, so do not
# split it. For now also do not support threading non-SFI baremetal mode.
# Non-ABI-stable pexes may have symbol naming and visibility issues that the
# current splitting scheme doesn't account for.
# If the link would require a non-standard command line, do not split the
# modules because the sandboxed linker doesn't support that combination.
# The x86->arm emulator is very flaky when threading is used, so don't
# do module splitting when using it.
env.set('SPLIT_MODULE', '1')
else:
modules = env.getone('SPLIT_MODULE')
if modules != '1':
env.append('LLC_FLAGS_EXTRA', '-split-module=' + modules)
env.append('LD_FLAGS', '-split-module=' + modules)
if not env.getbool('SANDBOXED') and env.getbool('STREAM_BITCODE'):
# Do not set -streaming-bitcode for sandboxed mode, because it is already
# in the default command line.
env.append('LLC_FLAGS_EXTRA', '-streaming-bitcode')
# If there's a bitcode file, translate it now.
tng = driver_tools.TempNameGen(inputs + bcfiles, output)
output_type = env.getone('OUTPUT_TYPE')
if bcfile:
sfile = None
if output_type == 's':
sfile = output
ofile = None
if output_type == 'o':
ofile = output
elif output_type != 's':
ofile = tng.TempNameForInput(bcfile, 'o')
if sfile:
RunLLC(bcfile, sfile, outfiletype='asm')
if ofile:
RunAS(sfile, ofile)
else:
RunLLC(bcfile, ofile, outfiletype='obj')
else:
ofile = None
# If we've been told to stop after translation, stop now.
if output_type in ('o', 's'):
return 0
# Replace the bitcode file with __BITCODE__ in the input list
if bcfile:
inputs = ListReplace(inputs, bcfile, '__BITCODE__')
env.set('INPUTS', *inputs)
if int(env.getone('SPLIT_MODULE')) > 1:
modules = int(env.getone('SPLIT_MODULE'))
for i in range(1, modules):
filename = ofile + '.module%d' % i
TempFiles.add(filename)
env.append('INPUTS', filename)
if env.getone('ARCH') == 'X8632_LINUX':
RunHostLD(ofile, output)
else:
RunLD(ofile, output)
return 0
def TempNameForOutput(self, imtype):
temp = self.ValidatePathLength(self.TempBase + '.' + imtype, imtype)
TempFiles.add(temp)
return temp