def ConvertArgsToFile(args): fd, outfile = tempfile.mkstemp() # Remember to delete this file afterwards. TempFiles.add(outfile) cmd = args[0] other_args = args[1:] os.write(fd, ' '.join(other_args)) os.close(fd) return [cmd, '@' + outfile]
def TempNameForInput(self, input, imtype): fullpath = pathtools.abspath(input) # If input is already a temporary name, just add an extension if fullpath.startswith(self.TempBase): temp = fullpath + '.' + imtype else: # Source file temp = self.TempMap[fullpath] + '.' + imtype TempFiles.add(temp) return temp
def TempNameForInput(self, input, imtype): # If input is already a temporary name, just change the extension if input.startswith(self.TempBase): temp = self.TempBase + '.' + imtype else: # Source file temp = self.TempMap[input] + '.' + imtype temp = self.ValidatePathLength(temp, imtype) TempFiles.add(temp) return temp
def TempNameForInput(self, input, imtype): fullpath = pathtools.abspath(input) # If input is already a temporary name, just change the extension if fullpath.startswith(self.TempBase): temp = self.TempBase + '.' + imtype else: # Source file temp = self.TempMap[fullpath] + '.' + imtype temp = self.ValidatePathLength(temp, imtype) TempFiles.add(temp) return temp
def __init__(self, inputs, output): self.TempBase = tempfile.mkdtemp() inputs = [ pathtools.abspath(i) for i in inputs ] output = pathtools.basename(output) TempFiles.add(self.TempBase) self.Output = output + '---linked' # Build the initial mapping self.TempMap = dict() for f in inputs: if f.startswith('-'): continue path = PathSplit(f) self.TempMap[f] = [1, path] def MangledName(path): return output + '---' + '_'.join(path[-n:]) + '---' while True: # Find conflicts ConflictMap = dict() Conflicts = set() for (f, [n, path]) in self.TempMap.iteritems(): candidate = pathtools.abspath(MangledName(path)) if candidate in ConflictMap: Conflicts.add(ConflictMap[candidate]) Conflicts.add(f) else: ConflictMap[candidate] = f if len(Conflicts) == 0: break # Resolve conflicts for f in Conflicts: n = self.TempMap[f][0] if n+1 > len(self.TempMap[f][1]): Log.Fatal('Unable to resolve naming conflicts') self.TempMap[f][0] = n+1 # Clean up the map and put the paths in tempdir NewMap = dict() for (f, [n, path]) in self.TempMap.iteritems(): NewMap[f] = os.path.join(self.TempBase, MangledName(path)) self.TempMap = NewMap return
def __init__(self, inputs, output): self.TempBase = tempfile.mkdtemp() inputs = [pathtools.abspath(i) for i in inputs] output = pathtools.basename(output) TempFiles.add(self.TempBase) self.Output = output + '---linked' # Build the initial mapping self.TempMap = dict() for f in inputs: if f.startswith('-'): continue path = PathSplit(f) self.TempMap[f] = [1, path] def MangledName(path): return output + '---' + '_'.join(path[-n:]) + '---' while True: # Find conflicts ConflictMap = dict() Conflicts = set() for (f, [n, path]) in self.TempMap.iteritems(): candidate = pathtools.abspath(MangledName(path)) if candidate in ConflictMap: Conflicts.add(ConflictMap[candidate]) Conflicts.add(f) else: ConflictMap[candidate] = f if len(Conflicts) == 0: break # Resolve conflicts for f in Conflicts: n = self.TempMap[f][0] if n + 1 > len(self.TempMap[f][1]): Log.Fatal('Unable to resolve naming conflicts') self.TempMap[f][0] = n + 1 # Clean up the map and put the paths in tempdir NewMap = dict() for (f, [n, path]) in self.TempMap.iteritems(): NewMap[f] = os.path.join(self.TempBase, MangledName(path)) self.TempMap = NewMap return
def TempNameForOutput(self, imtype): temp = self.TempBase + '.' + imtype TempFiles.add(temp) return temp
def main(argv): env.update(EXTRA_ENV) driver_tools.ParseArgs(argv, TranslatorPatterns) driver_tools.GetArch(required = True) SetUpArch() SetUpLinkOptions() # Now commit to whether or not Subzero is used. use_sz = env.getbool('USE_SZ') and not env.getbool('SZ_UNSUPPORTED') inputs = env.get('INPUTS') output = env.getone('OUTPUT') if len(inputs) == 0: Log.Fatal("No input files") for path in inputs: driver_tools.CheckPathLength(path) if output == '': Log.Fatal("Please specify output file with -o") # Find the bitcode file on the command line. bcfiles = [f for f in inputs if not ldtools.IsFlag(f) and (filetype.IsPNaClBitcode(f) or filetype.IsLLVMBitcode(f) or filetype.FileType(f) == 'll')] if len(bcfiles) > 1: Log.Fatal('Expecting at most 1 bitcode file') elif len(bcfiles) == 1: bcfile = bcfiles[0] else: bcfile = None if ((env.getbool('ALLOW_LLVM_BITCODE_INPUT') or env.getone('TARGET_OS') != 'nacl' or env.getbool('USE_EMULATOR')) and env.getone('SPLIT_MODULE') == 'auto'): # When llvm input is allowed, the pexe may not be ABI-stable, so do not # split it. Non-ABI-stable pexes may have symbol naming and visibility # issues that the current splitting scheme doesn't account for. # # For now, also do not enable multi-threaded translation when TARGET_OS != # 'nacl', since in these cases we will be using the host toolchain's # linker. # # The x86->arm emulator is very flaky when threading is used, so don't # do module splitting when using it. env.set('SPLIT_MODULE', 'seq') # Do not set -streaming-bitcode for sandboxed mode, because it is already # in the default command line. if not env.getbool('SANDBOXED') and env.getbool('STREAM_BITCODE'): env.append('LLC_FLAGS_EXTRA', '-streaming-bitcode') if env.getone('SPLIT_MODULE') == 'seq': env.set('SPLIT_MODULE', '1') env.set('SZ_THREADS', '0') elif env.getone('SPLIT_MODULE') == 'auto': try: num_modules = min(4, multiprocessing.cpu_count()) except NotImplementedError: num_modules = 2 env.set('SPLIT_MODULE', str(num_modules)) env.set('SZ_THREADS', str(num_modules)) else: num_modules = int(env.getone('SPLIT_MODULE')) if num_modules < 1: Log.Fatal('Value given for -split-module must be > 0') env.set('SPLIT_MODULE', str(num_modules)) env.set('SZ_THREADS', str(num_modules)) modules = env.getone('SPLIT_MODULE') module_sched = env.getone('SPLIT_MODULE_SCHED') sz_threads = env.getone('SZ_THREADS') # TODO(dschuff,jvoung): No need to specify -split-module=X since the IPC # already has a parameter for the number of threads and modules. env.append('LLC_FLAGS_EXTRA', '-split-module=' + modules) env.append('LD_FLAGS', '-split-module=' + ('1' if use_sz else modules)) env.append('LLC_FLAGS_EXTRA', '-split-module-sched=' + module_sched) # In sandboxed mode, the IPC already has a parameter for the number # of threads, so no need to specify that again via argv[]. if not env.getbool('SANDBOXED'): env.append('SZ_FLAGS_EXTRA', '--threads=' + sz_threads) # If there's a bitcode file, translate it now. tng = driver_tools.TempNameGen(inputs + bcfiles, output) output_type = env.getone('OUTPUT_TYPE') if bcfile: sfile = None if output_type == 's': sfile = output ofile = None if output_type == 'o': ofile = output elif output_type != 's': ofile = tng.TempNameForInput(bcfile, 'o') if sfile: RunCompiler(bcfile, sfile, outfiletype='asm', use_sz=use_sz) if ofile: RunAS(sfile, ofile) else: RunCompiler(bcfile, ofile, outfiletype='obj', use_sz=use_sz) else: ofile = None # If we've been told to stop after translation, stop now. if output_type in ('o','s'): return 0 if use_sz: # Reset SPLIT_MODULE to 1 to fall back to normal linking behavior. env.set('SPLIT_MODULE', '1') # Replace the bitcode file with __BITCODE__ in the input list if bcfile: inputs = ListReplace(inputs, bcfile, '__BITCODE__') env.set('INPUTS', *inputs) if int(env.getone('SPLIT_MODULE')) > 1: modules = int(env.getone('SPLIT_MODULE')) for i in range(1, modules): filename = ofile + '.module%d' % i TempFiles.add(filename) env.append('INPUTS', filename) if env.getone('TARGET_OS') != 'nacl': RunHostLD(ofile, output) else: RunLD(ofile, output) return 0
def TempNameForOutput(self, imtype): temp = self.ValidatePathLength(self.TempBase + '.' + imtype, imtype) TempFiles.add(temp) return temp
def main(argv): env.update(EXTRA_ENV) driver_tools.ParseArgs(argv, TranslatorPatterns) driver_tools.GetArch(required = True) SetUpArch() inputs = env.get('INPUTS') output = env.getone('OUTPUT') if len(inputs) == 0: Log.Fatal("No input files") if output == '': Log.Fatal("Please specify output file with -o") # Find the bitcode file on the command line. bcfiles = [f for f in inputs if not ldtools.IsFlag(f) and (filetype.IsPNaClBitcode(f) or filetype.IsLLVMBitcode(f) or filetype.FileType(f) == 'll')] if len(bcfiles) > 1: Log.Fatal('Expecting at most 1 bitcode file') elif len(bcfiles) == 1: bcfile = bcfiles[0] else: bcfile = None if not env.getbool('SPLIT_MODULE'): try: env.set('SPLIT_MODULE', str(min(4, multiprocessing.cpu_count()))) except NotImplementedError: env.set('SPLIT_MODULE', '2') elif int(env.getone('SPLIT_MODULE')) < 1: Log.Fatal('Value given for -split-module must be > 0') if (env.getbool('ALLOW_LLVM_BITCODE_INPUT') or env.getone('TARGET_OS') != 'nacl' or env.getbool('USE_EMULATOR')): # When llvm input is allowed, the pexe may not be ABI-stable, so do not # split it. Non-ABI-stable pexes may have symbol naming and visibility # issues that the current splitting scheme doesn't account for. # # For now, also do not enable multi-threaded translation when TARGET_OS != # 'nacl', since in these cases we will be using the host toolchain's # linker. # # The x86->arm emulator is very flaky when threading is used, so don't # do module splitting when using it. env.set('SPLIT_MODULE', '1') else: modules = env.getone('SPLIT_MODULE') if modules != '1': env.append('LLC_FLAGS_EXTRA', '-split-module=' + modules) env.append('LD_FLAGS', '-split-module=' + modules) if not env.getbool('SANDBOXED') and env.getbool('STREAM_BITCODE'): # Do not set -streaming-bitcode for sandboxed mode, because it is already # in the default command line. env.append('LLC_FLAGS_EXTRA', '-streaming-bitcode') # If there's a bitcode file, translate it now. tng = driver_tools.TempNameGen(inputs + bcfiles, output) output_type = env.getone('OUTPUT_TYPE') if bcfile: sfile = None if output_type == 's': sfile = output ofile = None if output_type == 'o': ofile = output elif output_type != 's': ofile = tng.TempNameForInput(bcfile, 'o') if sfile: RunLLC(bcfile, sfile, outfiletype='asm') if ofile: RunAS(sfile, ofile) else: RunLLC(bcfile, ofile, outfiletype='obj') else: ofile = None # If we've been told to stop after translation, stop now. if output_type in ('o','s'): return 0 # Replace the bitcode file with __BITCODE__ in the input list if bcfile: inputs = ListReplace(inputs, bcfile, '__BITCODE__') env.set('INPUTS', *inputs) if int(env.getone('SPLIT_MODULE')) > 1: modules = int(env.getone('SPLIT_MODULE')) for i in range(1, modules): filename = ofile + '.module%d' % i TempFiles.add(filename) env.append('INPUTS', filename) if env.getone('TARGET_OS') != 'nacl': RunHostLD(ofile, output) else: RunLD(ofile, output) return 0
def TempNameForOutput(self, imtype): temp = os.path.join(self.TempBase, self.Output + '.' + imtype) TempFiles.add(temp) return temp
def main(argv): env.update(EXTRA_ENV) driver_tools.ParseArgs(argv, TranslatorPatterns) driver_tools.GetArch(required=True) inputs = env.get('INPUTS') output = env.getone('OUTPUT') if len(inputs) == 0: Log.Fatal("No input files") if output == '': Log.Fatal("Please specify output file with -o") # Find the bitcode file on the command line. bcfiles = [ f for f in inputs if not ldtools.IsFlag(f) and (filetype.IsPNaClBitcode( f) or filetype.IsLLVMBitcode(f) or filetype.FileType(f) == 'll') ] if len(bcfiles) > 1: Log.Fatal('Expecting at most 1 bitcode file') elif len(bcfiles) == 1: bcfile = bcfiles[0] else: bcfile = None if not env.getbool('SPLIT_MODULE'): try: env.set('SPLIT_MODULE', str(min(4, multiprocessing.cpu_count()))) except NotImplementedError: env.set('SPLIT_MODULE', '2') elif int(env.getone('SPLIT_MODULE')) < 1: Log.Fatal('Value given for -split-module must be > 0') if (env.getbool('ALLOW_LLVM_BITCODE_INPUT') or env.getone('ARCH') == 'X8632_LINUX' or env.getbool('USE_EMULATOR')): # When llvm input is allowed, the pexe may not be ABI-stable, so do not # split it. For now also do not support threading non-SFI baremetal mode. # Non-ABI-stable pexes may have symbol naming and visibility issues that the # current splitting scheme doesn't account for. # If the link would require a non-standard command line, do not split the # modules because the sandboxed linker doesn't support that combination. # The x86->arm emulator is very flaky when threading is used, so don't # do module splitting when using it. env.set('SPLIT_MODULE', '1') else: modules = env.getone('SPLIT_MODULE') if modules != '1': env.append('LLC_FLAGS_EXTRA', '-split-module=' + modules) env.append('LD_FLAGS', '-split-module=' + modules) if not env.getbool('SANDBOXED') and env.getbool('STREAM_BITCODE'): # Do not set -streaming-bitcode for sandboxed mode, because it is already # in the default command line. env.append('LLC_FLAGS_EXTRA', '-streaming-bitcode') # If there's a bitcode file, translate it now. tng = driver_tools.TempNameGen(inputs + bcfiles, output) output_type = env.getone('OUTPUT_TYPE') if bcfile: sfile = None if output_type == 's': sfile = output ofile = None if output_type == 'o': ofile = output elif output_type != 's': ofile = tng.TempNameForInput(bcfile, 'o') if sfile: RunLLC(bcfile, sfile, outfiletype='asm') if ofile: RunAS(sfile, ofile) else: RunLLC(bcfile, ofile, outfiletype='obj') else: ofile = None # If we've been told to stop after translation, stop now. if output_type in ('o', 's'): return 0 # Replace the bitcode file with __BITCODE__ in the input list if bcfile: inputs = ListReplace(inputs, bcfile, '__BITCODE__') env.set('INPUTS', *inputs) if int(env.getone('SPLIT_MODULE')) > 1: modules = int(env.getone('SPLIT_MODULE')) for i in range(1, modules): filename = ofile + '.module%d' % i TempFiles.add(filename) env.append('INPUTS', filename) if env.getone('ARCH') == 'X8632_LINUX': RunHostLD(ofile, output) else: RunLD(ofile, output) return 0