Exemplo n.º 1
0
def ConvertArgsToFile(args):
    fd, outfile = tempfile.mkstemp()
    # Remember to delete this file afterwards.
    TempFiles.add(outfile)
    cmd = args[0]
    other_args = args[1:]
    os.write(fd, ' '.join(other_args))
    os.close(fd)
    return [cmd, '@' + outfile]
Exemplo n.º 2
0
def ConvertArgsToFile(args):
  fd, outfile = tempfile.mkstemp()
  # Remember to delete this file afterwards.
  TempFiles.add(outfile)
  cmd = args[0]
  other_args = args[1:]
  os.write(fd, ' '.join(other_args))
  os.close(fd)
  return [cmd, '@' + outfile]
Exemplo n.º 3
0
 def TempNameForInput(self, input, imtype):
   fullpath = pathtools.abspath(input)
   # If input is already a temporary name, just add an extension
   if fullpath.startswith(self.TempBase):
     temp = fullpath + '.' + imtype
   else:
     # Source file
     temp = self.TempMap[fullpath] + '.' + imtype
   TempFiles.add(temp)
   return temp
Exemplo n.º 4
0
 def TempNameForInput(self, input, imtype):
     fullpath = pathtools.abspath(input)
     # If input is already a temporary name, just add an extension
     if fullpath.startswith(self.TempBase):
         temp = fullpath + '.' + imtype
     else:
         # Source file
         temp = self.TempMap[fullpath] + '.' + imtype
     TempFiles.add(temp)
     return temp
Exemplo n.º 5
0
  def TempNameForInput(self, input, imtype):
    # If input is already a temporary name, just change the extension
    if input.startswith(self.TempBase):
      temp = self.TempBase + '.' + imtype
    else:
      # Source file
      temp = self.TempMap[input] + '.' + imtype

    temp = self.ValidatePathLength(temp, imtype)
    TempFiles.add(temp)
    return temp
Exemplo n.º 6
0
  def TempNameForInput(self, input, imtype):
    fullpath = pathtools.abspath(input)
    # If input is already a temporary name, just change the extension
    if fullpath.startswith(self.TempBase):
      temp = self.TempBase + '.' + imtype
    else:
      # Source file
      temp = self.TempMap[fullpath] + '.' + imtype

    temp = self.ValidatePathLength(temp, imtype)
    TempFiles.add(temp)
    return temp
Exemplo n.º 7
0
  def __init__(self, inputs, output):
    self.TempBase = tempfile.mkdtemp()
    inputs = [ pathtools.abspath(i) for i in inputs ]
    output = pathtools.basename(output)

    TempFiles.add(self.TempBase)

    self.Output = output + '---linked'


    # Build the initial mapping
    self.TempMap = dict()
    for f in inputs:
      if f.startswith('-'):
        continue
      path = PathSplit(f)
      self.TempMap[f] = [1, path]

    def MangledName(path):
      return output + '---' + '_'.join(path[-n:]) + '---'

    while True:
      # Find conflicts
      ConflictMap = dict()
      Conflicts = set()
      for (f, [n, path]) in self.TempMap.iteritems():
        candidate = pathtools.abspath(MangledName(path))
        if candidate in ConflictMap:
          Conflicts.add(ConflictMap[candidate])
          Conflicts.add(f)
        else:
          ConflictMap[candidate] = f

      if len(Conflicts) == 0:
        break

      # Resolve conflicts
      for f in Conflicts:
        n = self.TempMap[f][0]
        if n+1 > len(self.TempMap[f][1]):
          Log.Fatal('Unable to resolve naming conflicts')
        self.TempMap[f][0] = n+1

    # Clean up the map and put the paths in tempdir
    NewMap = dict()
    for (f, [n, path]) in self.TempMap.iteritems():
      NewMap[f] = os.path.join(self.TempBase, MangledName(path))
    self.TempMap = NewMap
    return
Exemplo n.º 8
0
    def __init__(self, inputs, output):
        self.TempBase = tempfile.mkdtemp()
        inputs = [pathtools.abspath(i) for i in inputs]
        output = pathtools.basename(output)

        TempFiles.add(self.TempBase)

        self.Output = output + '---linked'

        # Build the initial mapping
        self.TempMap = dict()
        for f in inputs:
            if f.startswith('-'):
                continue
            path = PathSplit(f)
            self.TempMap[f] = [1, path]

        def MangledName(path):
            return output + '---' + '_'.join(path[-n:]) + '---'

        while True:
            # Find conflicts
            ConflictMap = dict()
            Conflicts = set()
            for (f, [n, path]) in self.TempMap.iteritems():
                candidate = pathtools.abspath(MangledName(path))
                if candidate in ConflictMap:
                    Conflicts.add(ConflictMap[candidate])
                    Conflicts.add(f)
                else:
                    ConflictMap[candidate] = f

            if len(Conflicts) == 0:
                break

            # Resolve conflicts
            for f in Conflicts:
                n = self.TempMap[f][0]
                if n + 1 > len(self.TempMap[f][1]):
                    Log.Fatal('Unable to resolve naming conflicts')
                self.TempMap[f][0] = n + 1

        # Clean up the map and put the paths in tempdir
        NewMap = dict()
        for (f, [n, path]) in self.TempMap.iteritems():
            NewMap[f] = os.path.join(self.TempBase, MangledName(path))
        self.TempMap = NewMap
        return
Exemplo n.º 9
0
 def TempNameForOutput(self, imtype):
     temp = self.TempBase + '.' + imtype
     TempFiles.add(temp)
     return temp
Exemplo n.º 10
0
def main(argv):
  env.update(EXTRA_ENV)
  driver_tools.ParseArgs(argv, TranslatorPatterns)
  driver_tools.GetArch(required = True)
  SetUpArch()
  SetUpLinkOptions()

  # Now commit to whether or not Subzero is used.
  use_sz = env.getbool('USE_SZ') and not env.getbool('SZ_UNSUPPORTED')

  inputs = env.get('INPUTS')
  output = env.getone('OUTPUT')

  if len(inputs) == 0:
    Log.Fatal("No input files")
  for path in inputs:
    driver_tools.CheckPathLength(path)

  if output == '':
    Log.Fatal("Please specify output file with -o")

  # Find the bitcode file on the command line.
  bcfiles = [f for f in inputs
             if not ldtools.IsFlag(f) and
               (filetype.IsPNaClBitcode(f)
                or filetype.IsLLVMBitcode(f)
                or filetype.FileType(f) == 'll')]
  if len(bcfiles) > 1:
    Log.Fatal('Expecting at most 1 bitcode file')
  elif len(bcfiles) == 1:
    bcfile = bcfiles[0]
  else:
    bcfile = None

  if ((env.getbool('ALLOW_LLVM_BITCODE_INPUT') or
       env.getone('TARGET_OS') != 'nacl' or
       env.getbool('USE_EMULATOR')) and
      env.getone('SPLIT_MODULE') == 'auto'):
    # When llvm input is allowed, the pexe may not be ABI-stable, so do not
    # split it.  Non-ABI-stable pexes may have symbol naming and visibility
    # issues that the current splitting scheme doesn't account for.
    #
    # For now, also do not enable multi-threaded translation when TARGET_OS !=
    # 'nacl', since in these cases we will be using the host toolchain's
    # linker.
    #
    # The x86->arm emulator is very flaky when threading is used, so don't
    # do module splitting when using it.
    env.set('SPLIT_MODULE', 'seq')
  # Do not set -streaming-bitcode for sandboxed mode, because it is already
  # in the default command line.
  if not env.getbool('SANDBOXED') and env.getbool('STREAM_BITCODE'):
    env.append('LLC_FLAGS_EXTRA', '-streaming-bitcode')

  if env.getone('SPLIT_MODULE') == 'seq':
    env.set('SPLIT_MODULE', '1')
    env.set('SZ_THREADS', '0')
  elif env.getone('SPLIT_MODULE') == 'auto':
    try:
      num_modules = min(4, multiprocessing.cpu_count())
    except NotImplementedError:
      num_modules = 2
    env.set('SPLIT_MODULE', str(num_modules))
    env.set('SZ_THREADS', str(num_modules))
  else:
    num_modules = int(env.getone('SPLIT_MODULE'))
    if num_modules < 1:
      Log.Fatal('Value given for -split-module must be > 0')
    env.set('SPLIT_MODULE', str(num_modules))
    env.set('SZ_THREADS', str(num_modules))

  modules = env.getone('SPLIT_MODULE')
  module_sched = env.getone('SPLIT_MODULE_SCHED')
  sz_threads = env.getone('SZ_THREADS')
  # TODO(dschuff,jvoung): No need to specify -split-module=X since the IPC
  # already has a parameter for the number of threads and modules.
  env.append('LLC_FLAGS_EXTRA', '-split-module=' + modules)
  env.append('LD_FLAGS', '-split-module=' + ('1' if use_sz else modules))
  env.append('LLC_FLAGS_EXTRA', '-split-module-sched=' + module_sched)
  # In sandboxed mode, the IPC already has a parameter for the number
  # of threads, so no need to specify that again via argv[].
  if not env.getbool('SANDBOXED'):
    env.append('SZ_FLAGS_EXTRA', '--threads=' + sz_threads)

  # If there's a bitcode file, translate it now.
  tng = driver_tools.TempNameGen(inputs + bcfiles, output)
  output_type = env.getone('OUTPUT_TYPE')
  if bcfile:
    sfile = None
    if output_type == 's':
      sfile = output

    ofile = None
    if output_type == 'o':
      ofile = output
    elif output_type != 's':
      ofile = tng.TempNameForInput(bcfile, 'o')

    if sfile:
      RunCompiler(bcfile, sfile, outfiletype='asm', use_sz=use_sz)
      if ofile:
        RunAS(sfile, ofile)
    else:
      RunCompiler(bcfile, ofile, outfiletype='obj', use_sz=use_sz)
  else:
    ofile = None

  # If we've been told to stop after translation, stop now.
  if output_type in ('o','s'):
    return 0

  if use_sz:
    # Reset SPLIT_MODULE to 1 to fall back to normal linking behavior.
    env.set('SPLIT_MODULE', '1')

  # Replace the bitcode file with __BITCODE__ in the input list
  if bcfile:
    inputs = ListReplace(inputs, bcfile, '__BITCODE__')
    env.set('INPUTS', *inputs)
  if int(env.getone('SPLIT_MODULE')) > 1:
    modules = int(env.getone('SPLIT_MODULE'))
    for i in range(1, modules):
      filename = ofile + '.module%d' % i
      TempFiles.add(filename)
      env.append('INPUTS', filename)

  if env.getone('TARGET_OS') != 'nacl':
    RunHostLD(ofile, output)
  else:
    RunLD(ofile, output)
  return 0
Exemplo n.º 11
0
 def TempNameForOutput(self, imtype):
   temp = self.ValidatePathLength(self.TempBase + '.' + imtype, imtype)
   TempFiles.add(temp)
   return temp
Exemplo n.º 12
0
 def TempNameForOutput(self, imtype):
   temp = self.TempBase + '.' + imtype
   TempFiles.add(temp)
   return temp
Exemplo n.º 13
0
def main(argv):
  env.update(EXTRA_ENV)
  driver_tools.ParseArgs(argv, TranslatorPatterns)
  driver_tools.GetArch(required = True)
  SetUpArch()
  SetUpLinkOptions()

  # Now commit to whether or not Subzero is used.
  use_sz = env.getbool('USE_SZ') and not env.getbool('SZ_UNSUPPORTED')

  inputs = env.get('INPUTS')
  output = env.getone('OUTPUT')

  if len(inputs) == 0:
    Log.Fatal("No input files")
  for path in inputs:
    driver_tools.CheckPathLength(path)

  if output == '':
    Log.Fatal("Please specify output file with -o")

  # Find the bitcode file on the command line.
  bcfiles = [f for f in inputs
             if not ldtools.IsFlag(f) and
               (filetype.IsPNaClBitcode(f)
                or filetype.IsLLVMBitcode(f)
                or filetype.FileType(f) == 'll')]
  if len(bcfiles) > 1:
    Log.Fatal('Expecting at most 1 bitcode file')
  elif len(bcfiles) == 1:
    bcfile = bcfiles[0]
  else:
    bcfile = None

  if ((env.getbool('ALLOW_LLVM_BITCODE_INPUT') or
       env.getone('TARGET_OS') != 'nacl' or
       env.getbool('USE_EMULATOR')) and
      env.getone('SPLIT_MODULE') == 'auto'):
    # When llvm input is allowed, the pexe may not be ABI-stable, so do not
    # split it.  Non-ABI-stable pexes may have symbol naming and visibility
    # issues that the current splitting scheme doesn't account for.
    #
    # For now, also do not enable multi-threaded translation when TARGET_OS !=
    # 'nacl', since in these cases we will be using the host toolchain's
    # linker.
    #
    # The x86->arm emulator is very flaky when threading is used, so don't
    # do module splitting when using it.
    env.set('SPLIT_MODULE', 'seq')
  # Do not set -streaming-bitcode for sandboxed mode, because it is already
  # in the default command line.
  if not env.getbool('SANDBOXED') and env.getbool('STREAM_BITCODE'):
    env.append('LLC_FLAGS_EXTRA', '-streaming-bitcode')

  if env.getone('SPLIT_MODULE') == 'seq':
    env.set('SPLIT_MODULE', '1')
    env.set('SZ_THREADS', '0')
  elif env.getone('SPLIT_MODULE') == 'auto':
    try:
      num_modules = min(4, multiprocessing.cpu_count())
    except NotImplementedError:
      num_modules = 2
    env.set('SPLIT_MODULE', str(num_modules))
    env.set('SZ_THREADS', str(num_modules))
  else:
    num_modules = int(env.getone('SPLIT_MODULE'))
    if num_modules < 1:
      Log.Fatal('Value given for -split-module must be > 0')
    env.set('SPLIT_MODULE', str(num_modules))
    env.set('SZ_THREADS', str(num_modules))

  modules = env.getone('SPLIT_MODULE')
  module_sched = env.getone('SPLIT_MODULE_SCHED')
  sz_threads = env.getone('SZ_THREADS')
  # TODO(dschuff,jvoung): No need to specify -split-module=X since the IPC
  # already has a parameter for the number of threads and modules.
  env.append('LLC_FLAGS_EXTRA', '-split-module=' + modules)
  env.append('LD_FLAGS', '-split-module=' + ('1' if use_sz else modules))
  env.append('LLC_FLAGS_EXTRA', '-split-module-sched=' + module_sched)
  # In sandboxed mode, the IPC already has a parameter for the number
  # of threads, so no need to specify that again via argv[].
  if not env.getbool('SANDBOXED'):
    env.append('SZ_FLAGS_EXTRA', '--threads=' + sz_threads)

  # If there's a bitcode file, translate it now.
  tng = driver_tools.TempNameGen(inputs + bcfiles, output)
  output_type = env.getone('OUTPUT_TYPE')
  if bcfile:
    sfile = None
    if output_type == 's':
      sfile = output

    ofile = None
    if output_type == 'o':
      ofile = output
    elif output_type != 's':
      ofile = tng.TempNameForInput(bcfile, 'o')

    if sfile:
      RunCompiler(bcfile, sfile, outfiletype='asm', use_sz=use_sz)
      if ofile:
        RunAS(sfile, ofile)
    else:
      RunCompiler(bcfile, ofile, outfiletype='obj', use_sz=use_sz)
  else:
    ofile = None

  # If we've been told to stop after translation, stop now.
  if output_type in ('o','s'):
    return 0

  if use_sz:
    # Reset SPLIT_MODULE to 1 to fall back to normal linking behavior.
    env.set('SPLIT_MODULE', '1')

  # Replace the bitcode file with __BITCODE__ in the input list
  if bcfile:
    inputs = ListReplace(inputs, bcfile, '__BITCODE__')
    env.set('INPUTS', *inputs)
  if int(env.getone('SPLIT_MODULE')) > 1:
    modules = int(env.getone('SPLIT_MODULE'))
    for i in range(1, modules):
      filename = ofile + '.module%d' % i
      TempFiles.add(filename)
      env.append('INPUTS', filename)

  if env.getone('TARGET_OS') != 'nacl':
    RunHostLD(ofile, output)
  else:
    RunLD(ofile, output)
  return 0
Exemplo n.º 14
0
def main(argv):
  env.update(EXTRA_ENV)
  driver_tools.ParseArgs(argv, TranslatorPatterns)
  driver_tools.GetArch(required = True)
  SetUpArch()

  inputs = env.get('INPUTS')
  output = env.getone('OUTPUT')

  if len(inputs) == 0:
    Log.Fatal("No input files")

  if output == '':
    Log.Fatal("Please specify output file with -o")

  # Find the bitcode file on the command line.
  bcfiles = [f for f in inputs
             if not ldtools.IsFlag(f) and
               (filetype.IsPNaClBitcode(f)
                or filetype.IsLLVMBitcode(f)
                or filetype.FileType(f) == 'll')]
  if len(bcfiles) > 1:
    Log.Fatal('Expecting at most 1 bitcode file')
  elif len(bcfiles) == 1:
    bcfile = bcfiles[0]
  else:
    bcfile = None

  if not env.getbool('SPLIT_MODULE'):
    try:
      env.set('SPLIT_MODULE', str(min(4, multiprocessing.cpu_count())))
    except NotImplementedError:
      env.set('SPLIT_MODULE', '2')
  elif int(env.getone('SPLIT_MODULE')) < 1:
    Log.Fatal('Value given for -split-module must be > 0')
  if (env.getbool('ALLOW_LLVM_BITCODE_INPUT') or
      env.getone('TARGET_OS') != 'nacl' or
      env.getbool('USE_EMULATOR')):
    # When llvm input is allowed, the pexe may not be ABI-stable, so do not
    # split it.  Non-ABI-stable pexes may have symbol naming and visibility
    # issues that the current splitting scheme doesn't account for.
    #
    # For now, also do not enable multi-threaded translation when TARGET_OS !=
    # 'nacl', since in these cases we will be using the host toolchain's
    # linker.
    #
    # The x86->arm emulator is very flaky when threading is used, so don't
    # do module splitting when using it.
    env.set('SPLIT_MODULE', '1')
  else:
    modules = env.getone('SPLIT_MODULE')
    if modules != '1':
      env.append('LLC_FLAGS_EXTRA', '-split-module=' + modules)
      env.append('LD_FLAGS', '-split-module=' + modules)
    if not env.getbool('SANDBOXED') and env.getbool('STREAM_BITCODE'):
      # Do not set -streaming-bitcode for sandboxed mode, because it is already
      # in the default command line.
      env.append('LLC_FLAGS_EXTRA', '-streaming-bitcode')

  # If there's a bitcode file, translate it now.
  tng = driver_tools.TempNameGen(inputs + bcfiles, output)
  output_type = env.getone('OUTPUT_TYPE')
  if bcfile:
    sfile = None
    if output_type == 's':
      sfile = output

    ofile = None
    if output_type == 'o':
      ofile = output
    elif output_type != 's':
      ofile = tng.TempNameForInput(bcfile, 'o')

    if sfile:
      RunLLC(bcfile, sfile, outfiletype='asm')
      if ofile:
        RunAS(sfile, ofile)
    else:
      RunLLC(bcfile, ofile, outfiletype='obj')
  else:
    ofile = None

  # If we've been told to stop after translation, stop now.
  if output_type in ('o','s'):
    return 0

  # Replace the bitcode file with __BITCODE__ in the input list
  if bcfile:
    inputs = ListReplace(inputs, bcfile, '__BITCODE__')
    env.set('INPUTS', *inputs)
  if int(env.getone('SPLIT_MODULE')) > 1:
    modules = int(env.getone('SPLIT_MODULE'))
    for i in range(1, modules):
      filename = ofile + '.module%d' % i
      TempFiles.add(filename)
      env.append('INPUTS', filename)

  if env.getone('TARGET_OS') != 'nacl':
    RunHostLD(ofile, output)
  else:
    RunLD(ofile, output)
  return 0
Exemplo n.º 15
0
 def TempNameForOutput(self, imtype):
   temp = os.path.join(self.TempBase, self.Output + '.' + imtype)
   TempFiles.add(temp)
   return temp
Exemplo n.º 16
0
 def TempNameForOutput(self, imtype):
     temp = os.path.join(self.TempBase, self.Output + '.' + imtype)
     TempFiles.add(temp)
     return temp
Exemplo n.º 17
0
def main(argv):
    env.update(EXTRA_ENV)
    driver_tools.ParseArgs(argv, TranslatorPatterns)
    driver_tools.GetArch(required=True)

    inputs = env.get('INPUTS')
    output = env.getone('OUTPUT')

    if len(inputs) == 0:
        Log.Fatal("No input files")

    if output == '':
        Log.Fatal("Please specify output file with -o")

    # Find the bitcode file on the command line.
    bcfiles = [
        f for f in inputs
        if not ldtools.IsFlag(f) and (filetype.IsPNaClBitcode(
            f) or filetype.IsLLVMBitcode(f) or filetype.FileType(f) == 'll')
    ]
    if len(bcfiles) > 1:
        Log.Fatal('Expecting at most 1 bitcode file')
    elif len(bcfiles) == 1:
        bcfile = bcfiles[0]
    else:
        bcfile = None

    if not env.getbool('SPLIT_MODULE'):
        try:
            env.set('SPLIT_MODULE', str(min(4, multiprocessing.cpu_count())))
        except NotImplementedError:
            env.set('SPLIT_MODULE', '2')
    elif int(env.getone('SPLIT_MODULE')) < 1:
        Log.Fatal('Value given for -split-module must be > 0')
    if (env.getbool('ALLOW_LLVM_BITCODE_INPUT')
            or env.getone('ARCH') == 'X8632_LINUX'
            or env.getbool('USE_EMULATOR')):
        # When llvm input is allowed, the pexe may not be ABI-stable, so do not
        # split it. For now also do not support threading non-SFI baremetal mode.
        # Non-ABI-stable pexes may have symbol naming and visibility issues that the
        # current splitting scheme doesn't account for.
        # If the link would require a non-standard command line, do not split the
        # modules because the sandboxed linker doesn't support that combination.
        # The x86->arm emulator is very flaky when threading is used, so don't
        # do module splitting when using it.
        env.set('SPLIT_MODULE', '1')
    else:
        modules = env.getone('SPLIT_MODULE')
        if modules != '1':
            env.append('LLC_FLAGS_EXTRA', '-split-module=' + modules)
            env.append('LD_FLAGS', '-split-module=' + modules)
        if not env.getbool('SANDBOXED') and env.getbool('STREAM_BITCODE'):
            # Do not set -streaming-bitcode for sandboxed mode, because it is already
            # in the default command line.
            env.append('LLC_FLAGS_EXTRA', '-streaming-bitcode')

    # If there's a bitcode file, translate it now.
    tng = driver_tools.TempNameGen(inputs + bcfiles, output)
    output_type = env.getone('OUTPUT_TYPE')
    if bcfile:
        sfile = None
        if output_type == 's':
            sfile = output

        ofile = None
        if output_type == 'o':
            ofile = output
        elif output_type != 's':
            ofile = tng.TempNameForInput(bcfile, 'o')

        if sfile:
            RunLLC(bcfile, sfile, outfiletype='asm')
            if ofile:
                RunAS(sfile, ofile)
        else:
            RunLLC(bcfile, ofile, outfiletype='obj')
    else:
        ofile = None

    # If we've been told to stop after translation, stop now.
    if output_type in ('o', 's'):
        return 0

    # Replace the bitcode file with __BITCODE__ in the input list
    if bcfile:
        inputs = ListReplace(inputs, bcfile, '__BITCODE__')
        env.set('INPUTS', *inputs)
    if int(env.getone('SPLIT_MODULE')) > 1:
        modules = int(env.getone('SPLIT_MODULE'))
        for i in range(1, modules):
            filename = ofile + '.module%d' % i
            TempFiles.add(filename)
            env.append('INPUTS', filename)

    if env.getone('ARCH') == 'X8632_LINUX':
        RunHostLD(ofile, output)
    else:
        RunLD(ofile, output)
    return 0
Exemplo n.º 18
0
 def TempNameForOutput(self, imtype):
     temp = self.ValidatePathLength(self.TempBase + '.' + imtype, imtype)
     TempFiles.add(temp)
     return temp