def getSectionSizeAndOffset(sectionName, filename):
"""Returns the size and offset of the section, both in bytes.
Use objdump on the provided binary; parse out the fields
to find the given section. Parses the output,and
extracts thesize and offset of that section (in bytes).
"""
objdumpCmd = ['objdump', '-h', '-w', filename]
objdumpProc = Popen(objdumpCmd, stdout=sp.PIPE)
objdumpOutput = objdumpProc.communicate()[0]
if objdumpProc.returncode != 0:
logging.error('Could not dump %s' % filename)
sys.exit(-1)
for line in [l.decode('utf-8') for l in objdumpOutput.splitlines()] :
fields = line.split()
if len(fields) <= 7:
continue
if fields[1] != sectionName:
continue
try:
idx = int(fields[0])
size = int(fields[2], 16)
offset = int(fields[5], 16)
return (size, offset)
except ValueError:
continue
# The needed section could not be found
logging.warning('Could not find "{0}" ELF section in "{1}", so skipping this entry.'.format(sectionName,filename))
return None
def archiveFiles(pArgs, fileNames):
retCode = 0
# We do not want full paths in the archive so we need to chdir into each
# bitcode's folder. Handle this by calling llvm-ar once for all bitcode
# files in the same directory
# Map of directory names to list of bitcode files in that directory
dirToBCMap = {}
for bitCodeFile in fileNames:
dirName = os.path.dirname(bitCodeFile)
basename = os.path.basename(bitCodeFile)
if dirName in dirToBCMap:
dirToBCMap[dirName].append(basename)
else:
dirToBCMap[dirName] = [ basename ]
logging.debug('Built up directory to bitcode file list map:\n{0}'.format(
pprint.pformat(dirToBCMap)))
for (dirname, bcList) in dirToBCMap.items():
logging.debug('Changing directory to "{0}"'.format(dirname))
os.chdir(dirname)
larCmd = [pArgs.llvmArchiver, 'rs', pArgs.outputFile ] + bcList
larProc = Popen(larCmd)
retCode = larProc.wait()
if retCode != 0:
logging.error('Failed to execute:\n{0}'.format(pprint.pformat(larCmd)))
break
if retCode == 0:
logging.info('Generated LLVM bitcode archive {0}'.format(pArgs.outputFile))
else:
logging.error('Failed to generate LLVM bitcode archive')
return retCode
def extract_section_darwin(inputFile):
"""Extracts the section as a string, the darwin version.
Uses otool to extract the section, then processes it
to a usable state.
"""
otoolCmd = ['otool', '-X', '-s', darwinSegmentName, darwinSectionName, inputFile]
otoolProc = Popen(otoolCmd, stdout=sp.PIPE)
otoolOutput = otoolProc.communicate()[0]
if otoolProc.returncode != 0:
logging.error('otool failed on %s' % inputFile)
sys.exit(-1)
lines = otoolOutput.splitlines()
octets = []
for line in lines:
(_, octetline) = line.split('\t')
octets.extend(octetline.split())
octets = ''.join(octets)
contents = octets.decode('hex').splitlines()
if not contents:
logging.error('{0} contained no {1} segment'.format(inputFile, darwinSegmentName))
return contents
def buildObject(builder):
objCompiler = builder.getCompiler()
objCompiler.extend(builder.cmd)
proc = Popen(objCompiler)
rc = proc.wait()
if rc != 0:
sys.exit(rc)
def buildObject(builder):
objCompiler = builder.getCompiler()
objCompiler.extend(builder.cmd)
proc = Popen(objCompiler)
rc = proc.wait()
if rc != 0:
sys.exit(rc)
def buildObject(builder):
objCompiler = builder.getCompiler()
objCompiler.extend(builder.cmd)
_logger.warning(' '.join(objCompiler))
proc = Popen(objCompiler)
rc = proc.wait()
if rc != 0:
sys.exit(rc)
def attachBitcodePathToObject(bcPath, outFileName):
# Don't try to attach a bitcode path to a binary. Unfortunately
# that won't work.
(root, ext) = os.path.splitext(outFileName)
#iam: this also looks very dodgey; we need a more reliable way to do this:
if ext not in ('.o', '.lo', '.os', '.So', '.po'):
_logger.warning(
'Cannot attach bitcode path to "{0} of type {1}"'.format(
outFileName, FileType.getFileType(outFileName)))
return
# Now just build a temporary text file with the full path to the
# bitcode file that we'll write into the object file.
f = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
absBcPath = os.path.abspath(bcPath)
f.write(absBcPath.encode())
f.write('\n'.encode())
_logger.debug(
pprint.pformat('Wrote "{0}" to file "{1}"'.format(absBcPath, f.name)))
# Ensure buffers are flushed so that objcopy doesn't read an empty
# file
f.flush()
os.fsync(f.fileno())
f.close()
# Now write our bitcode section
if (sys.platform.startswith('darwin')):
objcopyCmd = [
'ld', '-r', '-keep_private_externs', outFileName, '-sectcreate',
darwinSegmentName, darwinSectionName, f.name, '-o', outFileName
]
else:
objcopyCmd = [
'objcopy', '--add-section',
'{0}={1}'.format(elfSectionName, f.name), outFileName
]
orc = 0
try:
if os.path.getsize(outFileName) > 0:
objProc = Popen(objcopyCmd)
orc = objProc.wait()
except OSError:
# configure loves to immediately delete things, causing issues for
# us here. Just ignore it
os.remove(f.name)
sys.exit(0)
os.remove(f.name)
if orc != 0:
_logger.error('objcopy failed with {0}'.format(orc))
sys.exit(-1)
def buildObjectFile(builder, srcFile, objFile):
af = builder.getBitcodeArglistFilter()
cc = builder.getCompiler()
cc.extend(af.compileArgs)
cc.append(srcFile)
cc.extend(['-c', '-o', objFile])
proc = Popen(cc)
rc = proc.wait()
if rc != 0:
_logger.warning('Failed to generate object "{0}" for "{1}"'.format(objFile, srcFile))
sys.exit(rc)
def buildBitcodeFile(builder, srcFile, bcFile):
af = builder.getBitcodeArglistFilter()
bcc = builder.getBitcodeCompiler()
bcc.extend(af.compileArgs)
bcc.extend(['-c', srcFile])
bcc.extend(['-o', bcFile])
proc = Popen(bcc)
rc = proc.wait()
if rc != 0:
_logger.warning('Failed to generate bitcode "{0}" for "{1}"'.format(bcFile, srcFile))
sys.exit(rc)
def buildBitcodeFile(builder, srcFile, bcFile):
af = builder.getBitcodeArglistFilter()
bcc = builder.getBitcodeCompiler()
bcc.extend(af.compileArgs)
bcc.extend(['-c', srcFile])
bcc.extend(['-o', bcFile])
proc = Popen(bcc)
rc = proc.wait()
if rc != 0:
_logger.warning('Failed to generate bitcode "{0}" for "{1}"'.format(
bcFile, srcFile))
sys.exit(rc)
def buildObjectFile(builder, srcFile, objFile):
af = builder.getBitcodeArglistFilter()
cc = builder.getCompiler()
cc.extend(af.compileArgs)
cc.append(srcFile)
cc.extend(['-c', '-o', objFile])
proc = Popen(cc)
rc = proc.wait()
if rc != 0:
_logger.warning('Failed to generate object "{0}" for "{1}"'.format(
objFile, srcFile))
sys.exit(rc)
def linkFiles(builder, objectFiles):
af = builder.getBitcodeArglistFilter()
outputFile = af.getOutputFilename()
cc = builder.getCompiler()
cc.extend(af.objectFiles)
cc.extend(objectFiles)
cc.extend(af.linkArgs)
cc.extend(['-o', outputFile])
proc = Popen(cc)
rc = proc.wait()
if rc != 0:
_logger.warning('Failed to link "{0}"'.format(str(cc)))
sys.exit(rc)
def linkFiles(builder, objectFiles):
af = builder.getBitcodeArglistFilter()
outputFile = af.getOutputFilename()
cc = builder.getCompiler()
cc.extend(objectFiles)
cc.extend(af.objectFiles)
cc.extend(af.linkArgs)
cc.extend(['-o', outputFile])
proc = Popen(cc)
rc = proc.wait()
if rc != 0:
_logger.warning('Failed to link "{0}"'.format(str(cc)))
sys.exit(rc)
def getFileType(cls, fileName):
# This is a hacky way of determining
# the type of file we are looking at.
# Maybe we should use python-magic instead?
fileP = Popen(['file', fileName], stdout=PIPE)
output = fileP.communicate()[0]
output = output.decode()
if 'ELF' in output and 'executable' in output:
return cls.EXECUTABLE
elif 'current ar archive' in output:
return cls.ARCHIVE
elif 'ELF' in output and 'relocatable' in output:
return cls.OBJECT
else:
return cls.UNKNOWN
def attachBitcodePathToObject(bcPath, outFileName):
# Don't try to attach a bitcode path to a binary. Unfortunately
# that won't work.
(root, ext) = os.path.splitext(outFileName)
#iam: this also looks very dodgey; we need a more reliable way to do this:
if ext not in ('.o', '.lo', '.os', '.So', '.po'):
_logger.warning('Cannot attach bitcode path to "{0} of type {1}"'.format(outFileName, FileType.getFileType(outFileName)))
return
# Now just build a temporary text file with the full path to the
# bitcode file that we'll write into the object file.
f = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
absBcPath = os.path.abspath(bcPath)
f.write(absBcPath.encode())
f.write('\n'.encode())
_logger.debug(pprint.pformat('Wrote "{0}" to file "{1}"'.format(absBcPath, f.name)))
# Ensure buffers are flushed so that objcopy doesn't read an empty
# file
f.flush()
os.fsync(f.fileno())
f.close()
# Now write our bitcode section
if (sys.platform.startswith('darwin')):
objcopyCmd = ['ld', '-r', '-keep_private_externs', outFileName, '-sectcreate', darwinSegmentName, darwinSectionName, f.name, '-o', outFileName]
else:
objcopyCmd = ['objcopy', '--add-section', '{0}={1}'.format(elfSectionName, f.name), outFileName]
orc = 0
try:
if os.path.getsize(outFileName) > 0:
objProc = Popen(objcopyCmd)
orc = objProc.wait()
except OSError:
# configure loves to immediately delete things, causing issues for
# us here. Just ignore it
os.remove(f.name)
sys.exit(0)
os.remove(f.name)
if orc != 0:
_logger.error('objcopy failed with {0}'.format(orc))
sys.exit(-1)
def attachBitcodePathToObject(bcPath, outFileName):
# Don't try to attach a bitcode path to a binary. Unfortunately
# that won't work.
(root, ext) = os.path.splitext(outFileName)
if ext not in ('.o', '.lo', '.os'):
_logger.warning(
'Cannot attach bitcode path to "{0}"'.format(outFileName))
return
# Now just build a temporary text file with the full path to the
# bitcode file that we'll write into the object file.
f = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
absBcPath = os.path.abspath(bcPath)
f.write(absBcPath.encode())
f.write('\n'.encode())
_logger.debug(
pprint.pformat('Wrote "{0}" to file "{1}"'.format(absBcPath, f.name)))
# Ensure buffers are flushed so that objcopy doesn't read an empty
# file
f.flush()
os.fsync(f.fileno())
f.close()
# Now write our .llvm_bc section
objcopyCmd = [
'objcopy', '--add-section', '{0}={1}'.format(elfSectionName, f.name),
outFileName
]
orc = 0
try:
if os.path.getsize(outFileName) > 0:
objProc = Popen(objcopyCmd)
orc = objProc.wait()
except OSError:
# configure loves to immediately delete things, causing issues for
# us here. Just ignore it
os.remove(f.name)
sys.exit(0)
os.remove(f.name)
if orc != 0:
_logger.error('objcopy failed with {0}'.format(orc))
sys.exit(-1)
def attachBitcodePathToObject(bcPath, outFileName):
if not os.path.isfile(outFileName):
_logger.critical('***** "{0}" output file not exists. Please debug!'.format(outFileName))
sys.exit(0)
# Don't try to attach a bitcode path to a binary. Unfortunately
# that won't work.
(root, ext) = os.path.splitext(outFileName)
if ext not in ('.o', '.lo', '.os'):
_logger.warning('>>>>> Cannot attach bitcode path to "{0}"'.format(outFileName))
return
# Now just build a temporary text file with the full path to the
# bitcode file that we'll write into the object file.
f = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
absBcPath = os.path.abspath(bcPath)
f.write(absBcPath.decode('utf8').encode('utf8'))
f.write('\n'.encode())
_logger.debug(pprint.pformat('Wrote "{0}" to file "{1}"'.format(absBcPath, f.name)))
# Ensure buffers are flushed so that objcopy doesn't read an empty
# file
f.flush()
os.fsync(f.fileno())
f.close()
# Now write our .llvm_bc section
objcopyCmd = ['objcopy', '--add-section', '{0}={1}'.format(elfSectionName, f.name), outFileName]
orc = 0
try:
if os.path.getsize(outFileName) > 0:
objProc = Popen(objcopyCmd)
orc = objProc.wait()
except OSError:
# configure loves to immediately delete things, causing issues for
# us here. Just ignore it
os.remove(f.name)
sys.exit(0)
os.remove(f.name)
if orc != 0:
_logger.error('objcopy failed with {0}'.format(orc))
sys.exit(-1)
def getFileType(cls, fileName):
# This is a hacky way of determining
# the type of file we are looking at.
# Maybe we should use python-magic instead?
fileP = Popen(['file',os.path.realpath(fileName)], stdout=PIPE)
output = fileP.communicate()[0]
output = output.decode('utf8')
if 'ELF' in output and 'executable' in output:
return cls.EXECUTABLE
elif 'ELF' in output and 'shared' in output:
return cls.SHARED
elif 'current ar archive' in output:
return cls.ARCHIVE
elif 'ELF' in output and 'relocatable' in output:
return cls.OBJECT
else:
return cls.UNKNOWN
def buildAndAttachBitcode(builder):
af = builder.getBitcodeArglistFilter()
if len(af.inputFiles) == 0 or af.isAssembly or af.isAssembleOnly:
return
bcc = builder.getBitcodeCompiler()
bcc.extend(af.filteredArgs)
bcc.append('-c')
bcc.extend(builder.extraBitcodeArgs(af))
# Filter out linker options since we are compiling with -c. If we
# leave them in, clang will emit warnings. Some configure scripts
# check to see if there was any output on stderr instead of the
# return code of commands, so warnings about unused link flags can
# cause spurious failures here.
bcc = [arg for arg in bcc if not isLinkOption(arg)]
proc = Popen(bcc)
rc = proc.wait()
if rc == 0:
builder.attachBitcode(af)
sys.exit(rc)
def buildAndAttachBitcode(builder):
af = builder.getBitcodeArglistFilter()
if len(af.inputFiles) == 0 or af.isAssembly or af.isAssembleOnly:
return
bcc = builder.getBitcodeCompiler()
bcc.extend(af.filteredArgs)
bcc.append('-c')
bcc.extend(builder.extraBitcodeArgs(af))
# Filter out linker options since we are compiling with -c. If we
# leave them in, clang will emit warnings. Some configure scripts
# check to see if there was any output on stderr instead of the
# return code of commands, so warnings about unused link flags can
# cause spurious failures here.
bcc = [arg for arg in bcc if not isLinkOption(arg)]
proc = Popen(bcc)
rc = proc.wait()
if rc == 0:
builder.attachBitcode(af)
sys.exit(rc)
def linkFiles(pArgs, fileNames):
linkCmd = [ pArgs.llvmLinker, '-v' ] if pArgs.verboseFlag else [ pArgs.llvmLinker ]
linkCmd.extend(['-o', pArgs.outputFile ])
linkCmd.extend([x for x in fileNames if x != ''])
logging.info('Writing output to {0}'.format(pArgs.outputFile))
try:
linkProc = Popen(linkCmd)
except OSError as e:
if e.errno == 2:
errorMsg = 'Your llvm-link does not seem to be easy to find.\nEither install it or use the -l llvmLinker option.'
else:
errorMsg = 'OS error({0}): {1}'.format(e.errno, e.strerror)
logging.error(errorMsg)
raise Exception(errorMsg)
else:
exitCode = linkProc.wait()
logging.info('{0} returned {1}'.format(pArgs.llvmLinker, str(exitCode)))
return exitCode
def getFileType(cls, fileName):
# This is a hacky way of determining
# the type of file we are looking at.
# Maybe we should use python-magic instead?
fileP = Popen(['file', os.path.realpath(fileName)], stdout=PIPE)
output = fileP.communicate()[0]
output = output.decode()
if 'ELF' in output and 'executable' in output:
return cls.ELF_EXECUTABLE
if 'Mach-O' in output and 'executable' in output:
return cls.MACH_EXECUTABLE
elif 'ELF' in output and 'shared' in output:
return cls.ELF_SHARED
elif 'Mach-O' in output and 'dynamically linked shared' in output:
return cls.MACH_SHARED
elif 'current ar archive' in output:
return cls.ARCHIVE
elif 'ELF' in output and 'relocatable' in output:
return cls.ELF_OBJECT
elif 'Mach-O' in output and 'object' in output:
return cls.MACH_OBJECT
else:
return cls.UNKNOWN
def handleArchive(pArgs):
originalDir = os.getcwd() # This will be the destination
pArgs.arCmd.append(pArgs.inputFile)
# Make temporary directory to extract objects to
tempDir = ''
bitCodeFiles = [ ]
retCode=0
try:
tempDir = tempfile.mkdtemp(suffix='wllvm')
os.chdir(tempDir)
# Extract objects from archive
try:
arP = Popen(pArgs.arCmd)
except OSError as e:
if e.errno == 2:
errorMsg = 'Your ar does not seem to be easy to find.\n'
else:
errorMsg = 'OS error({0}): {1}'.format(e.errno, e.strerror)
logging.error(errorMsg)
raise Exception(errorMsg)
arPE = arP.wait()
if arPE != 0:
errorMsg = 'Failed to execute archiver with command {0}'.format(pArgs.arCmd)
logging.error(errorMsg)
raise Exception(errorMsg)
# Iterate over objects and examine their bitcode inserts
for (root, dirs, files) in os.walk(tempDir):
logging.debug('Exploring "{0}"'.format(root))
for f in files:
fPath = os.path.join(root, f)
if FileType.getFileType(fPath) == pArgs.fileType:
# Extract bitcode locations from object
contents = pArgs.extractor(fPath)
for bcFile in contents:
if bcFile != '':
if not os.path.exists(bcFile):
logging.warning('{0} lists bitcode library "{1}" but it could not be found'.format(f, bcFile))
else:
bitCodeFiles.append(bcFile)
else:
logging.info('Ignoring file "{0}" in archive'.format(f))
logging.info('Found the following bitcode file names to build bitcode archive:\n{0}'.format(
pprint.pformat(bitCodeFiles)))
finally:
# Delete the temporary folder
logging.debug('Deleting temporary folder "{0}"'.format(tempDir))
shutil.rmtree(tempDir)
#write the manifest file if asked for
if pArgs.manifestFlag:
manifestFile = '{0}.llvm.manifest'.format(pArgs.inputFile)
with open(manifestFile, 'w') as output:
for f in bitCodeFiles:
output.write('{0}\n'.format(f))
# Build bitcode archive
os.chdir(originalDir)
return buildArchive(pArgs, bitCodeFiles)
