def run(self, cfg, flags, args):
output = target.flag(flags, "-o", "-")
if ("--join", "") in flags:
if len(args) < 1:
raise target.ArgError()
ifs = [parse(x) for x in args]
result = ifs[0]
for x in ifs[1:]:
interface.joinInterfaces(result, x)
else:
# This is computing the interface
if len(args) < 1:
raise target.ArgError()
if args[0] == "@*":
iface = None
else:
iface = parse(args[0])
libs = args[1:]
if "--deep" in flags:
result = deep(libs, iface)
else:
result = shallow(libs, iface)
interface.writeInterface(result, output)
return 0
def shallow(libs, iface):
tf = tempfile.NamedTemporaryFile(suffix=".iface", delete=False)
tf.close()
if not (iface is None):
interface.writeInterface(iface, tf.name)
else:
iface = interface.emptyInterface()
for l in libs:
passes.interface(l, tf.name, [tf.name], quiet=True)
x = interface.parseInterface(tf.name)
interface.joinInterfaces(iface, x)
tf.unlink(tf.name)
return iface
class PrevirtTool(target.Target):
def opts(self, args):
return getopt.getopt(args, ':o', ['work-dir=', 'force', 'no-strip'])
def usage(self):
return "%s [--work-dir=<dir>] [--force] [--no-strip] <manifest>" % self.name
def desc(self):
return """ Previrtualize a compilation unit based on its manifest."""
def args(self):
return [('--work-dir',
"Output intermediate files to the given location"),
('--no-strip', "Leave symbol information in the binary"),
('--force', "Proceed after dependency warnings")]
def run(self, cfg, flags, args):
if len(args) != 1:
raise ArgError()
manifest_file = args[0]
manifest = json.load(open(manifest_file, 'r'))
work_dir = get_work_dir(flags)
if work_dir is None:
work_dir = os.getcwd()
force = get_flag(flags, 'force', False)
if not os.path.exists(work_dir):
sys.stderr.write("making working directory... '%s'\n" % work_dir)
os.mkdir(work_dir)
if not os.path.isdir(work_dir):
sys.stderr.write("working directory '%s' is not a directory\n" %
work_dir)
return 1
modules = get_default(manifest, 'modules', [])
if not (get_default(manifest, 'module', None) is None):
modules += [get_default(manifest, 'module', None)]
if len(modules) == 0:
sys.stderr.write(
"The manifest file '%s' does not have a main module!\n" %
manifest_file)
return 1
libs = get_default(manifest, 'libs', [])
ldflags = get_default(manifest, 'ldflags', [])
native_libs = get_default(manifest, 'native-libs', [])
shared = get_default(manifest, 'shared', [])
search = get_default(manifest, 'search', [])
arguments = get_default(manifest, 'args', None)
name = get_default(manifest, 'name', None)
watches = [
os.path.abspath(x) for x in get_default(manifest, 'watch', [])
]
binary = get_default(manifest, 'binary', modules[0] + '.exe')
try:
found_libs = [get_library(x, search) for x in libs]
except NotFound, e:
sys.stderr.write(
"The library '%s' could not be found. Search path:" %
e.__str__())
sys.stderr.write('\n'.join(search))
return 1
print "\n\nldflags: ", ldflags, "\n\n"
print "\n\nFound libraries: ", found_libs, "\n\n"
non_llvm_libs = []
if not all(map(lambda x: x[1], found_libs)):
sys.stderr.write(
"LLVM versions could not be found for all libraries\n")
ok = True
native = False
for (x, tf) in found_libs:
if tf and native:
ok = False
if not tf:
native = True
if tf:
sys.stderr.write("%s -- llvm\n" % x)
else:
sys.stderr.write("%s -- native\n" % x)
non_llvm_libs.append(x)
if not ok:
sys.stderr.write(
"If native libraries refer to llvm libraries, this is NOT safe!\n"
)
if not force:
return 1
llvm_libs = [x for (x, y) in found_libs if y]
print "\n\nLLVM libs: ", llvm_libs, "\n\n"
print "\n\nnon LLVM libs: ", non_llvm_libs, "\n\n"
print "\n\nmodules: ", modules, "\n\n"
temp_llvm_libs = []
files = {}
all_my_modules = [] + modules
#+ Adding an archive_libs list to hold the native libraries that could not be converted to bitcode
archive_libs = []
for x in modules + llvm_libs:
libCreated = False
bn = prevent_collisions(x)
target = os.path.join(work_dir, bn)
if os.path.abspath(x) != target:
shutil.copyfile(x, target)
if target.endswith('.a'):
if files.has_key(x):
# We've already seen this file, but we may need to pull in more symbols
print "don't handle linking the same file multiple times!"
return 1
else:
sys.stderr.write(
"got archive %s, need to convert to .bc\n" % x)
#+ If the Bitcode equivalent of the library was found, the lib is added to the temp_llvm_libs instead to archive_libs(native)
libCreated = toolchain.archive_to_module(
target, target[:-2] + '.bc', minimal=all_my_modules)
if libCreated:
files[x] = FileStream(target[:-2], 'bc')
temp_llvm_libs.append(x)
else:
sys.stderr.write(
"Bitcode library not created; using native instead \n"
)
archive_libs.append(target)
else:
idx = target.rfind('.bc')
if target[idx:] != '.bc':
shutil.copyfile(target, target[:idx] + '.bc')
files[x] = FileStream(target[:idx], 'bc')
if x not in modules:
temp_llvm_libs.append(x)
if libCreated:
all_my_modules += [files[x].get()]
llvm_libs = temp_llvm_libs
#print "\n\nLLVM libs after: ", llvm_libs, "\n\n"
# Change directory
os.chdir(work_dir)
if not (arguments is None):
# We need to specialize main for program arguments
main = files[modules[0]]
pre = main.get()
post = main.new('a')
passes.specialize_program_args(pre,
post,
arguments,
'arguments',
name=name)
if watches != []:
def _watch(m):
"Applying watches"
pre = m.get()
post = m.new('watch')
print pre
toolchain.watch(pre, post, watches)
InParallel(_watch, files.values())
else:
print "no watches to apply"
# Internalize everything that we can
# We can never internalize main
interface.writeInterface(interface.mainInterface(), 'main.iface')
# First compute the simple interfaces
vals = files.items()
refs = dict([(k,
VersionedFile(prevent_collisions(k[:k.rfind('.bc')]),
'iface')) for (k, _) in vals])
def _references((m, f)):
"Computing references"
name = refs[m].new()
passes.interface(f.get(), name, [])
InParallel(_references, vals)
def _internalize((m, i)):
"Internalizing from references"
pre = i.get()
post = i.new('i')
# sys.stderr.write("%s interfaces\n%s" % (m, '\n'.join([refs[f].get() for f in refs.keys() if f != m] +
# ['main.iface'])))
passes.intern(pre, post,
[refs[f].get()
for f in refs.keys() if f != m] + ['main.iface'])
InParallel(_internalize, vals)
# Strip everything
# This is safe because external names are not stripped
def _strip(m):
"Stripping symbols"
pre = m.get()
post = m.new('x')
passes.strip(pre, post)
if get_flag(flags, 'no-strip', None) is None:
InParallel(_strip, files.values())
# Begin main loop
iface_before_file = VersionedFile('interface_before', 'iface')
iface_after_file = VersionedFile('interface_after', 'iface')
progress = True
rewrite_files = {}
for m in files.keys():
base = prevent_collisions(m[:m.rfind('.bc')])
rewrite_files[m] = VersionedFile(base, 'rw')
iteration = 0
while progress:
iteration += 1
progress = False
# Intra-module previrt
def intra(m):
"Intra-module previrtualization"
# for m in files.values():
# intra-module previrt
pre = m.get()
pre_base = os.path.basename(pre)
post = m.new('p')
post_base = os.path.basename(post)
fn = 'previrt_%s-%s' % (pre_base, post_base)
print '%s === passes.peval ===> %s' % (pre_base, post_base)
passes.peval(pre, post, log=open(fn, 'w'))
InParallel(intra, files.values())
# Inter-module previrt
iface = passes.deep([x.get() for x in files.values()],
['main.iface'])
interface.writeInterface(iface, iface_before_file.new())
# Specialize
def _spec((nm, m)):
"Inter-module module specialization"
pre = m.get()
post = m.new('s')
rw = rewrite_files[nm].new()
passes.specialize(pre, post, rw, [iface_before_file.get()])
InParallel(_spec, files.items())
# Rewrite
def rewrite((nm, m)):
"Inter-module module rewriting"
pre = m.get()
post = m.new('r')
rws = [rewrite_files[x].get() for x in files.keys() if x != nm]
out = [None]
retcode = passes.rewrite(pre, post, rws, output=out)
fn = 'rewrite_%s-%s' % (os.path.basename(pre),
os.path.basename(post))
dbg = open(fn, 'w')
dbg.write(out[0])
dbg.close()
return retcode
rws = InParallel(rewrite, files.items())
progress = any(rws)
# Aggressive internalization
InParallel(_references, vals)
InParallel(_internalize, vals)
# Compute the interface again
iface = passes.deep([x.get() for x in files.values()],
['main.iface'])
interface.writeInterface(iface, iface_after_file.new())
# Prune
def prune(m):
"Pruning dead code/variables"
pre = m.get()
post = m.new('occam')
passes.intern(pre, post, [iface_after_file.get()])
InParallel(prune, files.values())
# Make symlinks for the "final" versions
for x in files.values():
trg = x.base('-final')
if os.path.exists(trg):
os.unlink(trg)
os.symlink(x.get(), trg)
#+ final_libs contains the native and bitcode libraries in a list
final_libs = [files[x].get()
for x in llvm_libs] + [x for x in archive_libs]
print "\n\nFiles: ", files, "\n\n"
print "\n\nArchive libs: ", archive_libs, "\n\n"
print "\n\nLLVM libs: ", llvm_libs, "\n\n"
print "\n\nFinal libs: ", final_libs, "\n\n"
def toLflag(path):
if os.path.exists(path):
return "-L%s" % path
else:
return None
searchflags = [x for x in map(toLflag, search) if x is not None]
#removed these in the apache build. are they ever needed?
xlinker_start = ['']
xlinker_end = ['']
#xlinker_start = ['-Wl,-static']
#xlinker_end = ['-Wl,-call_shared']
#iam: these two flags are not for Darwin.
if platform.system() == 'Darwin':
xlinker_start = ['']
xlinker_end = ['']
if '-lpthread' in native_libs:
shared.append('-pthread')
# Link everything together
sys.stderr.write("linking...\n")
if binary.endswith('.bc'):
binary = binary[:-3]
toolchain.llvmLDWrapper(binary, [files[m].get() for m in modules],
final_libs, searchflags, shared, xlinker_start,
native_libs, xlinker_end, [])
sys.stderr.write("done\n")
if not (POOL is None):
POOL.shutdown()
return 0