def __init__(self, arch, input_xar, output_path, use_xml=None):
self.output = os.path.realpath(output_path)
self.returncode = 0
self.stdout = ""
self.arch = arch
self.input = input_xar
self.is_executable = False
self.contain_swift = False
self.deployment_target = None
self.force_optimize_swift = False
self.is_compile_with_clang = env.compile_with_clang
# zf: read from xar xml during super class initialization
super(BitcodeBundle, self).__init__(input_xar)
if use_xml:
data = ""
with open(use_xml, 'r') as file:
data = file.read()
self.xml = ET.fromstring(data)
try:
self.platform = self.subdoc.find("platform").text
self.sdk_version = self.subdoc.find("sdkversion").text
self.version = self.subdoc.find("version").text
except AttributeError:
env.error("Malformed Header for bundle")
else:
env.setVersion(self.version)
env.setPlatform(self.platform)
if env.translate_watchos and env.getPlatform(
) == "watchos" and arch == "armv7k":
self.arch = "arm64_32"
def writeDsymUUIDMap(self, bundle_path):
resource_dir = os.path.join(bundle_path, "Contents", "Resources")
plist_template = u"""<?xml version="1.0" encoding="UTF-8"?>""" \
"""<!DOCTYPE plist PUBLIC""" \
""" "-//Apple//DTD PLIST 1.0//EN" """ \
""""http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>DBGOriginalUUID</key>
<string>{UUID}</string>
</dict>
</plist>"""
if not os.access(resource_dir, os.W_OK):
env.error(u"Dsym bunlde not writeable: {}".format(bundle_path))
for arch in self.archs:
try:
old_uuid = self.uuid[arch]
if env.translate_watchos and arch == "armv7k":
new_uuid = self.output_uuid["arm64_32"]
else:
new_uuid = self.output_uuid[arch]
except KeyError:
env.error("Cannot generate uuid map in dsym bundle")
with open(os.path.join(resource_dir, new_uuid + ".plist"),
"w") as f:
f.write(plist_template.format(UUID=old_uuid))
def run_job(self, job):
"""Run sub command and catch errors"""
try:
rv = job.run()
except BitcodeBuildFailure:
# Catch and log an error
env.error(u"Failed to compile bundle: {}".format(self.input))
else:
return rv
def installOutput(self, path):
if len(self.output_slices) == 0:
env.error("Install failed: no bitcode build yet")
elif len(self.output_slices) == 1:
try:
shutil.move(self.output_slices[0].output, path)
except IOError:
env.error(u"Install failed: can't create {}".format(path))
else:
cmdtool.LipoCreate([x.output for x in self.output_slices],
path).run()
self.output_uuid = MachoType.getUUID(path)
def getArch(path):
macho_info = cmdtool.MachoInfo(path).run()
if macho_info.returncode != 0:
env.error(u"{} is not valid macho file".format(path))
elif macho_info.stdout.startswith("Non-fat"):
arch = macho_info.stdout.split()[-1] # Last phrase is arch
return [arch]
else:
message = macho_info.stdout.split()
try:
begin = message.index("are:") + 1
except ValueError:
env.error("Cound not detect architecture of the MachO file")
else:
return message[begin:]
def linkOptions(self):
"""Return all the link options"""
linker_options = [
x.text if x.text is not None else ""
for x in self.subdoc.find("link-options").findall("option")
]
if not ld_option_verifier.verify(linker_options):
env.error(u"Linker option verification "
"failed for bundle {} ({})".format(
self.input, ld_option_verifier.error_msg))
if linker_options.count("-execute") != 0:
self.is_executable = True
if self.platform is not None and self.platform != "Unknown":
linker_options.extend(["-syslibroot", env.getSDK()])
if self.sdk_version is not None and self.sdk_version != "NA":
linker_options.extend(["-sdk_version", self.sdk_version])
return linker_options
def __init__(self, arch, input_xar, output_path):
self.output = os.path.realpath(output_path)
self.returncode = 0
self.stdout = ""
self.arch = arch
self.is_executable = False
self.contain_swift = False
super(BitcodeBundle, self).__init__(input_xar)
try:
self.platform = self.subdoc.find("platform").text
self.sdk_version = self.subdoc.find("sdkversion").text
self.version = self.subdoc.find("version").text
except AttributeError:
env.error("Malformed Header for bundle")
else:
env.setVersion(self.version)
env.setPlatform(self.platform)
def constructBitcodeJob(self, xml_node):
"""construct a single bitcode workload"""
name = xml_node.find("name").text
output_name = name + ".o"
if xml_node.find("clang") is not None:
clang = Clang(name, output_name, self.dir)
options = [
x.text if x.text is not None else ""
for x in xml_node.find("clang").findall("cmd")
]
options = ClangCC1Translator.translate(options)
if (clang_option_verifier.verify(options)):
clang.addArgs(options)
else:
env.error(u"Clang option verification "
"failed for bitcode {} ({})".format(
name, clang_option_verifier.error_msg))
return clang
elif xml_node.find("swift") is not None:
# swift uses extension to distinguish input type
# we need to move the file to have .bc extension first
self.contain_swift = True
if env.compile_with_clang:
clang = Clang(name, output_name, self.dir)
options = [
x.text if x.text is not None else ""
for x in xml_node.find("swift").findall("cmd")
]
if (swift_option_verifier.verify(options)):
clang.addArgs(SwiftArgTranslator.translate(options))
else:
env.error(u"Swift option verification "
"failed for bitcode {} ({})".format(
name, clang_option_verifier.error_msg))
return clang
else:
bcname = name + ".bc"
shutil.move(os.path.join(self.dir, name),
os.path.join(self.dir, bcname))
swift = Swift(bcname, output_name, self.dir)
options = [
x.text if x.text is not None else ""
for x in xml_node.find("swift").findall("cmd")
]
if (swift_option_verifier.verify(options)):
swift.addArgs(options)
else:
env.error(u"Swift option verification "
"failed for bitcode {} ({})".format(
name, swift_option_verifier.error_msg))
return swift
else:
env.error("Cannot figure out bitcode kind: {}".format(name))
def getXAR(self, arch):
try:
file = self._bitcode_cache[arch]
except KeyError:
macho_thin = self.getSlice(arch)
extract_path = os.path.join(self._temp_dir,
self.name + "." + arch + ".xar")
extract_xar = cmdtool.ExtractXAR(macho_thin, extract_path).run()
if extract_xar.returncode != 0:
env.error(u"Cannot extract bundle from {} ({})".format(
self.path, arch))
if os.stat(extract_path).st_size <= 1:
env.error(
u"Bundle only contains bitcode-marker {} ({})".format(
self.path, arch))
self._bitcode_cache[arch] = extract_path
return extract_path
else:
return file
def getSlice(self, arch):
if arch not in self.archs:
env.error(u"Requested arch {} doesn't exist in {}".format(
arch, self.path))
if self.type == MachoType.Thin:
return self.path
elif self.type == MachoType.Fat:
try:
file = self._slice_cache[arch]
except KeyError:
extract_path = os.path.join(self._temp_dir,
self.name + "." + arch)
extract_job = cmdtool.ExtractSlice(self.path, arch,
extract_path).run()
if extract_job.returncode != 0:
env.error(u"Cannot extract arch {} from {}".format(
arch, self.path))
self._slice_cache[arch] = extract_path
return extract_path
else:
return file
def main(args=None):
"""Run the program, can override args for testing."""
if args is None:
args = sys.argv
args = parse_args(args)
try:
env.initState(args)
if not os.path.isfile(args.input_macho_file):
env.error(u"Input macho file doesn't exist: {}".format(
args.input_macho_file))
if args.symbol_map is not None and args.dsym_output is None:
env.error("--symbol-map can only be used "
"together with --generate-dsym")
if args.symbol_map is not None and not os.path.exists(args.symbol_map):
env.error(u"path passed to --symbol-map doesn't exists: {}".format(
args.symbol_map))
input_macho = Macho(args.input_macho_file)
if input_macho == MachoType.Error:
env.error(u"Input is not a macho file: {}".format(
args.input_macho_file))
map(input_macho.buildBitcode, input_macho.getArchs())
if (args.dsym_output is not None and
not any([x.contain_symbols for x in input_macho.output_slices])
and args.symbol_map is None):
env.warning(
u"Cannot genarte useful dsym from input macho file: {}".format(
args.input_macho_file))
if not args.verify:
input_macho.installOutput(args.output)
if args.dsym_output is not None:
cmdtool.Dsymutil(args.output, args.dsym_output).run()
input_macho.writeDsymUUIDMap(args.dsym_output)
if args.symbol_map is not None:
cmdtool.DsymMap(args.dsym_output, args.symbol_map).run()
# always strip the output
if input_macho.is_executable:
cmdtool.StripSymbols(args.output).run()
else:
cmdtool.StripDebug(args.output, args.strip_swift).run()
finally:
env.cleanupTempDirectories()
def linkOptions(self):
"""Return all the link options"""
linker_options = [
x.text if x.text is not None else ""
for x in self.subdoc.find("link-options").findall("option")
]
if not ld_option_verifier.verify(linker_options):
env.error(u"Linker option verification "
"failed for bundle {} ({})".format(
self.input, ld_option_verifier.error_msg))
if linker_options.count("-execute") != 0:
self.is_executable = True
# make sure linker has a none zero version min for watchos.
try:
# check watchos version.
version_min = linker_options.index("-watchos_version_min")
# if valid version min location, check if it is 0.0
if version_min < (len(linker_options) - 1) and linker_options[
version_min + 1] == "0.0.0":
# write a default watchos version.
if self.is_translate_watchos:
linker_options[version_min + 1] = "5.0.0"
else:
linker_options[version_min + 1] = "2.0.0"
self.deployment_target = linker_options[version_min + 1]
except ValueError:
# if watchos is not specified during translate, add default deployment target.
if self.is_translate_watchos:
linker_options.extend(["-watchos_version_min", "5.0.0"])
if self.platform is not None and self.platform != "Unknown":
linker_options.extend(["-syslibroot", env.getSDK()])
if self.sdk_version is not None and self.sdk_version != "NA":
linker_options.extend(["-sdk_version", self.sdk_version])
return linker_options
def run_cmd(self, xfail=False):
"""Run a command in a working directory."""
start_time = datetime.datetime.now()
try:
if not os.environ.get('TESTING', False):
out = subprocess.check_output(self.cmd,
stderr=subprocess.STDOUT,
cwd=self.working_dir)
else:
out = "Skipped for testing mode."
end_time = datetime.datetime.now()
except subprocess.CalledProcessError as e:
self.returncode = e.returncode
self.stdout = e.output
if xfail:
env.log(self)
else:
env.error(self)
else:
self.returncode = 0
self.stdout = out
env.log(self)
env.debug("Command took {} seconds".format(
(end_time - start_time).seconds))
def __init__(self, xar_path):
if os.path.isfile(xar_path):
self.input = xar_path
else:
env.error(u"Input XAR doesn't exist: {}".format(xar_path))
cmd = [self.XAR_EXEC, "-d", "-", "-f", self.input]
try:
out = subprocess.check_output(cmd)
except subprocess.CalledProcessError:
env.error(u"toc cannot be extracted: {}".format(xar_path))
else:
self.xml = ET.fromstring(out)
self.dir = env.createTempDirectory()
cmd = [self.XAR_EXEC, "-x", "-C", self.dir, "-f", self.input]
try:
out = subprocess.check_output(cmd)
except subprocess.CalledProcessError:
env.error(u"XAR cannot be extracted: {}".format(xar_path))
cmd = ['/bin/chmod', "-R", "+r", self.dir]
try:
out = subprocess.check_output(cmd)
except subprocess.CalledProcessError:
env.error(u"Permission fixup failed: {}".format(xar_path))
def main(args=None):
"""Run the program, can override args for testing."""
if args is None:
args = sys.argv
args = parse_args(args)
try:
env.initState(args)
if not os.path.isfile(args.input_macho_file):
env.error(u"Input macho file doesn't exist: {}".format(
args.input_macho_file))
if args.symbol_map is not None and args.dsym_output is None:
env.error("--symbol-map can only be used "
"together with --generate-dsym")
if args.symbol_map is not None and not os.path.exists(args.symbol_map):
env.error(u"path passed to --symbol-map doesn't exists: {}".format(
args.symbol_map))
## todo: add pre-processing for lipo a static library and extract prelinked single .o
##
## After pre-processing, we have a macho file
input_macho = Macho(args.input_macho_file)
if input_macho == MachoType.Error:
env.error(u"Input is not a macho file: {}".format(
args.input_macho_file))
use_xml = args.use_xml
if use_xml is not None and \
not os.path.isfile(args.input_macho_file):
env.error(u"Input XML file doesn't exist: {}".format(args.use_xml))
map(lambda arch: input_macho.buildBitcode(arch, use_xml),
input_macho.getArchs())
if (args.dsym_output is not None and
not any([x.contain_symbols for x in input_macho.output_slices])
and args.symbol_map is None):
env.warning(
u"Cannot genarte useful dsym from input macho file: {}".format(
unicode(args.input_macho_file, 'utf-8')))
if not args.verify:
input_macho.installOutput(args.output)
if args.dsym_output is not None:
cmdtool.Dsymutil(args.output, args.dsym_output).run()
input_macho.writeDsymUUIDMap(args.dsym_output)
if args.symbol_map is not None:
cmdtool.DsymMap(args.dsym_output, args.symbol_map).run()
# always strip the output
if input_macho.is_executable:
cmdtool.StripSymbols(args.output).run()
else:
cmdtool.StripDebug(args.output, args.strip_swift).run()
## todo: add post-processing
##
## After pre-processing, we have a macho file
finally:
env.cleanupTempDirectories()
def run(self):
"""Build Bitcode Bundle"""
linker_inputs = []
linker = Ld(self.output, self.dir)
linker.addArgs(["-arch", self.arch])
linker.addArgs(self.linkOptions)
# handle bitcode input
bitcode_files = self.getFileNode("Bitcode")
if len(bitcode_files) > 0:
compiler_jobs = map(self.constructBitcodeJob, bitcode_files)
linker_inputs.extend(compiler_jobs)
# object input
object_files = self.getFileNode("Object")
if len(object_files) > 0:
if env.getPlatform() == "watchos":
env.error("Watch platform doesn't support object inputs")
object_jobs = map(self.constructObjectJob, object_files)
linker_inputs.extend(object_jobs)
# run compilation
env.map(self.run_job, linker_inputs)
# run bundle compilation in sequential to avoid dead-lock
bundle_files = self.getFileNode("Bundle")
if len(bundle_files) > 0:
bundle_jobs = map(self.constructBundleJob, bundle_files)
map(self.run_job, bundle_jobs)
linker_inputs.extend(bundle_jobs)
# sort object inputs
inputs = sorted([os.path.basename(x.output) for x in linker_inputs])
# handle LTO inputs
LTO_inputs = self.getFileNode("LTO")
if (len(LTO_inputs)) != 0:
inputs.extend([x.find("name").text for x in LTO_inputs])
linker.addArgs(["-flto-codegen-only"])
# add inputs to a LinkFileList
LinkFileList = os.path.join(self.dir, self.output + ".LinkFileList")
with open(LinkFileList, 'w') as f:
for i in inputs:
f.write(os.path.join(self.dir, i))
f.write('\n')
linker.addArgs(["-filelist", LinkFileList])
# version specific arguments
if env.satifiesLinkerVersion("253.2"):
linker.addArgs(["-ignore_auto_link"])
if env.satifiesLinkerVersion("253.3.1"):
linker.addArgs(["-allow_dead_duplicates"])
# add libLTO.dylib if needed
if env.liblto is not None:
linker.addArgs(["-lto_library", env.liblto])
# handle dylibs
dylibs_node = self.subdoc.find("dylibs")
if dylibs_node is not None:
for lib_node in dylibs_node.iter():
if lib_node.tag == "lib":
lib_path = env.resolveDylibs(self.arch, lib_node.text)
linker.addArgs([lib_path])
elif lib_node.tag == "weak":
# allow weak framework to be missing. If they provide no
# symbols, the link will succeed.
lib_path = env.resolveDylibs(self.arch, lib_node.text,
True)
if lib_path is not None:
linker.addArgs(["-weak_library", lib_path])
# add swift library search path, only when auto-link cannot be ignored.
if self.contain_swift and not env.satifiesLinkerVersion("253.2"):
swiftLibPath = env.getlibSwiftPath(self.arch)
if swiftLibPath is not None:
linker.addArgs(["-L", swiftLibPath])
# add libclang_rt
if self.forceload_compiler_rt:
linker.addArgs(["-force_load"])
linker.addArgs([env.getlibclang_rt(self.arch)])
# linking
self.run_job(linker)
return self
def constructBitcodeJob(self, xml_node):
"""construct a single bitcode workload"""
name = xml_node.find("name").text
output_name = name + ".o"
if xml_node.find("clang") is not None:
clang = Clang(name, output_name, self.dir)
options = [
x.text if x.text is not None else ""
for x in xml_node.find("clang").findall("cmd")
]
options = ClangCC1Translator.upgrade(options, self.arch)
if self.is_translate_watchos:
options = ClangCC1Translator.translate_triple(options)
if clang_option_verifier.verify(options):
clang.addArgs(options)
else:
env.error(u"Clang option verification "
"failed for bitcode {} ({})".format(
name, clang_option_verifier.error_msg))
if env.getPlatform() == "watchos":
clang.addArgs(["-fno-gnu-inline-asm"])
# gongjl build bitcode and cmd
clang.addArgs(["-fembed-bitcode=all"])
return clang
elif xml_node.find("swift") is not None:
# swift uses extension to distinguish input type
# we need to move the file to have .bc extension first
self.contain_swift = True
if self.is_compile_with_clang:
clang = Clang(name, output_name, self.dir)
options = [
x.text if x.text is not None else ""
for x in xml_node.find("swift").findall("cmd")
]
options = SwiftArgTranslator.upgrade(options, self.arch)
if swift_option_verifier.verify(options):
options = SwiftArgTranslator.translate_to_clang(options)
if self.force_optimize_swift:
options = ClangCC1Translator.add_optimization(options)
if self.is_translate_watchos:
options = ClangCC1Translator.translate_triple(options)
clang.addArgs(options)
else:
env.error(u"Swift option verification "
"failed for bitcode {} ({})".format(
name, clang_option_verifier.error_msg))
return clang
else:
bcname = name + ".bc"
shutil.move(os.path.join(self.dir, name),
os.path.join(self.dir, bcname))
swift = Swift(bcname, output_name, self.dir)
options = [
x.text if x.text is not None else ""
for x in xml_node.find("swift").findall("cmd")
]
if swift_option_verifier.verify(options):
if self.force_optimize_swift:
options = SwiftArgTranslator.add_optimization(options)
if self.is_translate_watchos:
options = SwiftArgTranslator.translate_triple(options)
swift.addArgs(options)
else:
env.error(u"Swift option verification "
"failed for bitcode {} ({})".format(
name, swift_option_verifier.error_msg))
return swift
else:
env.error("Cannot figure out bitcode kind: {}".format(name))
def run(self):
"""Build Bitcode Bundle"""
linker_inputs = []
linker = Ld(self.output, self.dir)
linker.addArgs(["-arch", self.arch])
linker.addArgs(self.linkOptions)
# gongjl ld add __LLVM __Bundle
linker.addArgs(["-bitcode_bundle"])
# handle bitcode input
bitcode_files = self.getFileNode("Bitcode")
if len(bitcode_files) > 0:
compiler_jobs = map(self.constructBitcodeJob, bitcode_files)
linker_inputs.extend(compiler_jobs)
# object input
object_files = self.getFileNode("Object")
if len(object_files) > 0:
if env.getPlatform() == "watchos":
env.error("Watch platform doesn't support object inputs")
object_jobs = map(self.constructObjectJob, object_files)
linker_inputs.extend(object_jobs)
# run compilation
env.map(self.run_job, linker_inputs)
# run bundle compilation in sequential to avoid dead-lock
bundle_files = self.getFileNode("Bundle")
if len(bundle_files) > 0:
bundle_jobs = map(self.constructBundleJob, bundle_files)
map(self.run_job, bundle_jobs)
linker_inputs.extend(bundle_jobs)
# sort object inputs
inputs = sorted([os.path.basename(x.output) for x in linker_inputs])
# handle LTO inputs
LTO_inputs = self.getFileNode("LTO")
if (len(LTO_inputs)) != 0:
lto_input_files = [x.find("name").text for x in LTO_inputs]
linker.addArgs(["-flto-codegen-only"])
linker.addArgs(["-object_path_lto", self.output + ".lto.o"])
linker.addArgs(ClangCC1Translator.compatibility_flags(self.arch))
# watchOS doesn't support inline asm.
if env.getPlatform() == "watchos":
linker.addArgs(["-mllvm", "-lto-module-no-asm"])
if self.is_translate_watchos:
lto_input_files = self.rewriteLTOInputFiles(lto_input_files)
linker.addArgs(
["-mllvm", "-aarch64-watch-bitcode-compatibility"])
inputs.extend(lto_input_files)
# add inputs to a LinkFileList
LinkFileList = os.path.join(self.dir, self.output + ".LinkFileList")
with open(LinkFileList, 'w') as f:
for i in inputs:
f.write(os.path.join(self.dir, i))
f.write('\n')
linker.addArgs(["-filelist", LinkFileList])
# version specific arguments
if env.satifiesLinkerVersion("253.2"):
linker.addArgs(["-ignore_auto_link"])
if env.satifiesLinkerVersion("253.3.1"):
linker.addArgs(["-allow_dead_duplicates"])
# add libLTO.dylib if needed
if env.liblto is not None:
linker.addArgs(["-lto_library", env.liblto])
# handle dylibs
dylibs_node = self.subdoc.find("dylibs")
if dylibs_node is not None:
for lib_node in dylibs_node.iter():
if lib_node.tag == "lib":
lib_path = env.resolveDylibs(self.arch, lib_node.text)
linker.addArgs([lib_path])
elif lib_node.tag == "weak":
# allow weak framework to be missing. If they provide no
# symbols, the link will succeed.
lib_path = env.resolveDylibs(self.arch, lib_node.text,
True)
if lib_path is not None:
linker.addArgs(["-weak_library", lib_path])
# add swift library search path, only when auto-link cannot be ignored.
if self.contain_swift and not env.satifiesLinkerVersion("253.2"):
swiftLibPath = env.getlibSwiftPath(self.arch)
if swiftLibPath is not None:
linker.addArgs(["-L", swiftLibPath])
# add libclang_rt
if self.forceload_compiler_rt:
linker.addArgs(["-force_load"])
linker.addArgs([env.getlibclang_rt(self.arch)])
# linking
try:
self.run_job(linker)
except BitcodeBuildFailure as e:
if self.contain_swift and self.is_translate_watchos and not self.force_optimize_swift:
env.warning("Rebuild failing swift project with optimization")
rebuild = BitcodeBundle(self.arch, self.input, self.output)
rebuild.force_optimize_swift = True
rebuild.is_compile_with_clang = True
return rebuild.run()
else:
raise e
else:
return self
