def __init__(self, arch, input_xar, output_path, use_xml=None): self.output = os.path.realpath(output_path) self.returncode = 0 self.stdout = "" self.arch = arch self.input = input_xar self.is_executable = False self.contain_swift = False self.deployment_target = None self.force_optimize_swift = False self.is_compile_with_clang = env.compile_with_clang # zf: read from xar xml during super class initialization super(BitcodeBundle, self).__init__(input_xar) if use_xml: data = "" with open(use_xml, 'r') as file: data = file.read() self.xml = ET.fromstring(data) try: self.platform = self.subdoc.find("platform").text self.sdk_version = self.subdoc.find("sdkversion").text self.version = self.subdoc.find("version").text except AttributeError: env.error("Malformed Header for bundle") else: env.setVersion(self.version) env.setPlatform(self.platform) if env.translate_watchos and env.getPlatform( ) == "watchos" and arch == "armv7k": self.arch = "arm64_32"
def writeDsymUUIDMap(self, bundle_path): resource_dir = os.path.join(bundle_path, "Contents", "Resources") plist_template = u"""<?xml version="1.0" encoding="UTF-8"?>""" \ """<!DOCTYPE plist PUBLIC""" \ """ "-//Apple//DTD PLIST 1.0//EN" """ \ """"http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> <key>DBGOriginalUUID</key> <string>{UUID}</string> </dict> </plist>""" if not os.access(resource_dir, os.W_OK): env.error(u"Dsym bunlde not writeable: {}".format(bundle_path)) for arch in self.archs: try: old_uuid = self.uuid[arch] if env.translate_watchos and arch == "armv7k": new_uuid = self.output_uuid["arm64_32"] else: new_uuid = self.output_uuid[arch] except KeyError: env.error("Cannot generate uuid map in dsym bundle") with open(os.path.join(resource_dir, new_uuid + ".plist"), "w") as f: f.write(plist_template.format(UUID=old_uuid))
def run_job(self, job): """Run sub command and catch errors""" try: rv = job.run() except BitcodeBuildFailure: # Catch and log an error env.error(u"Failed to compile bundle: {}".format(self.input)) else: return rv
def installOutput(self, path): if len(self.output_slices) == 0: env.error("Install failed: no bitcode build yet") elif len(self.output_slices) == 1: try: shutil.move(self.output_slices[0].output, path) except IOError: env.error(u"Install failed: can't create {}".format(path)) else: cmdtool.LipoCreate([x.output for x in self.output_slices], path).run() self.output_uuid = MachoType.getUUID(path)
def getArch(path): macho_info = cmdtool.MachoInfo(path).run() if macho_info.returncode != 0: env.error(u"{} is not valid macho file".format(path)) elif macho_info.stdout.startswith("Non-fat"): arch = macho_info.stdout.split()[-1] # Last phrase is arch return [arch] else: message = macho_info.stdout.split() try: begin = message.index("are:") + 1 except ValueError: env.error("Cound not detect architecture of the MachO file") else: return message[begin:]
def linkOptions(self): """Return all the link options""" linker_options = [ x.text if x.text is not None else "" for x in self.subdoc.find("link-options").findall("option") ] if not ld_option_verifier.verify(linker_options): env.error(u"Linker option verification " "failed for bundle {} ({})".format( self.input, ld_option_verifier.error_msg)) if linker_options.count("-execute") != 0: self.is_executable = True if self.platform is not None and self.platform != "Unknown": linker_options.extend(["-syslibroot", env.getSDK()]) if self.sdk_version is not None and self.sdk_version != "NA": linker_options.extend(["-sdk_version", self.sdk_version]) return linker_options
def __init__(self, arch, input_xar, output_path): self.output = os.path.realpath(output_path) self.returncode = 0 self.stdout = "" self.arch = arch self.is_executable = False self.contain_swift = False super(BitcodeBundle, self).__init__(input_xar) try: self.platform = self.subdoc.find("platform").text self.sdk_version = self.subdoc.find("sdkversion").text self.version = self.subdoc.find("version").text except AttributeError: env.error("Malformed Header for bundle") else: env.setVersion(self.version) env.setPlatform(self.platform)
def constructBitcodeJob(self, xml_node): """construct a single bitcode workload""" name = xml_node.find("name").text output_name = name + ".o" if xml_node.find("clang") is not None: clang = Clang(name, output_name, self.dir) options = [ x.text if x.text is not None else "" for x in xml_node.find("clang").findall("cmd") ] options = ClangCC1Translator.translate(options) if (clang_option_verifier.verify(options)): clang.addArgs(options) else: env.error(u"Clang option verification " "failed for bitcode {} ({})".format( name, clang_option_verifier.error_msg)) return clang elif xml_node.find("swift") is not None: # swift uses extension to distinguish input type # we need to move the file to have .bc extension first self.contain_swift = True if env.compile_with_clang: clang = Clang(name, output_name, self.dir) options = [ x.text if x.text is not None else "" for x in xml_node.find("swift").findall("cmd") ] if (swift_option_verifier.verify(options)): clang.addArgs(SwiftArgTranslator.translate(options)) else: env.error(u"Swift option verification " "failed for bitcode {} ({})".format( name, clang_option_verifier.error_msg)) return clang else: bcname = name + ".bc" shutil.move(os.path.join(self.dir, name), os.path.join(self.dir, bcname)) swift = Swift(bcname, output_name, self.dir) options = [ x.text if x.text is not None else "" for x in xml_node.find("swift").findall("cmd") ] if (swift_option_verifier.verify(options)): swift.addArgs(options) else: env.error(u"Swift option verification " "failed for bitcode {} ({})".format( name, swift_option_verifier.error_msg)) return swift else: env.error("Cannot figure out bitcode kind: {}".format(name))
def getXAR(self, arch): try: file = self._bitcode_cache[arch] except KeyError: macho_thin = self.getSlice(arch) extract_path = os.path.join(self._temp_dir, self.name + "." + arch + ".xar") extract_xar = cmdtool.ExtractXAR(macho_thin, extract_path).run() if extract_xar.returncode != 0: env.error(u"Cannot extract bundle from {} ({})".format( self.path, arch)) if os.stat(extract_path).st_size <= 1: env.error( u"Bundle only contains bitcode-marker {} ({})".format( self.path, arch)) self._bitcode_cache[arch] = extract_path return extract_path else: return file
def getSlice(self, arch): if arch not in self.archs: env.error(u"Requested arch {} doesn't exist in {}".format( arch, self.path)) if self.type == MachoType.Thin: return self.path elif self.type == MachoType.Fat: try: file = self._slice_cache[arch] except KeyError: extract_path = os.path.join(self._temp_dir, self.name + "." + arch) extract_job = cmdtool.ExtractSlice(self.path, arch, extract_path).run() if extract_job.returncode != 0: env.error(u"Cannot extract arch {} from {}".format( arch, self.path)) self._slice_cache[arch] = extract_path return extract_path else: return file
def main(args=None): """Run the program, can override args for testing.""" if args is None: args = sys.argv args = parse_args(args) try: env.initState(args) if not os.path.isfile(args.input_macho_file): env.error(u"Input macho file doesn't exist: {}".format( args.input_macho_file)) if args.symbol_map is not None and args.dsym_output is None: env.error("--symbol-map can only be used " "together with --generate-dsym") if args.symbol_map is not None and not os.path.exists(args.symbol_map): env.error(u"path passed to --symbol-map doesn't exists: {}".format( args.symbol_map)) input_macho = Macho(args.input_macho_file) if input_macho == MachoType.Error: env.error(u"Input is not a macho file: {}".format( args.input_macho_file)) map(input_macho.buildBitcode, input_macho.getArchs()) if (args.dsym_output is not None and not any([x.contain_symbols for x in input_macho.output_slices]) and args.symbol_map is None): env.warning( u"Cannot genarte useful dsym from input macho file: {}".format( args.input_macho_file)) if not args.verify: input_macho.installOutput(args.output) if args.dsym_output is not None: cmdtool.Dsymutil(args.output, args.dsym_output).run() input_macho.writeDsymUUIDMap(args.dsym_output) if args.symbol_map is not None: cmdtool.DsymMap(args.dsym_output, args.symbol_map).run() # always strip the output if input_macho.is_executable: cmdtool.StripSymbols(args.output).run() else: cmdtool.StripDebug(args.output, args.strip_swift).run() finally: env.cleanupTempDirectories()
def linkOptions(self): """Return all the link options""" linker_options = [ x.text if x.text is not None else "" for x in self.subdoc.find("link-options").findall("option") ] if not ld_option_verifier.verify(linker_options): env.error(u"Linker option verification " "failed for bundle {} ({})".format( self.input, ld_option_verifier.error_msg)) if linker_options.count("-execute") != 0: self.is_executable = True # make sure linker has a none zero version min for watchos. try: # check watchos version. version_min = linker_options.index("-watchos_version_min") # if valid version min location, check if it is 0.0 if version_min < (len(linker_options) - 1) and linker_options[ version_min + 1] == "0.0.0": # write a default watchos version. if self.is_translate_watchos: linker_options[version_min + 1] = "5.0.0" else: linker_options[version_min + 1] = "2.0.0" self.deployment_target = linker_options[version_min + 1] except ValueError: # if watchos is not specified during translate, add default deployment target. if self.is_translate_watchos: linker_options.extend(["-watchos_version_min", "5.0.0"]) if self.platform is not None and self.platform != "Unknown": linker_options.extend(["-syslibroot", env.getSDK()]) if self.sdk_version is not None and self.sdk_version != "NA": linker_options.extend(["-sdk_version", self.sdk_version]) return linker_options
def run_cmd(self, xfail=False): """Run a command in a working directory.""" start_time = datetime.datetime.now() try: if not os.environ.get('TESTING', False): out = subprocess.check_output(self.cmd, stderr=subprocess.STDOUT, cwd=self.working_dir) else: out = "Skipped for testing mode." end_time = datetime.datetime.now() except subprocess.CalledProcessError as e: self.returncode = e.returncode self.stdout = e.output if xfail: env.log(self) else: env.error(self) else: self.returncode = 0 self.stdout = out env.log(self) env.debug("Command took {} seconds".format( (end_time - start_time).seconds))
def __init__(self, xar_path): if os.path.isfile(xar_path): self.input = xar_path else: env.error(u"Input XAR doesn't exist: {}".format(xar_path)) cmd = [self.XAR_EXEC, "-d", "-", "-f", self.input] try: out = subprocess.check_output(cmd) except subprocess.CalledProcessError: env.error(u"toc cannot be extracted: {}".format(xar_path)) else: self.xml = ET.fromstring(out) self.dir = env.createTempDirectory() cmd = [self.XAR_EXEC, "-x", "-C", self.dir, "-f", self.input] try: out = subprocess.check_output(cmd) except subprocess.CalledProcessError: env.error(u"XAR cannot be extracted: {}".format(xar_path)) cmd = ['/bin/chmod', "-R", "+r", self.dir] try: out = subprocess.check_output(cmd) except subprocess.CalledProcessError: env.error(u"Permission fixup failed: {}".format(xar_path))
def main(args=None): """Run the program, can override args for testing.""" if args is None: args = sys.argv args = parse_args(args) try: env.initState(args) if not os.path.isfile(args.input_macho_file): env.error(u"Input macho file doesn't exist: {}".format( args.input_macho_file)) if args.symbol_map is not None and args.dsym_output is None: env.error("--symbol-map can only be used " "together with --generate-dsym") if args.symbol_map is not None and not os.path.exists(args.symbol_map): env.error(u"path passed to --symbol-map doesn't exists: {}".format( args.symbol_map)) ## todo: add pre-processing for lipo a static library and extract prelinked single .o ## ## After pre-processing, we have a macho file input_macho = Macho(args.input_macho_file) if input_macho == MachoType.Error: env.error(u"Input is not a macho file: {}".format( args.input_macho_file)) use_xml = args.use_xml if use_xml is not None and \ not os.path.isfile(args.input_macho_file): env.error(u"Input XML file doesn't exist: {}".format(args.use_xml)) map(lambda arch: input_macho.buildBitcode(arch, use_xml), input_macho.getArchs()) if (args.dsym_output is not None and not any([x.contain_symbols for x in input_macho.output_slices]) and args.symbol_map is None): env.warning( u"Cannot genarte useful dsym from input macho file: {}".format( unicode(args.input_macho_file, 'utf-8'))) if not args.verify: input_macho.installOutput(args.output) if args.dsym_output is not None: cmdtool.Dsymutil(args.output, args.dsym_output).run() input_macho.writeDsymUUIDMap(args.dsym_output) if args.symbol_map is not None: cmdtool.DsymMap(args.dsym_output, args.symbol_map).run() # always strip the output if input_macho.is_executable: cmdtool.StripSymbols(args.output).run() else: cmdtool.StripDebug(args.output, args.strip_swift).run() ## todo: add post-processing ## ## After pre-processing, we have a macho file finally: env.cleanupTempDirectories()
def run(self): """Build Bitcode Bundle""" linker_inputs = [] linker = Ld(self.output, self.dir) linker.addArgs(["-arch", self.arch]) linker.addArgs(self.linkOptions) # handle bitcode input bitcode_files = self.getFileNode("Bitcode") if len(bitcode_files) > 0: compiler_jobs = map(self.constructBitcodeJob, bitcode_files) linker_inputs.extend(compiler_jobs) # object input object_files = self.getFileNode("Object") if len(object_files) > 0: if env.getPlatform() == "watchos": env.error("Watch platform doesn't support object inputs") object_jobs = map(self.constructObjectJob, object_files) linker_inputs.extend(object_jobs) # run compilation env.map(self.run_job, linker_inputs) # run bundle compilation in sequential to avoid dead-lock bundle_files = self.getFileNode("Bundle") if len(bundle_files) > 0: bundle_jobs = map(self.constructBundleJob, bundle_files) map(self.run_job, bundle_jobs) linker_inputs.extend(bundle_jobs) # sort object inputs inputs = sorted([os.path.basename(x.output) for x in linker_inputs]) # handle LTO inputs LTO_inputs = self.getFileNode("LTO") if (len(LTO_inputs)) != 0: inputs.extend([x.find("name").text for x in LTO_inputs]) linker.addArgs(["-flto-codegen-only"]) # add inputs to a LinkFileList LinkFileList = os.path.join(self.dir, self.output + ".LinkFileList") with open(LinkFileList, 'w') as f: for i in inputs: f.write(os.path.join(self.dir, i)) f.write('\n') linker.addArgs(["-filelist", LinkFileList]) # version specific arguments if env.satifiesLinkerVersion("253.2"): linker.addArgs(["-ignore_auto_link"]) if env.satifiesLinkerVersion("253.3.1"): linker.addArgs(["-allow_dead_duplicates"]) # add libLTO.dylib if needed if env.liblto is not None: linker.addArgs(["-lto_library", env.liblto]) # handle dylibs dylibs_node = self.subdoc.find("dylibs") if dylibs_node is not None: for lib_node in dylibs_node.iter(): if lib_node.tag == "lib": lib_path = env.resolveDylibs(self.arch, lib_node.text) linker.addArgs([lib_path]) elif lib_node.tag == "weak": # allow weak framework to be missing. If they provide no # symbols, the link will succeed. lib_path = env.resolveDylibs(self.arch, lib_node.text, True) if lib_path is not None: linker.addArgs(["-weak_library", lib_path]) # add swift library search path, only when auto-link cannot be ignored. if self.contain_swift and not env.satifiesLinkerVersion("253.2"): swiftLibPath = env.getlibSwiftPath(self.arch) if swiftLibPath is not None: linker.addArgs(["-L", swiftLibPath]) # add libclang_rt if self.forceload_compiler_rt: linker.addArgs(["-force_load"]) linker.addArgs([env.getlibclang_rt(self.arch)]) # linking self.run_job(linker) return self
def constructBitcodeJob(self, xml_node): """construct a single bitcode workload""" name = xml_node.find("name").text output_name = name + ".o" if xml_node.find("clang") is not None: clang = Clang(name, output_name, self.dir) options = [ x.text if x.text is not None else "" for x in xml_node.find("clang").findall("cmd") ] options = ClangCC1Translator.upgrade(options, self.arch) if self.is_translate_watchos: options = ClangCC1Translator.translate_triple(options) if clang_option_verifier.verify(options): clang.addArgs(options) else: env.error(u"Clang option verification " "failed for bitcode {} ({})".format( name, clang_option_verifier.error_msg)) if env.getPlatform() == "watchos": clang.addArgs(["-fno-gnu-inline-asm"]) # gongjl build bitcode and cmd clang.addArgs(["-fembed-bitcode=all"]) return clang elif xml_node.find("swift") is not None: # swift uses extension to distinguish input type # we need to move the file to have .bc extension first self.contain_swift = True if self.is_compile_with_clang: clang = Clang(name, output_name, self.dir) options = [ x.text if x.text is not None else "" for x in xml_node.find("swift").findall("cmd") ] options = SwiftArgTranslator.upgrade(options, self.arch) if swift_option_verifier.verify(options): options = SwiftArgTranslator.translate_to_clang(options) if self.force_optimize_swift: options = ClangCC1Translator.add_optimization(options) if self.is_translate_watchos: options = ClangCC1Translator.translate_triple(options) clang.addArgs(options) else: env.error(u"Swift option verification " "failed for bitcode {} ({})".format( name, clang_option_verifier.error_msg)) return clang else: bcname = name + ".bc" shutil.move(os.path.join(self.dir, name), os.path.join(self.dir, bcname)) swift = Swift(bcname, output_name, self.dir) options = [ x.text if x.text is not None else "" for x in xml_node.find("swift").findall("cmd") ] if swift_option_verifier.verify(options): if self.force_optimize_swift: options = SwiftArgTranslator.add_optimization(options) if self.is_translate_watchos: options = SwiftArgTranslator.translate_triple(options) swift.addArgs(options) else: env.error(u"Swift option verification " "failed for bitcode {} ({})".format( name, swift_option_verifier.error_msg)) return swift else: env.error("Cannot figure out bitcode kind: {}".format(name))
def run(self): """Build Bitcode Bundle""" linker_inputs = [] linker = Ld(self.output, self.dir) linker.addArgs(["-arch", self.arch]) linker.addArgs(self.linkOptions) # gongjl ld add __LLVM __Bundle linker.addArgs(["-bitcode_bundle"]) # handle bitcode input bitcode_files = self.getFileNode("Bitcode") if len(bitcode_files) > 0: compiler_jobs = map(self.constructBitcodeJob, bitcode_files) linker_inputs.extend(compiler_jobs) # object input object_files = self.getFileNode("Object") if len(object_files) > 0: if env.getPlatform() == "watchos": env.error("Watch platform doesn't support object inputs") object_jobs = map(self.constructObjectJob, object_files) linker_inputs.extend(object_jobs) # run compilation env.map(self.run_job, linker_inputs) # run bundle compilation in sequential to avoid dead-lock bundle_files = self.getFileNode("Bundle") if len(bundle_files) > 0: bundle_jobs = map(self.constructBundleJob, bundle_files) map(self.run_job, bundle_jobs) linker_inputs.extend(bundle_jobs) # sort object inputs inputs = sorted([os.path.basename(x.output) for x in linker_inputs]) # handle LTO inputs LTO_inputs = self.getFileNode("LTO") if (len(LTO_inputs)) != 0: lto_input_files = [x.find("name").text for x in LTO_inputs] linker.addArgs(["-flto-codegen-only"]) linker.addArgs(["-object_path_lto", self.output + ".lto.o"]) linker.addArgs(ClangCC1Translator.compatibility_flags(self.arch)) # watchOS doesn't support inline asm. if env.getPlatform() == "watchos": linker.addArgs(["-mllvm", "-lto-module-no-asm"]) if self.is_translate_watchos: lto_input_files = self.rewriteLTOInputFiles(lto_input_files) linker.addArgs( ["-mllvm", "-aarch64-watch-bitcode-compatibility"]) inputs.extend(lto_input_files) # add inputs to a LinkFileList LinkFileList = os.path.join(self.dir, self.output + ".LinkFileList") with open(LinkFileList, 'w') as f: for i in inputs: f.write(os.path.join(self.dir, i)) f.write('\n') linker.addArgs(["-filelist", LinkFileList]) # version specific arguments if env.satifiesLinkerVersion("253.2"): linker.addArgs(["-ignore_auto_link"]) if env.satifiesLinkerVersion("253.3.1"): linker.addArgs(["-allow_dead_duplicates"]) # add libLTO.dylib if needed if env.liblto is not None: linker.addArgs(["-lto_library", env.liblto]) # handle dylibs dylibs_node = self.subdoc.find("dylibs") if dylibs_node is not None: for lib_node in dylibs_node.iter(): if lib_node.tag == "lib": lib_path = env.resolveDylibs(self.arch, lib_node.text) linker.addArgs([lib_path]) elif lib_node.tag == "weak": # allow weak framework to be missing. If they provide no # symbols, the link will succeed. lib_path = env.resolveDylibs(self.arch, lib_node.text, True) if lib_path is not None: linker.addArgs(["-weak_library", lib_path]) # add swift library search path, only when auto-link cannot be ignored. if self.contain_swift and not env.satifiesLinkerVersion("253.2"): swiftLibPath = env.getlibSwiftPath(self.arch) if swiftLibPath is not None: linker.addArgs(["-L", swiftLibPath]) # add libclang_rt if self.forceload_compiler_rt: linker.addArgs(["-force_load"]) linker.addArgs([env.getlibclang_rt(self.arch)]) # linking try: self.run_job(linker) except BitcodeBuildFailure as e: if self.contain_swift and self.is_translate_watchos and not self.force_optimize_swift: env.warning("Rebuild failing swift project with optimization") rebuild = BitcodeBundle(self.arch, self.input, self.output) rebuild.force_optimize_swift = True rebuild.is_compile_with_clang = True return rebuild.run() else: raise e else: return self