def _copy_swift_stdlibs(binaries_to_scan, swift_dylibs_path, sdk_platform, destination_path): """Copies the Swift stdlibs required by the binaries to the destination.""" # Use the currently selected Xcode developer directory and SDK to determine # exactly which Swift stdlibs we should be copying. developer_dir_cmd = ["xcode-select", "--print-path"] _, stdout, stderr = execute.execute_and_filter_output(developer_dir_cmd, raise_on_failure=True) if stderr: print(stderr) developer_dir = stdout.strip() library_source_dir = os.path.join( developer_dir, swift_dylibs_path, sdk_platform ) # Rely on the swift-stdlib-tool to determine the subset of Swift stdlibs that # these binaries require. cmd = [ "xcrun", "swift-stdlib-tool", "--copy", "--source-libraries", library_source_dir, "--platform", sdk_platform, "--destination", destination_path ] for binary_to_scan in binaries_to_scan: cmd.extend(["--scan-executable", binary_to_scan]) _, stdout, stderr = execute.execute_and_filter_output(cmd, raise_on_failure=True) if stderr: print(stderr) if stdout: print(stdout)
def invoke(input_path, output_path): """Wraps bitcode_strip with the given input and output.""" xcrunargs = ["xcrun", "bitcode_strip", input_path, "-r", "-o", output_path] execute.execute_and_filter_output(xcrunargs, print_output=True, raise_on_failure=True)
def _invoke_codesign(codesign_path, identity, entitlements, force_signing, disable_timestamp, full_path_to_sign): """Invokes the codesign tool on the given path to sign. Args: codesign_path: Path to the codesign tool as a string. identity: The unique identifier string to identify code signatures. entitlements: Path to the file with entitlement data. Optional. force_signing: If true, replaces any existing signature on the path given. disable_timestamp: If true, disables the use of timestamp services. full_path_to_sign: Path to the bundle or binary to code sign as a string. """ cmd = [codesign_path, '-v', '--sign', identity] if entitlements: cmd.extend([ '--entitlements', entitlements, ]) if force_signing: cmd.append('--force') if disable_timestamp: cmd.append('--timestamp=none') cmd.append(full_path_to_sign) # Just like Xcode, ensure CODESIGN_ALLOCATE is set to point to the correct # version. custom_env = {'CODESIGN_ALLOCATE': _find_codesign_allocate()} execute.execute_and_filter_output(cmd, filtering=_filter_codesign_tool_output, custom_env=custom_env, raise_on_failure=True, print_output=True)
def _execute_and_filter_with_retry(xcrunargs, filtering): # Note: `actool`/`ibtool` is problematic on all Xcode 12 builds including to 12.1. 25% # of the time, it fails with the error: # "failed to open # liblaunch_sim.dylib" # # This workaround adds a retry it works due to logic in `actool`: # The first time `actool` runs, it spawns a dependent service as the current # user. After a failure, `actool` spawns it in a way that subsequent # invocations will not have the error. It only needs 1 retry. return_code, stdout, stderr = execute.execute_and_filter_output( xcrunargs, trim_paths=True, filtering=filtering, print_output=False) # If there's a retry, don't print the first failing output. if return_code == 0: if stdout: sys.stdout.write("%s" % stdout) if stderr: sys.stderr.write("%s" % stderr) return return_code return_code, _, _ = execute.execute_and_filter_output( xcrunargs, trim_paths=True, filtering=filtering, print_output=True) return return_code
def _find_codesign_identities(identity=None): """Finds code signing identities on the current system.""" ids = [] _, output, _ = execute.execute_and_filter_output([ "security", "find-identity", "-v", "-p", "codesigning", ], raise_on_failure=True) output = output.strip() pattern = "(?P<hash>[A-F0-9]{40})" if identity: name_requirement = re.escape(identity) pattern += r'\s+".*?{}.*?"'.format(name_requirement) regex = re.compile(pattern) for line in output.splitlines(): # CSSMERR_TP_CERT_REVOKED comes from Security.framework/cssmerr.h if "CSSMERR_TP_CERT_REVOKED" in line: continue m = regex.search(line) if m: groups = m.groupdict() id = groups["hash"] ids.append(id) return ids
def find_archs_for_binaries(binary_list): """Queries lipo to identify binary archs from each of the binaries.""" found_architectures = set() for binary in binary_list: cmd = ["xcrun", "lipo", "-info", binary] _, stdout, stderr = execute.execute_and_filter_output(cmd, raise_on_failure=True) if stderr: print(stderr) if not stdout: print("Internal Error: Did not receive output from lipo for inputs: " + " ".join(cmd)) return None cut_output = stdout.split(":") if len(cut_output) < 3: print("Internal Error: Unexpected output from lipo, received: " + stdout) return None archs_found = cut_output[2].strip().split(" ") if not archs_found: print("Internal Error: Could not find architecture for binary: " + binary) return None for arch_found in archs_found: found_architectures.add(arch_found) return found_architectures
def ibtool(_, toolargs): """Assemble the call to "xcrun ibtool".""" xcrunargs = ["xcrun", "ibtool", "--errors", "--warnings", "--notices", "--auto-activate-custom-fonts", "--output-format", "human-readable-text"] _apply_realpath(toolargs) xcrunargs += toolargs # If we are running into problems figuring out "ibtool" issues, there are a # couple of environment variables that may help. Both of the following must be # set to work. # IBToolDebugLogFile=<OUTPUT FILE PATH> # IBToolDebugLogLevel=4 # You may also see if # IBToolNeverDeque=1 # helps. return execute.execute_and_filter_output( xcrunargs, trim_paths=True, filtering=ibtool_filtering)
def _find_smartcard_identities(identity=None): """Finds smartcard identitites on the current system.""" ids = [] _, xml, _ = execute.execute_and_filter_output([ "/usr/sbin/system_profiler", "SPSmartCardsDataType", "-xml" ], raise_on_failure=True) xml = plistlib.loads(str.encode(xml)) if len(xml) == 0: return [] xml = xml[0].get("_items", None) if not xml: return [] tokens = _get_smartcard_tokens(xml) keychain = _get_smartcard_keychain(xml) # For each 'token' finds non-expired certs and: # # 1. Check if 'identity' was provided and if it matches a 'CN', in that case stop the loop # and return the respective fingerprint (SHA1) # 2. Otherwise append fingerprints found to 'ids' to be returned at the end # # ps: note that if 'identity' is provided and it does not match any existing item in the # smartcard keychain 'ids' will be empty, making this function's behaviour consistent with # '_find_codesign_identities' where it's being called for token in tokens: token_data = [x for x in keychain if x.get("_name", None) == token] if len(token_data) == 0: continue token_data = token_data[0] for (k, data) in token_data.items(): if k == "_name": continue # Extract expiry date and ignore expired certs. The row being processed looks like this: # # Valid from: 2021-02-12 21:35:04 +0000 to: 2022-02-12 21:35:05 +0000, SSL trust: NO, X509 trust: YES # expiry_date = re.search(r"(?<=to:)(.*?)(?=,)", data, re.DOTALL).group().strip() expiry_date = datetime.datetime.strptime(expiry_date, "%Y-%m-%d %H:%M:%S %z") now = datetime.datetime.now(expiry_date.tzinfo) if now > expiry_date: continue # This is a valid identity, decode the certificate, extract # Common Name and Fingerprint and handle their values accordingly # as described above cert = re.search(r"(?<=-----BEGIN CERTIFICATE-----)(.*?)(?=-----END CERTIFICATE-----)", data, re.DOTALL).group().strip() cert = base64.b64decode(cert) cert = _certificate_data(cert) common_name = _certificate_common_name(cert) fingerprint = _certificate_fingerprint(cert) if identity == common_name: return [fingerprint] if not identity: ids.append(fingerprint) return ids
def actool(_, toolargs): """Assemble the call to "xcrun actool".""" xcrunargs = ["xcrun", "actool", "--errors", "--warnings", "--notices", "--compress-pngs", "--output-format", "human-readable-text"] _apply_realpath(toolargs) xcrunargs += toolargs # If we are running into problems figuring out "actool" issues, there are a # couple of environment variables that may help. Both of the following must be # set to work. # IBToolDebugLogFile=<OUTPUT FILE PATH> # IBToolDebugLogLevel=4 # You may also see if # IBToolNeverDeque=1 # helps. # Yes, IBTOOL appears to be correct here due to "actool" and "ibtool" being # based on the same codebase. return execute.execute_and_filter_output( xcrunargs, trim_paths=True, filtering=actool_filtering)
def _build_library_binary(archs, sdk, minimum_os_version, source_file, output_path): """Builds the library binary from a source file, writes to output_path.""" output_lib = os.path.join(os.path.dirname(output_path), os.path.basename(source_file) + ".o") # Start assembling the list of arguments with what we know will remain. # constant. library_cmd = [ "xcrun", "-sdk", sdk, "clang", _version_arg_for_sdk(sdk, minimum_os_version) ] # Append archs. for arch in archs: library_cmd.extend(["-arch", arch]) # Append source file. library_cmd.extend(["-c", source_file]) # Add the output library. if os.path.exists(output_lib): os.remove(output_lib) library_cmd.extend(["-o", output_lib]) # Run the command to assemble the output library. _, stdout, stderr = execute.execute_and_filter_output( library_cmd, raise_on_failure=True) if stdout: print(stdout) if stderr: print(stderr) return output_lib
def mapc(_, toolargs): """Assemble the call to "xcrun mapc".""" xcrunargs = ["xcrun", "mapc"] _apply_realpath(toolargs) xcrunargs += toolargs return execute.execute_and_filter_output(xcrunargs)
def _copy_swift_stdlibs(binaries_to_scan, swift_dylibs_paths, sdk_platform, destination_path): """Copies the Swift stdlibs required by the binaries to the destination.""" # Rely on the swift-stdlib-tool to determine the subset of Swift stdlibs that # these binaries require. developer_dir = os.environ["DEVELOPER_DIR"] swift_library_dirs = [ os.path.join(developer_dir, dylibs_path, sdk_platform) for dylibs_path in swift_dylibs_paths ] cmd = [ "xcrun", "swift-stdlib-tool", "--copy", "--platform", sdk_platform, "--destination", destination_path ] for swift_library_dir in swift_library_dirs: cmd.extend(["--source-libraries", swift_library_dir]) for binary_to_scan in binaries_to_scan: cmd.extend(["--scan-executable", binary_to_scan]) _, stdout, stderr = execute.execute_and_filter_output( cmd, raise_on_failure=True) if stderr: print(stderr) if stdout: print(stdout)
def main(): parser = argparse.ArgumentParser( prog="Merges multiple plist files into a single one", description="""This script takes multiple `--input` arguments that point to each input plist file. When key duplication occurs, the value from the first input file wins.""") parser.add_argument( "--input", required=True, action="append", help="Path to an input plist, may be used multiple times.") parser.add_argument("--output", required=True, help="Path to the output plist.") parser.add_argument( "--output_format", required=False, default="xml1", help="The output plist format, can be one of 'xml1', 'binary1', " "'json', 'swift', and 'objc'.") args = parser.parse_args() # Merge each plist to the output file for input in args.input: cmd = [ "/usr/libexec/PlistBuddy", "-c", "Merge {}".format(input), args.output ] _, _, stderr = execute.execute_and_filter_output(cmd, raise_on_failure=True) if stderr: print(stderr) # Convert to a non-XML format if requested if args.output_format != "xml1": cmd = ["/usr/bin/plutil", "-convert", args.output_format, args.output] _, stdout, stderr = execute.execute_and_filter_output( cmd, raise_on_failure=True) if stdout: print(stdout) if stderr: print(stderr)
def test_execute_unicode(self): bytes_out = u'\u201d '.encode('utf8') + _INVALID_UTF8 args = ['echo', '-n', bytes_out] with self._mock_streams() as (mock_stdout, mock_stderr): execute.execute_and_filter_output(args, filtering=_cmd_filter) stdout = mock_stdout.getvalue() stderr = mock_stderr.getvalue() if _PY3: expected = bytes_out.decode('utf8', 'replace') else: expected = bytes_out expected += ' filtered' self.assertEqual(expected, stdout) self.assertIn('filtered', stderr)
def mapc(_, toolargs): """Assemble the call to "xcrun mapc".""" xcrunargs = ["xcrun", "mapc"] _apply_realpath(toolargs) xcrunargs += toolargs return_code, _, _ = execute.execute_and_filter_output(xcrunargs, print_output=True) return return_code
def intentbuilderc(args, toolargs): """Assemble the call to "xcrun intentbuilderc".""" xcrunargs = ["xcrun", "intentbuilderc"] _apply_realpath(toolargs) is_swift = args.language == "Swift" output_path = None objc_output_srcs = None objc_output_hdrs = None # If the language is Swift, create a temporary directory for codegen output. # If the language is Objective-C, ensure the module name directory and headers # are created and empty (clean). if is_swift: output_path = "{}.out.tmp".format(args.swift_output_src) else: output_path = args.objc_output_srcs _ensure_clean_path(args.objc_output_hdrs) _ensure_clean_path(output_path) output_path = os.path.realpath(output_path) toolargs += [ "-language", args.language, "-output", output_path, ] xcrunargs += toolargs return_code, _, _ = execute.execute_and_filter_output(xcrunargs, print_output=True) if return_code != 0: return return_code # If the language is Swift, concatenate all the output files into one. # If the language is Objective-C, put the headers into the pre-declared # headers directory. Because the .m files reference headers via quotes, copy # them instead of moving them and doing some -iquote fu. if is_swift: with open(args.swift_output_src, "w") as output_src: for src in _listdir_full(output_path): with open(src) as intput_src: shutil.copyfileobj(intput_src, output_src) else: with open(args.objc_public_header, "w") as public_header_f: for source_file in _listdir_full(output_path): if source_file.endswith(_HEADER_SUFFIX): out_hdr = os.path.join(args.objc_output_hdrs, os.path.basename(source_file)) shutil.copy(source_file, out_hdr) public_header_f.write("#import \"{}\"\n".format( os.path.relpath(out_hdr))) return return_code
def test_execute_unicode(self): bytes_out = u'\u201d '.encode('utf8') + _INVALID_UTF8 args = ['echo', '-n', bytes_out] with contextlib.redirect_stdout(io.StringIO()) as mock_stdout, \ contextlib.redirect_stderr(io.StringIO()) as mock_stderr: execute.execute_and_filter_output(args, filtering=_cmd_filter, print_output=True, raise_on_failure=False) stdout = mock_stdout.getvalue() stderr = mock_stderr.getvalue() expected = bytes_out.decode('utf8', 'replace') expected += ' filtered' self.assertEqual(expected, stdout) self.assertIn('filtered', stderr)
def _build_framework_binary(name, sdk, minimum_os_version, framework_path, libtype, embed_bitcode, embed_debug_info, archs, source_file): """Builds the framework binary from a source file, saves to framework_path.""" output_lib = _build_library_binary(archs, sdk, minimum_os_version, embed_bitcode, embed_debug_info, source_file, framework_path) # Delete any existing framework files, if they are already there. if os.path.exists(framework_path): shutil.rmtree(framework_path) os.makedirs(framework_path) # Run the command to assemble the framework output. framework_cmd = "" custom_env = None if libtype == "dynamic": framework_cmd = _generate_dynamic_cmd(name, sdk, minimum_os_version, framework_path, archs) elif libtype == "static": framework_cmd = ["xcrun", "libtool"] custom_env = {"ZERO_AR_DATE": "1"} else: print("Internal Error: Unexpected library type: {}".format(libtype)) return 1 framework_cmd.append(output_lib) framework_cmd.extend([ "-o", os.path.join(framework_path, name), ]) if embed_bitcode: bcsymbolmap_path = os.path.join(os.path.dirname(framework_path), os.path.basename(name) + ".bcsymbolmap") framework_cmd.extend([ "-fembed-bitcode", "-Xlinker", "-bitcode_verify", "-Xlinker", "-bitcode_hide_symbols", "-Xlinker", "-bitcode_symbol_map", "-Xlinker", bcsymbolmap_path, ]) _, stdout, stderr = execute.execute_and_filter_output(framework_cmd, custom_env=custom_env, raise_on_failure=True) if stdout: print(stdout) if stderr: print(stderr) return 0
def invoke(input_path, output_path): """Wraps bitcode_strip with the given input and output.""" xcrunargs = ["xcrun", "bitcode_strip", input_path, "-r", "-o", output_path] _, stdout, stderr = execute.execute_and_filter_output( xcrunargs, raise_on_failure=True) if stdout: print(stdout) if stderr: print(stderr)
def find_archs_for_binaries(binary_list): """Queries lipo to identify binary archs from each of the binaries. Args: binary_list: A list of strings, each of which is the path to a binary whose architectures should be retrieved. Returns: A tuple containing two values: 1. A set containing the union of all architectures found in every binary. 2. A dictionary where each key is one of the elements in `binary_list` and the corresponding value is the set of architectures found in that binary. If there was an error invoking `lipo` or the output was something unexpected, `None` will be returned for both tuple elements. """ found_architectures = set() archs_by_binary = dict() for binary in binary_list: cmd = ["xcrun", "lipo", "-info", binary] _, stdout, stderr = execute.execute_and_filter_output( cmd, raise_on_failure=True) if stderr: print(stderr) if not stdout: print( "Internal Error: Did not receive output from lipo for inputs: " + " ".join(cmd)) return (None, None) cut_output = stdout.split(":") if len(cut_output) < 3: print("Internal Error: Unexpected output from lipo, received: " + stdout) return (None, None) archs_found = cut_output[2].strip().split(" ") if not archs_found: print("Internal Error: Could not find architecture for binary: " + binary) return (None, None) archs_by_binary[binary] = set(archs_found) for arch_found in archs_found: found_architectures.add(arch_found) return (found_architectures, archs_by_binary)
def _certificate_fingerprint(identity): """Extracts a fingerprint given identity in a mobileprovision file.""" _, fingerprint, _ = execute.execute_and_filter_output([ "openssl", "x509", "-inform", "DER", "-noout", "-fingerprint", ], inputstr=identity, raise_on_failure=True) fingerprint = fingerprint.strip() fingerprint = fingerprint.replace("SHA1 Fingerprint=", "") fingerprint = fingerprint.replace(":", "") return fingerprint
def _certificate_common_name(cert): _, subject, _ = execute.execute_and_filter_output( ["openssl", "x509", "-noout", "-inform", "DER", "-subject"], inputstr=cert, raise_on_failure=True) subject = subject.strip().split('/') cert_cn = [f for f in subject if "CN=" in f] if len(cert_cn) == 0: return None cert_cn = cert_cn[0] cert_cn = cert_cn.replace("CN=", "") return cert_cn
def _certificate_fingerprint(identity): """Extracts a fingerprint given identity in a provisioning profile.""" openssl_command = [ 'openssl', 'x509', '-inform', 'DER', '-noout', '-fingerprint', ] _, fingerprint, _ = execute.execute_and_filter_output( openssl_command, inputstr=identity, raise_on_failure=True) fingerprint = fingerprint.strip() fingerprint = fingerprint.replace('SHA1 Fingerprint=', '') fingerprint = fingerprint.replace(':', '') return fingerprint
def _copy_swift_stdlibs(binaries_to_scan, sdk_platform, destination_path): """Copies the Swift stdlibs required by the binaries to the destination.""" # Rely on the swift-stdlib-tool to determine the subset of Swift stdlibs that # these binaries require. cmd = [ "xcrun", "swift-stdlib-tool", "--copy", "--platform", sdk_platform, "--destination", destination_path ] for binary_to_scan in binaries_to_scan: cmd.extend(["--scan-executable", binary_to_scan]) _, stdout, stderr = execute.execute_and_filter_output( cmd, raise_on_failure=True) if stderr: print(stderr) if stdout: print(stdout)
def invoke_lipo(binary_path, binary_slices, output_path): """Wraps lipo with given arguments for inputs and outputs.""" cmd = ["xcrun", "lipo", binary_path] # Create a thin binary if there's only one needed slice, otherwise create a # universal binary if len(binary_slices) == 1: cmd.extend(["-thin", next(iter(binary_slices))]) else: for binary_slice in binary_slices: cmd.extend(["-extract", binary_slice]) cmd.extend(["-output", output_path]) _, stdout, stderr = execute.execute_and_filter_output(cmd, raise_on_failure=True) if stdout: print(stdout) if stderr: print(stderr)
def swift_stdlib_tool(args, toolargs): """Assemble the call to "xcrun swift-stdlib-tool" and zip the output.""" tmpdir = tempfile.mkdtemp(prefix="swiftstdlibtoolZippingOutput.") destination = os.path.join(tmpdir, args.bundle) xcrunargs = [ "xcrun", "swift-stdlib-tool", "--copy", "--destination", destination ] xcrunargs += toolargs result = execute.execute_and_filter_output(xcrunargs) if not result: _zip_directory(tmpdir, os.path.splitext(args.output)[0]) shutil.rmtree(tmpdir) return result
def momc(args, toolargs): """Assemble the call to "xcrun momc".""" xcrunargs = ["xcrun", "momc"] _apply_realpath(toolargs) xcrunargs += toolargs return_code, _, _ = execute.execute_and_filter_output( xcrunargs, print_output=True) destination_dir = args.xctoolrunner_assert_nonempty_dir if args.xctoolrunner_assert_nonempty_dir and not os.listdir(destination_dir): raise FileNotFoundError( f"xcrun momc did not generate artifacts at: {destination_dir}\n" "Core Data model was not configured to have code generation.") return return_code
def actool(_, toolargs): """Assemble the call to "xcrun actool".""" xcrunargs = ["xcrun", "actool", "--errors", "--warnings", "--notices", "--output-format", "human-readable-text"] _apply_realpath(toolargs) xcrunargs += toolargs # The argument coming after "--compile" is the output directory. "actool" # expects an directory to exist at that path. Create an empty directory there # if one doesn't exist yet. for idx, arg in enumerate(toolargs): if arg == "--compile": output_dir = toolargs[idx + 1] if not os.path.exists(output_dir): os.makedirs(output_dir) break # If we are running into problems figuring out "actool" issues, there are a # couple of environment variables that may help. Both of the following must be # set to work. # IBToolDebugLogFile=<OUTPUT FILE PATH> # IBToolDebugLogLevel=4 # You may also see if # IBToolNeverDeque=1 # helps. # Yes, IBTOOL appears to be correct here due to "actool" and "ibtool" being # based on the same codebase. return_code, _, _ = execute.execute_and_filter_output( xcrunargs, trim_paths=True, filtering=actool_filtering, print_output=True) return return_code
def _invoke_codesign(codesign_path, identity, entitlements, force_signing, disable_timestamp, full_path_to_sign, extra): """Invokes the codesign tool on the given path to sign. Args: codesign_path: Path to the codesign tool as a string. identity: The unique identifier string to identify code signatures. entitlements: Path to the file with entitlement data. Optional. force_signing: If true, replaces any existing signature on the path given. disable_timestamp: If true, disables the use of timestamp services. full_path_to_sign: Path to the bundle or binary to code sign as a string. """ cmd = [codesign_path, "-v", "--sign", identity] if entitlements: cmd.extend([ "--generate-entitlement-der", "--entitlements", entitlements, ]) if force_signing: cmd.append("--force") if disable_timestamp: cmd.append("--timestamp=none") cmd.append(full_path_to_sign) cmd.extend(extra) # Just like Xcode, ensure CODESIGN_ALLOCATE is set to point to the correct # version. custom_env = {"CODESIGN_ALLOCATE": _find_codesign_allocate()} _, stdout, stderr = execute.execute_and_filter_output(cmd, custom_env=custom_env, raise_on_failure=True) if stdout: filtered_stdout = _filter_codesign_output(stdout) if filtered_stdout: print(filtered_stdout) if stderr: filtered_stderr = _filter_codesign_output(stderr) if filtered_stderr: print(filtered_stderr)
def _find_codesign_allocate(): cmd = ['xcrun', '--find', 'codesign_allocate'] _, stdout, _ = execute.execute_and_filter_output(cmd, raise_on_failure=True) return stdout.strip()