def get_uuid_from_log(log_path): """Get the UUID from the notarization log. Args: log_path (str): the path to the log Raises: IScriptError: if we can't find the UUID ThrottledNotarization: if there's an ``ERROR ITMS-10004`` in the response UnknownNotarizationError: if there's an unknown ``ERROR`` in the response Returns: str: the uuid """ regex = re.compile(r"RequestUUID = (?P<uuid>[a-zA-Z0-9-]+)") try: with open(log_path, "r") as fh: contents = fh.read() log.info(f"{log_path} notarization response:\n{contents}") exception = None if "ERROR ITMS-10004" in contents: exception = ThrottledNotarization elif "ERROR " in contents: exception = UnknownNotarizationError if exception is not None: raise exception(f"Error response from Apple!\n{contents}") for line in contents.splitlines(): m = regex.search(line) if m: return m["uuid"] except OSError as err: raise IScriptError("Can't find UUID in {}: {}".format(log_path, err)) from err raise IScriptError("Can't find UUID in {}!".format(log_path))
async def async_main(config, task): """Sign all the things. Args: config (dict): the running config. task (dict): the running task. """ await run_command(["hostname"]) base_key = "mac_config" # We may support ios_config someday key_config = get_key_config(config, task, base_key=base_key) behavior = task["payload"].get("behavior", "mac_sign") if behavior == "mac_notarize" and "mac_notarize" not in key_config[ "supported_behaviors"] and "mac_sign_and_pkg" in key_config[ "supported_behaviors"]: behavior = "mac_sign_and_pkg" if behavior not in key_config["supported_behaviors"]: raise IScriptError("Unsupported behavior {} given scopes {}!".format( behavior, task["scopes"])) if behavior == "mac_geckodriver": await geckodriver_behavior(config, task) return elif behavior == "mac_notarize": await notarize_behavior(config, task) return elif behavior == "mac_sign": await sign_behavior(config, task) return elif behavior == "mac_sign_and_pkg": # For staging releases; or should we mac_notarize but skip notarization # for dep? await sign_and_pkg_behavior(config, task) return raise IScriptError("Unknown iscript behavior {}!".format(behavior))
def _ensure_one_precomplete(tmp_dir, adj): """Ensure we only have one `precomplete` file in `tmp_dir`.""" precompletes = glob.glob(os.path.join(tmp_dir, "**", "precomplete"), recursive=True) if len(precompletes) < 1: raise IScriptError('No `precomplete` file found in "%s"', tmp_dir) if len(precompletes) > 1: raise IScriptError('More than one `precomplete` file %s in "%s"', adj, tmp_dir) return precompletes[0]
async def create_one_notarization_zipfile(work_dir, all_paths, sign_config, path_attrs=("app_path", "pkg_path")): """Create a single notarization zipfile for all the apps. Args: work_dir (str): the script work directory all_paths (list): list of ``App`` objects path_attrs (tuple, optional): the attributes for the paths we'll be zipping up. Defaults to ``("app_path", "pkg_path")`` Raises: IScriptError: on failure Returns: str: the zip path """ required_attrs = path_attrs app_paths = [] zip_path = os.path.join(work_dir, "notarization.zip") for app in all_paths: app.check_required_attrs(required_attrs) for path_attr in path_attrs: app_paths.append(os.path.relpath(getattr(app, path_attr), work_dir)) if sign_config["zipfile_cmd"] == "zip": await run_command(["zip", "-r", zip_path, *app_paths], cwd=work_dir, exception=IScriptError) elif sign_config["zipfile_cmd"] == "ditto": await run_command(["ditto", "-c", "-k", "--sequesterRsrc", "--keepParent", "0", zip_path], cwd=work_dir, exception=IScriptError) else: raise IScriptError(f"Unknown zipfile_cmd {sign_config['zipfile_cmd']}!") return zip_path
async def sign_langpacks(config, key_config, all_paths): """Signs langpacks that are specified in all_paths. Raises: IScriptError if we don't have any valid language packs to sign in any path. """ for app in all_paths: app.check_required_attrs(["orig_path", "formats", "artifact_prefix"]) if not {"autograph_langpack"} & set(app.formats): raise IScriptError( f"{app.formats} does not contain 'autograph_langpack'") app.target_tar_path = "{}/{}{}".format( config["artifact_dir"], app.artifact_prefix, app.orig_path.split(app.artifact_prefix)[1], ) id = langpack_id(app) log.info("Identified {} as extension id: {}".format(app.orig_path, id)) makedirs(os.path.dirname(app.target_tar_path)) await sign_file_with_autograph( key_config, app.orig_path, "autograph_langpack", to=app.target_tar_path, extension_id=id, )
async def fake_raise_future_exceptions(futures, **kwargs): """``raise_future_exceptions`` mocker.""" await asyncio.wait(futures) assert len(futures) == futures_len.pop(0) if raises: raise IScriptError("foo")
def get_key_config(config, task, base_key="mac_config"): """Sanity check the task scopes and return the appropriate ``key_config``. The ``key_config`` is, e.g. the ``config.mac_config.dep`` dictionary, for mac dep-signing. Args: config (dict): the running config task (dict): the running task base_key (str, optional): the base key in the dictionary. Defaults to ``mac_config``. Raises: IScriptError: on failure to verify the scopes. Returns: dict: the ``key_config`` """ try: cert_type = task_cert_type(config, task) return config[base_key][_CERT_TYPE_TO_KEY_CONFIG[cert_type]] except KeyError as exc: raise IScriptError("get_key_config error: {}".format( str(exc))) from exc
async def sign_geckodriver(config, sign_config, all_paths): """Sign geckodriver. Args: sign_config (dict): the running config all_paths (list): list of App objects Raises: IScriptError: on error. """ identity = sign_config["identity"] keychain = sign_config["signing_keychain"] sign_command = _get_sign_command(identity, keychain, sign_config) for app in all_paths: app.check_required_attrs(["orig_path", "parent_dir", "artifact_prefix"]) app.target_tar_path = "{}/{}{}".format(config["artifact_dir"], app.artifact_prefix, app.orig_path.split(app.artifact_prefix)[1]) file_ = "geckodriver" path = os.path.join(app.parent_dir, file_) if not os.path.exists(path): raise IScriptError(f"No such file {path}!") await retry_async( run_command, args=[sign_command + [file_]], kwargs={"cwd": app.parent_dir, "exception": IScriptError, "output_log_on_exception": True}, retry_exceptions=(IScriptError,), ) env = deepcopy(os.environ) # https://superuser.com/questions/61185/why-do-i-get-files-like-foo-in-my-tarball-on-os-x env["COPYFILE_DISABLE"] = "1" makedirs(os.path.dirname(app.target_tar_path)) await run_command( ["tar", _get_tar_create_options(app.target_tar_path), app.target_tar_path, file_], cwd=app.parent_dir, env=env, exception=IScriptError )
async def unlock_keychain(signing_keychain, keychain_password): """Unlock the signing keychain. Args: signing_keychain (str): the path to the signing keychain keychain_password (str): the keychain password Raises: IScriptError: on failure TimeoutFailure: on timeout """ log.info("Unlocking signing keychain {}".format(signing_keychain)) child = pexpect.spawn("security", ["unlock-keychain", signing_keychain], encoding="utf-8") try: while True: index = await child.expect( [ pexpect.EOF, r"password to unlock {}: ".format(signing_keychain) ], async_=True, ) if index == 0: break child.sendline(keychain_password) except (pexpect.exceptions.TIMEOUT) as exc: raise TimeoutError( "Timeout trying to unlock the keychain {}: {}!".format( signing_keychain, exc)) from exc child.close() if child.exitstatus != 0 or child.signalstatus is not None: raise IScriptError("Failed unlocking {}! exit {} signal {}".format( signing_keychain, child.exitstatus, child.signalstatus))
async def fake_run_command(*args, **kwargs): assert args[0] == [ "zip", "-r", os.path.join(work_dir, "app_path.zip"), "0/0.app", "1/1.app", "2/2.app" ] if raises: raise IScriptError("foo")
def _get_tar_create_options(path): base_opts = "c" if path.endswith(".tar.gz"): return "{}zf".format(base_opts) elif path.endswith(".tar.bz2"): return "{}jf".format(base_opts) else: raise IScriptError("Unknown tarball suffix in path {}".format(path))
def task_cert_type(config, task): """Get the signing cert type from the task scopes. Args: config (dict): the running config task (dict): the running task Returns: str: the cert type, e.g. ``dep-signing`` """ cert_prefix = "{}cert:".format(config["taskcluster_scope_prefix"]) cert_scopes = [i for i in task["scopes"] if i.startswith(cert_prefix)] if len(cert_scopes) > 1: raise IScriptError("Too many cert scopes found! {}".format(cert_scopes)) if len(cert_scopes) < 1: raise IScriptError("Unable to find a cert scope! {}".format(task["scopes"])) return cert_scopes[0].replace(cert_prefix, "")
async def fake_retry_async(_, args, kwargs, **kw): cmd = args[0] end = len(cmd) - 1 assert cmd[0] == "xcrun" log_cmd = kwargs["log_cmd"] assert cmd[0:end] == log_cmd[0:end] assert cmd[end] != log_cmd[end] assert cmd[end] == pw assert log_cmd[end].replace("*", "") == "" if exception is IScriptError: raise IScriptError("foo")
def langpack_id(app): """Return a list of id's for the langpacks. Side effect of checking if filenames are actually langpacks. """ _, file_extension = os.path.splitext(app.orig_path) if not file_extension == ".xpi": raise IScriptError(f"Expected an xpi got {app.orig_path}") langpack = zipfile.ZipFile(app.orig_path, "r") id = None with langpack.open("manifest.json", "r") as f: manifest = json.load(f) if not ("languages" in manifest and "langpack_id" in manifest and "applications" in manifest and "gecko" in manifest["applications"] and "id" in manifest["applications"]["gecko"] and LANGPACK_RE.match(manifest["applications"]["gecko"]["id"])): raise IScriptError(f"{app.orig_path} is not a valid langpack") id = manifest["applications"]["gecko"]["id"] return id
def check_required_attrs(self, required_attrs): """Make sure the ``required_attrs`` are set. Args: required_attrs (list): list of attribute strings Raises: IScriptError: on missing attr """ for att in required_attrs: if not hasattr(self, att) or not getattr(self, att): raise IScriptError("Missing {} attr!".format(att))
async def sign_with_autograph(key_config, input_bytes, fmt, autograph_method, keyid=None, extension_id=None): """Signs data with autograph and returns the result. Args: key_config (dict): the running config for this key input_bytes (bytes): the source data to sign fmt (str): the format to sign with autograph_method (str): which autograph method to use to sign. must be one of 'file', 'hash', or 'data' keyid (str): which key to use on autograph (optional) extension_id (str): which id to send to autograph for the extension (optional) Raises: Requests.RequestException: on failure Returns: bytes: the signed data """ if autograph_method not in {"file", "hash", "data"}: raise IScriptError(f"Unsupported autograph method: {autograph_method}") sign_req = make_signing_req(input_bytes, fmt, keyid=keyid, extension_id=extension_id) short_fmt = fmt.replace("autograph_", "") url = key_config[f"{short_fmt}_url"] user = key_config[f"{short_fmt}_user"] pw = key_config[f"{short_fmt}_pass"] log.debug("signing data with format %s with %s", fmt, autograph_method) url = f"{url}/sign/{autograph_method}" sign_resp = await retry_async( call_autograph, args=(url, user, pw, sign_req), attempts=3, sleeptime_kwargs={"delay_factor": 2.0}, ) if autograph_method == "file": return sign_resp[0]["signed_file"] else: return sign_resp[0]["signature"]
def get_uuid_from_log(log_path): """Get the UUID from the notarization log. Args: log_path (str): the path to the log Raises: IScriptError: if we can't find the UUID Returns: str: the uuid """ regex = re.compile(r"RequestUUID = (?P<uuid>[a-zA-Z0-9-]+)") try: with open(log_path, "r") as fh: for line in fh.readlines(): m = regex.search(line) if m: return m["uuid"] except OSError as err: raise IScriptError("Can't find UUID in {}: {}".format(log_path, err)) from err raise IScriptError("Can't find UUID in {}!".format(log_path))
async def extract_all_apps(config, all_paths): """Extract all the apps into their own directories. Args: work_dir (str): the ``work_dir`` path all_paths (list): a list of ``App`` objects with their ``orig_path`` set Raises: IScriptError: on failure """ log.info("Extracting all apps") futures = [] work_dir = config["work_dir"] unpack_dmg = os.path.join(os.path.dirname(__file__), "data", "unpack-diskimage") for counter, app in enumerate(all_paths): app.check_required_attrs(["orig_path"]) app.parent_dir = os.path.join(work_dir, str(counter)) rm(app.parent_dir) makedirs(app.parent_dir) if app.orig_path.endswith((".tar.bz2", ".tar.gz", ".tgz")): futures.append( asyncio.ensure_future( run_command( ["tar", "xf", app.orig_path], cwd=app.parent_dir, exception=IScriptError, ))) elif app.orig_path.endswith(".dmg"): unpack_mountpoint = os.path.join( "/tmp", f"{config.get('dmg_prefix', 'dmg')}-{counter}-unpack") futures.append( asyncio.ensure_future( run_command( [ unpack_dmg, app.orig_path, unpack_mountpoint, app.parent_dir ], cwd=app.parent_dir, exception=IScriptError, log_level=logging.DEBUG, ))) else: raise IScriptError(f"unknown file type {app.orig_path}") await raise_future_exceptions(futures) if app.orig_path.endswith(".dmg"): # nuke the softlink to /Applications for counter, app in enumerate(all_paths): rm(os.path.join(app.parent_dir, " "))
async def fake_retry_async(*args, **kwargs): assert kwargs["args"][0][0] == "xcrun" if raises: raise IScriptError("foo")
async def fail_async(*args, **kwargs): raise IScriptError("fail_async exception")
def _get_artifact_prefix(path): for prefix in KNOWN_ARTIFACT_PREFIXES: if path.startswith(prefix): return prefix raise IScriptError(f"Unknown artifact prefix for {path}!")
def _get_pkg_name_from_tarball(path): for ext in (".tar.gz", ".tar.bz2", ".dmg"): if path.endswith(ext): return path.replace(ext, ".pkg") raise IScriptError("Unknown tarball suffix in path {}".format(path))
async def fake_run_command(*args, **kwargs): assert args[0][0] == command if raises: raise IScriptError("foo")
async def fake_run_command(*args, **kwargs): assert args[0][0:2] == ["zip", "-r"] if raises: raise IScriptError("foo")
async def fake_sign(arg1, arg2, arg3): assert arg1 == key_config assert arg2 in app_paths assert arg3 == entitlements_path if raises: raise IScriptError("foo")
async def fake_run_command(cmd, **kwargs): assert cmd[0:2] == ["sudo", "pkgbuild"] if raises: raise IScriptError("foo")
async def fake_raise_future_exceptions(futures): await asyncio.wait(futures) if raises: raise IScriptError("foo")
async def fake_raise_future_exceptions(futures): await asyncio.wait(futures) assert len(futures) == len(poll_uuids) if raises: raise IScriptError("foo")