def get_uuid_from_log(log_path):
"""Get the UUID from the notarization log.
Args:
log_path (str): the path to the log
Raises:
IScriptError: if we can't find the UUID
ThrottledNotarization: if there's an ``ERROR ITMS-10004`` in the response
UnknownNotarizationError: if there's an unknown ``ERROR`` in the response
Returns:
str: the uuid
"""
regex = re.compile(r"RequestUUID = (?P<uuid>[a-zA-Z0-9-]+)")
try:
with open(log_path, "r") as fh:
contents = fh.read()
log.info(f"{log_path} notarization response:\n{contents}")
exception = None
if "ERROR ITMS-10004" in contents:
exception = ThrottledNotarization
elif "ERROR " in contents:
exception = UnknownNotarizationError
if exception is not None:
raise exception(f"Error response from Apple!\n{contents}")
for line in contents.splitlines():
m = regex.search(line)
if m:
return m["uuid"]
except OSError as err:
raise IScriptError("Can't find UUID in {}: {}".format(log_path,
err)) from err
raise IScriptError("Can't find UUID in {}!".format(log_path))
async def async_main(config, task):
"""Sign all the things.
Args:
config (dict): the running config.
task (dict): the running task.
"""
await run_command(["hostname"])
base_key = "mac_config" # We may support ios_config someday
key_config = get_key_config(config, task, base_key=base_key)
behavior = task["payload"].get("behavior", "mac_sign")
if behavior == "mac_notarize" and "mac_notarize" not in key_config[
"supported_behaviors"] and "mac_sign_and_pkg" in key_config[
"supported_behaviors"]:
behavior = "mac_sign_and_pkg"
if behavior not in key_config["supported_behaviors"]:
raise IScriptError("Unsupported behavior {} given scopes {}!".format(
behavior, task["scopes"]))
if behavior == "mac_geckodriver":
await geckodriver_behavior(config, task)
return
elif behavior == "mac_notarize":
await notarize_behavior(config, task)
return
elif behavior == "mac_sign":
await sign_behavior(config, task)
return
elif behavior == "mac_sign_and_pkg":
# For staging releases; or should we mac_notarize but skip notarization
# for dep?
await sign_and_pkg_behavior(config, task)
return
raise IScriptError("Unknown iscript behavior {}!".format(behavior))
def _ensure_one_precomplete(tmp_dir, adj):
"""Ensure we only have one `precomplete` file in `tmp_dir`."""
precompletes = glob.glob(os.path.join(tmp_dir, "**", "precomplete"),
recursive=True)
if len(precompletes) < 1:
raise IScriptError('No `precomplete` file found in "%s"', tmp_dir)
if len(precompletes) > 1:
raise IScriptError('More than one `precomplete` file %s in "%s"', adj,
tmp_dir)
return precompletes[0]
async def create_one_notarization_zipfile(work_dir, all_paths, sign_config, path_attrs=("app_path", "pkg_path")):
"""Create a single notarization zipfile for all the apps.
Args:
work_dir (str): the script work directory
all_paths (list): list of ``App`` objects
path_attrs (tuple, optional): the attributes for the paths we'll be zipping
up. Defaults to ``("app_path", "pkg_path")``
Raises:
IScriptError: on failure
Returns:
str: the zip path
"""
required_attrs = path_attrs
app_paths = []
zip_path = os.path.join(work_dir, "notarization.zip")
for app in all_paths:
app.check_required_attrs(required_attrs)
for path_attr in path_attrs:
app_paths.append(os.path.relpath(getattr(app, path_attr), work_dir))
if sign_config["zipfile_cmd"] == "zip":
await run_command(["zip", "-r", zip_path, *app_paths], cwd=work_dir, exception=IScriptError)
elif sign_config["zipfile_cmd"] == "ditto":
await run_command(["ditto", "-c", "-k", "--sequesterRsrc", "--keepParent", "0", zip_path], cwd=work_dir, exception=IScriptError)
else:
raise IScriptError(f"Unknown zipfile_cmd {sign_config['zipfile_cmd']}!")
return zip_path
async def sign_langpacks(config, key_config, all_paths):
"""Signs langpacks that are specified in all_paths.
Raises:
IScriptError if we don't have any valid language packs to sign in any path.
"""
for app in all_paths:
app.check_required_attrs(["orig_path", "formats", "artifact_prefix"])
if not {"autograph_langpack"} & set(app.formats):
raise IScriptError(
f"{app.formats} does not contain 'autograph_langpack'")
app.target_tar_path = "{}/{}{}".format(
config["artifact_dir"],
app.artifact_prefix,
app.orig_path.split(app.artifact_prefix)[1],
)
id = langpack_id(app)
log.info("Identified {} as extension id: {}".format(app.orig_path, id))
makedirs(os.path.dirname(app.target_tar_path))
await sign_file_with_autograph(
key_config,
app.orig_path,
"autograph_langpack",
to=app.target_tar_path,
extension_id=id,
)
async def fake_raise_future_exceptions(futures, **kwargs):
"""``raise_future_exceptions`` mocker."""
await asyncio.wait(futures)
assert len(futures) == futures_len.pop(0)
if raises:
raise IScriptError("foo")
def get_key_config(config, task, base_key="mac_config"):
"""Sanity check the task scopes and return the appropriate ``key_config``.
The ``key_config`` is, e.g. the ``config.mac_config.dep`` dictionary,
for mac dep-signing.
Args:
config (dict): the running config
task (dict): the running task
base_key (str, optional): the base key in the dictionary. Defaults to
``mac_config``.
Raises:
IScriptError: on failure to verify the scopes.
Returns:
dict: the ``key_config``
"""
try:
cert_type = task_cert_type(config, task)
return config[base_key][_CERT_TYPE_TO_KEY_CONFIG[cert_type]]
except KeyError as exc:
raise IScriptError("get_key_config error: {}".format(
str(exc))) from exc
async def sign_geckodriver(config, sign_config, all_paths):
"""Sign geckodriver.
Args:
sign_config (dict): the running config
all_paths (list): list of App objects
Raises:
IScriptError: on error.
"""
identity = sign_config["identity"]
keychain = sign_config["signing_keychain"]
sign_command = _get_sign_command(identity, keychain, sign_config)
for app in all_paths:
app.check_required_attrs(["orig_path", "parent_dir", "artifact_prefix"])
app.target_tar_path = "{}/{}{}".format(config["artifact_dir"], app.artifact_prefix, app.orig_path.split(app.artifact_prefix)[1])
file_ = "geckodriver"
path = os.path.join(app.parent_dir, file_)
if not os.path.exists(path):
raise IScriptError(f"No such file {path}!")
await retry_async(
run_command,
args=[sign_command + [file_]],
kwargs={"cwd": app.parent_dir, "exception": IScriptError, "output_log_on_exception": True},
retry_exceptions=(IScriptError,),
)
env = deepcopy(os.environ)
# https://superuser.com/questions/61185/why-do-i-get-files-like-foo-in-my-tarball-on-os-x
env["COPYFILE_DISABLE"] = "1"
makedirs(os.path.dirname(app.target_tar_path))
await run_command(
["tar", _get_tar_create_options(app.target_tar_path), app.target_tar_path, file_], cwd=app.parent_dir, env=env, exception=IScriptError
)
async def unlock_keychain(signing_keychain, keychain_password):
"""Unlock the signing keychain.
Args:
signing_keychain (str): the path to the signing keychain
keychain_password (str): the keychain password
Raises:
IScriptError: on failure
TimeoutFailure: on timeout
"""
log.info("Unlocking signing keychain {}".format(signing_keychain))
child = pexpect.spawn("security", ["unlock-keychain", signing_keychain],
encoding="utf-8")
try:
while True:
index = await child.expect(
[
pexpect.EOF,
r"password to unlock {}: ".format(signing_keychain)
],
async_=True,
)
if index == 0:
break
child.sendline(keychain_password)
except (pexpect.exceptions.TIMEOUT) as exc:
raise TimeoutError(
"Timeout trying to unlock the keychain {}: {}!".format(
signing_keychain, exc)) from exc
child.close()
if child.exitstatus != 0 or child.signalstatus is not None:
raise IScriptError("Failed unlocking {}! exit {} signal {}".format(
signing_keychain, child.exitstatus, child.signalstatus))
async def fake_run_command(*args, **kwargs):
assert args[0] == [
"zip", "-r",
os.path.join(work_dir, "app_path.zip"), "0/0.app", "1/1.app",
"2/2.app"
]
if raises:
raise IScriptError("foo")
def _get_tar_create_options(path):
base_opts = "c"
if path.endswith(".tar.gz"):
return "{}zf".format(base_opts)
elif path.endswith(".tar.bz2"):
return "{}jf".format(base_opts)
else:
raise IScriptError("Unknown tarball suffix in path {}".format(path))
def task_cert_type(config, task):
"""Get the signing cert type from the task scopes.
Args:
config (dict): the running config
task (dict): the running task
Returns:
str: the cert type, e.g. ``dep-signing``
"""
cert_prefix = "{}cert:".format(config["taskcluster_scope_prefix"])
cert_scopes = [i for i in task["scopes"] if i.startswith(cert_prefix)]
if len(cert_scopes) > 1:
raise IScriptError("Too many cert scopes found! {}".format(cert_scopes))
if len(cert_scopes) < 1:
raise IScriptError("Unable to find a cert scope! {}".format(task["scopes"]))
return cert_scopes[0].replace(cert_prefix, "")
async def fake_retry_async(_, args, kwargs, **kw):
cmd = args[0]
end = len(cmd) - 1
assert cmd[0] == "xcrun"
log_cmd = kwargs["log_cmd"]
assert cmd[0:end] == log_cmd[0:end]
assert cmd[end] != log_cmd[end]
assert cmd[end] == pw
assert log_cmd[end].replace("*", "") == ""
if exception is IScriptError:
raise IScriptError("foo")
def langpack_id(app):
"""Return a list of id's for the langpacks.
Side effect of checking if filenames are actually langpacks.
"""
_, file_extension = os.path.splitext(app.orig_path)
if not file_extension == ".xpi":
raise IScriptError(f"Expected an xpi got {app.orig_path}")
langpack = zipfile.ZipFile(app.orig_path, "r")
id = None
with langpack.open("manifest.json", "r") as f:
manifest = json.load(f)
if not ("languages" in manifest and "langpack_id" in manifest
and "applications" in manifest
and "gecko" in manifest["applications"]
and "id" in manifest["applications"]["gecko"] and
LANGPACK_RE.match(manifest["applications"]["gecko"]["id"])):
raise IScriptError(f"{app.orig_path} is not a valid langpack")
id = manifest["applications"]["gecko"]["id"]
return id
def check_required_attrs(self, required_attrs):
"""Make sure the ``required_attrs`` are set.
Args:
required_attrs (list): list of attribute strings
Raises:
IScriptError: on missing attr
"""
for att in required_attrs:
if not hasattr(self, att) or not getattr(self, att):
raise IScriptError("Missing {} attr!".format(att))
async def sign_with_autograph(key_config,
input_bytes,
fmt,
autograph_method,
keyid=None,
extension_id=None):
"""Signs data with autograph and returns the result.
Args:
key_config (dict): the running config for this key
input_bytes (bytes): the source data to sign
fmt (str): the format to sign with
autograph_method (str): which autograph method to use to sign. must be
one of 'file', 'hash', or 'data'
keyid (str): which key to use on autograph (optional)
extension_id (str): which id to send to autograph for the extension (optional)
Raises:
Requests.RequestException: on failure
Returns:
bytes: the signed data
"""
if autograph_method not in {"file", "hash", "data"}:
raise IScriptError(f"Unsupported autograph method: {autograph_method}")
sign_req = make_signing_req(input_bytes,
fmt,
keyid=keyid,
extension_id=extension_id)
short_fmt = fmt.replace("autograph_", "")
url = key_config[f"{short_fmt}_url"]
user = key_config[f"{short_fmt}_user"]
pw = key_config[f"{short_fmt}_pass"]
log.debug("signing data with format %s with %s", fmt, autograph_method)
url = f"{url}/sign/{autograph_method}"
sign_resp = await retry_async(
call_autograph,
args=(url, user, pw, sign_req),
attempts=3,
sleeptime_kwargs={"delay_factor": 2.0},
)
if autograph_method == "file":
return sign_resp[0]["signed_file"]
else:
return sign_resp[0]["signature"]
def get_uuid_from_log(log_path):
"""Get the UUID from the notarization log.
Args:
log_path (str): the path to the log
Raises:
IScriptError: if we can't find the UUID
Returns:
str: the uuid
"""
regex = re.compile(r"RequestUUID = (?P<uuid>[a-zA-Z0-9-]+)")
try:
with open(log_path, "r") as fh:
for line in fh.readlines():
m = regex.search(line)
if m:
return m["uuid"]
except OSError as err:
raise IScriptError("Can't find UUID in {}: {}".format(log_path, err)) from err
raise IScriptError("Can't find UUID in {}!".format(log_path))
async def extract_all_apps(config, all_paths):
"""Extract all the apps into their own directories.
Args:
work_dir (str): the ``work_dir`` path
all_paths (list): a list of ``App`` objects with their ``orig_path`` set
Raises:
IScriptError: on failure
"""
log.info("Extracting all apps")
futures = []
work_dir = config["work_dir"]
unpack_dmg = os.path.join(os.path.dirname(__file__), "data",
"unpack-diskimage")
for counter, app in enumerate(all_paths):
app.check_required_attrs(["orig_path"])
app.parent_dir = os.path.join(work_dir, str(counter))
rm(app.parent_dir)
makedirs(app.parent_dir)
if app.orig_path.endswith((".tar.bz2", ".tar.gz", ".tgz")):
futures.append(
asyncio.ensure_future(
run_command(
["tar", "xf", app.orig_path],
cwd=app.parent_dir,
exception=IScriptError,
)))
elif app.orig_path.endswith(".dmg"):
unpack_mountpoint = os.path.join(
"/tmp", f"{config.get('dmg_prefix', 'dmg')}-{counter}-unpack")
futures.append(
asyncio.ensure_future(
run_command(
[
unpack_dmg, app.orig_path, unpack_mountpoint,
app.parent_dir
],
cwd=app.parent_dir,
exception=IScriptError,
log_level=logging.DEBUG,
)))
else:
raise IScriptError(f"unknown file type {app.orig_path}")
await raise_future_exceptions(futures)
if app.orig_path.endswith(".dmg"):
# nuke the softlink to /Applications
for counter, app in enumerate(all_paths):
rm(os.path.join(app.parent_dir, " "))
async def fake_retry_async(*args, **kwargs):
assert kwargs["args"][0][0] == "xcrun"
if raises:
raise IScriptError("foo")
async def fail_async(*args, **kwargs):
raise IScriptError("fail_async exception")
def _get_artifact_prefix(path):
for prefix in KNOWN_ARTIFACT_PREFIXES:
if path.startswith(prefix):
return prefix
raise IScriptError(f"Unknown artifact prefix for {path}!")
def _get_pkg_name_from_tarball(path):
for ext in (".tar.gz", ".tar.bz2", ".dmg"):
if path.endswith(ext):
return path.replace(ext, ".pkg")
raise IScriptError("Unknown tarball suffix in path {}".format(path))
async def fake_run_command(*args, **kwargs):
assert args[0][0] == command
if raises:
raise IScriptError("foo")
async def fake_run_command(*args, **kwargs):
assert args[0][0:2] == ["zip", "-r"]
if raises:
raise IScriptError("foo")
async def fake_sign(arg1, arg2, arg3):
assert arg1 == key_config
assert arg2 in app_paths
assert arg3 == entitlements_path
if raises:
raise IScriptError("foo")
async def fake_run_command(cmd, **kwargs):
assert cmd[0:2] == ["sudo", "pkgbuild"]
if raises:
raise IScriptError("foo")
async def fake_raise_future_exceptions(futures):
await asyncio.wait(futures)
if raises:
raise IScriptError("foo")
async def fake_raise_future_exceptions(futures):
await asyncio.wait(futures)
assert len(futures) == len(poll_uuids)
if raises:
raise IScriptError("foo")
