async def sign_file_with_autograph(context, from_, fmt, to=None): """Signs file with autograph and writes the results to a file. Args: context (Context): the signing context from_ (str): the source file to sign fmt (str): the format to sign with to (str, optional): the target path to sign to. If None, overwrite `from_`. Defaults to None. Raises: Requests.RequestException: on failure SigningScriptError: when no suitable signing server is found for fmt Returns: str: the path to the signed file """ if not utils.is_autograph_signing_format(fmt): raise SigningScriptError(f"Not an autograph format: {fmt}") cert_type = task.task_cert_type(context) servers = get_suitable_signing_servers(context.signing_servers, cert_type, [fmt], raise_on_empty_list=True) s = servers[0] to = to or from_ input_bytes = open(from_, 'rb').read() signed_bytes = base64.b64decode(await sign_with_autograph(s, input_bytes, fmt, 'file')) with open(to, 'wb') as fout: fout.write(signed_bytes) return to
async def sign_gpg_with_autograph(context, from_, fmt): """Signs file with autograph and writes the results to a file. Args: context (Context): the signing context from_ (str): the source file to sign fmt (str): the format to sign with Raises: Requests.RequestException: on failure SigningScriptError: when no suitable signing server is found for fmt Returns: list: the path to the signed file, and sig. """ if not utils.is_autograph_signing_format(fmt): raise SigningScriptError(f"Not an autograph format: {fmt}") cert_type = task.task_cert_type(context) servers = get_suitable_signing_servers(context.signing_servers, cert_type, [fmt], raise_on_empty_list=True) s = servers[0] to = f"{from_}.asc" input_bytes = open(from_, 'rb').read() signature = await sign_with_autograph(s, input_bytes, fmt, 'data') with open(to, 'w') as fout: fout.write(signature) return [from_, to]
async def sign_hash_with_autograph(context, hash_, fmt, keyid=None): """Signs hash with autograph and returns the result. Args: context (Context): the signing context hash_ (bytes): the input hash to sign fmt (str): the format to sign with keyid (str): which key to use on autograph (optional) Raises: Requests.RequestException: on failure SigningScriptError: when no suitable signing server is found for fmt Returns: bytes: the signature """ if not utils.is_autograph_signing_format(fmt): raise SigningScriptError(f"Not an autograph format: {fmt}") cert_type = task.task_cert_type(context) servers = get_suitable_signing_servers(context.signing_servers, cert_type, [fmt], raise_on_empty_list=True) s = servers[0] signature = base64.b64decode(await sign_with_autograph(s, hash_, fmt, 'hash', keyid)) return signature
async def sign_file(context, from_, fmt, to=None): """Send the file to signtool or autograph to be signed. Args: context (Context): the signing context from_ (str): the source file to sign fmt (str): the format to sign with to (str, optional): the target path to sign to. If None, overwrite `from_`. Defaults to None. Raises: FailedSubprocess: on subprocess error while signing. Returns: str: the path to the signed file """ if utils.is_autograph_signing_format(fmt): log.info("sign_file(): signing %s with %s... using autograph /sign/file", from_, fmt) await sign_file_with_autograph(context, from_, fmt, to=to) else: log.info("sign_file(): signing %s with %s... using signing server", from_, fmt) cmd = build_signtool_cmd(context, from_, fmt, to=to) await utils.execute_subprocess(cmd) return to or from_
async def async_main(context): """Sign all the things. Args: context (Context): the signing context. """ connector = _craft_aiohttp_connector(context) async with aiohttp.ClientSession(connector=connector) as session: context.session = session work_dir = context.config['work_dir'] context.signing_servers = load_signing_server_config(context) all_signing_formats = task_signing_formats(context) if 'gpg' in all_signing_formats or 'autograph_gpg' in all_signing_formats: if not context.config.get('gpg_pubkey'): raise Exception( "GPG format is enabled but gpg_pubkey is not defined") if not os.path.exists(context.config['gpg_pubkey']): raise Exception("gpg_pubkey ({}) doesn't exist!".format( context.config['gpg_pubkey'])) if 'autograph_widevine' in all_signing_formats: if not context.config.get('widevine_cert'): raise Exception( "Widevine format is enabled, but widevine_cert is not defined" ) if not all( is_autograph_signing_format(format_) for format_ in all_signing_formats): log.info("getting signingserver token") await get_token(context, os.path.join(work_dir, 'token'), task_cert_type(context), all_signing_formats) filelist_dict = build_filelist_dict(context) for path, path_dict in filelist_dict.items(): copy_to_dir(path_dict['full_path'], context.config['work_dir'], target=path) log.info("signing %s", path) output_files = await sign(context, os.path.join(work_dir, path), path_dict['formats']) for source in output_files: source = os.path.relpath(source, work_dir) copy_to_dir(os.path.join(work_dir, source), context.config['artifact_dir'], target=source) if 'gpg' in path_dict['formats'] or 'autograph_gpg' in path_dict[ 'formats']: copy_to_dir(context.config['gpg_pubkey'], context.config['artifact_dir'], target="public/build/KEY") log.info("Done!")
async def sign_file_with_autograph(context, from_, fmt, to=None): """Signs a file with autograph and writes the result to arg `to` or `from_` if `to` is None. Args: context (Context): the signing context from_ (str): the source file to sign fmt (str): the format to sign with to (str, optional): the target path to sign to. If None, overwrite `from_`. Defaults to None. Raises: Requests.RequestException: on failure Returns: str: the path to the signed file """ if not utils.is_autograph_signing_format(fmt): raise SigningScriptError( "No signing servers found supporting signing format {}".format( fmt)) cert_type = task.task_cert_type(context) servers = get_suitable_signing_servers(context.signing_servers, cert_type, [fmt]) if not servers: raise SigningScriptError( "No signing servers found with cert type {}".format(cert_type)) s = servers[0] to = to or from_ with open(from_, 'rb') as fin: input_bytes = fin.read() # build and run the signature request sign_req = [{"input": base64.b64encode(input_bytes)}] log.debug("using the default autograph keyid for %s", s.user) url = "%s/sign/file" % s.server async def make_sign_req(): auth = HawkAuth(id=s.user, key=s.password) with requests.Session() as session: r = session.post(url, json=sign_req, auth=auth) log.debug("Autograph response: {}".format( r.text[:120] if len(r.text) >= 120 else r.text)) r.raise_for_status() return r.json() sign_resp = await retry_async(make_sign_req) with open(to, 'wb') as fout: fout.write(base64.b64decode(sign_resp[0]['signed_file'])) log.info("autograph wrote signed_file %s to %s", from_, to) return to
async def get_token(context, output_file, cert_type, signing_formats): """Retrieve a token from the signingserver tied to my ip. Args: context (Context): the signing context output_file (str): the path to write the token to. cert_type (str): the cert type used to find an appropriate signing server signing_formats (list): the signing formats used to find an appropriate signing server Raises: SigningServerError: on failure """ token = None data = { "slave_ip": context.config["my_ip"], "duration": context.config["token_duration_seconds"], } signing_servers = get_suitable_signing_servers( context.signing_servers, cert_type, [ fmt for fmt in signing_formats if not is_autograph_signing_format(fmt) ], ) random.shuffle(signing_servers) for s in signing_servers: log.info("getting token from %s", s.server) url = "https://{}/token".format(s.server) auth = aiohttp.BasicAuth(s.user, password=s.password) try: token = await retry_request(context, url, method="post", data=data, auth=auth, return_type="text") if token: break except ( ScriptWorkerException, aiohttp.ClientError, asyncio.TimeoutError, ) as exc: log.warning( "Error retrieving token: {}\nTrying the next server.".format( str(exc))) continue else: raise SigningServerError( "Cannot retrieve signing token from any signing server.") with open(output_file, "w") as fh: print(token, file=fh, end="")
async def sign_widevine_zip(context, orig_path, fmt): """Sign the internals of a zipfile with the widevine key. Extract the files to sign (see `_WIDEVINE_BLESSED_FILENAMES` and `_WIDEVINE_UNBLESSED_FILENAMES), skipping already-signed files. The blessed files should be signed with the `widevine_blessed` format. Then append the sigfiles to the zipfile. Args: context (Context): the signing context orig_path (str): the source file to sign fmt (str): the format to sign with Returns: str: the path to the signed archive """ # This will get cleaned up when we nuke `work_dir`. Clean up at that point # rather than immediately after `sign_widevine`, to optimize task runtime # speed over disk space. tmp_dir = tempfile.mkdtemp(prefix="wvzip", dir=context.config['work_dir']) # Get file list all_files = await _get_zipfile_files(orig_path) files_to_sign = _get_widevine_signing_files(all_files) is_autograph = utils.is_autograph_signing_format(fmt) log.debug("Widevine files to sign: %s", files_to_sign) if files_to_sign: # Extract all files so we can create `precomplete` with the full # file list all_files = await _extract_zipfile(context, orig_path, tmp_dir=tmp_dir) tasks = [] # Sign the appropriate inner files for from_, fmt in files_to_sign.items(): from_ = os.path.join(tmp_dir, from_) to = f"{from_}.sig" if is_autograph: tasks.append(asyncio.ensure_future(sign_widevine_with_autograph( context, from_, "blessed" in fmt, to=to ))) else: tasks.append(asyncio.ensure_future(sign_file( context, from_, fmt, to=to ))) all_files.append(to) await raise_future_exceptions(tasks) remove_extra_files(tmp_dir, all_files) # Regenerate the `precomplete` file, which is used for cleanup before # applying a complete mar. _run_generate_precomplete(context, tmp_dir) await _create_zipfile( context, orig_path, all_files, mode='w', tmp_dir=tmp_dir ) return orig_path
async def sign_widevine_tar(context, orig_path, fmt): """Sign the internals of a tarfile with the widevine key. Extract the entire tarball, but only sign a handful of files (see `_WIDEVINE_BLESSED_FILENAMES` and `_WIDEVINE_UNBLESSED_FILENAMES). The blessed files should be signed with the `widevine_blessed` format. Then recreate the tarball. Ideally we would be able to append the sigfiles to the original tarball, but that's not possible with compressed tarballs. Args: context (Context): the signing context orig_path (str): the source file to sign fmt (str): the format to sign with Returns: str: the path to the signed archive """ _, compression = os.path.splitext(orig_path) # This will get cleaned up when we nuke `work_dir`. Clean up at that point # rather than immediately after `sign_widevine`, to optimize task runtime # speed over disk space. tmp_dir = tempfile.mkdtemp(prefix="wvtar", dir=context.config['work_dir']) # Get file list all_files = await _get_tarfile_files(orig_path, compression) files_to_sign = _get_widevine_signing_files(all_files) is_autograph = utils.is_autograph_signing_format(fmt) log.debug("Widevine files to sign: %s", files_to_sign) if files_to_sign: # Extract all files so we can create `precomplete` with the full # file list all_files = await _extract_tarfile( context, orig_path, compression, tmp_dir=tmp_dir ) tasks = [] # Sign the appropriate inner files for from_, fmt in files_to_sign.items(): from_ = os.path.join(tmp_dir, from_) # Don't try to sign directories if not os.path.isfile(from_): continue # Move the sig location on mac. This should be noop on linux. to = _get_mac_sigpath(from_) log.debug("Adding %s to the sigfile paths...", to) makedirs(os.path.dirname(to)) if is_autograph: tasks.append(asyncio.ensure_future(sign_widevine_with_autograph( context, from_, "blessed" in fmt, to=to ))) else: tasks.append(asyncio.ensure_future(sign_file( context, from_, fmt, to=to ))) all_files.append(to) await raise_future_exceptions(tasks) remove_extra_files(tmp_dir, all_files) # Regenerate the `precomplete` file, which is used for cleanup before # applying a complete mar. _run_generate_precomplete(context, tmp_dir) await _create_tarfile( context, orig_path, all_files, compression, tmp_dir=tmp_dir ) return orig_path
async def sign_mar384_with_autograph_hash(context, from_, fmt, to=None): """Signs a hash with autograph, injects it into the file, and writes the result to arg `to` or `from_` if `to` is None. Args: context (Context): the signing context from_ (str): the source file to sign fmt (str): the format to sign with to (str, optional): the target path to sign to. If None, overwrite `from_`. Defaults to None. Raises: Requests.RequestException: on failure SigningScriptError: when no suitable signing server is found for fmt Returns: str: the path to the signed file """ log.info( "sign_mar384_with_autograph_hash(): signing {} with {}... using autograph /sign/hash" .format(from_, fmt)) if not utils.is_autograph_signing_format(fmt): raise SigningScriptError( "No signing servers found supporting signing format {}".format( fmt)) cert_type = task.task_cert_type(context) servers = get_suitable_signing_servers(context.signing_servers, cert_type, [fmt], raise_on_empty_list=True) s = servers[0] to = to or from_ hash_algo, expected_signature_length = 'sha384', 512 # Add a dummy signature into a temporary file (TODO: dedup with mardor.cli do_hash) tmp = tempfile.TemporaryFile() with open(from_, 'rb') as f: add_signature_block(f, tmp, hash_algo) tmp.seek(0) with MarReader(tmp) as m: hashes = m.calculate_hashes() h = hashes[0][1] tmp.close() # build and run the hash signature request sign_req = [{"input": base64.b64encode(h).decode('ascii')}] log.debug( "signing mar with hash alg %s using the default autograph keyid for %s", hash_algo, s.user) url = "%s/sign/hash" % s.server async def make_sign_req(): auth = HawkAuth(id=s.user, key=s.password) with requests.Session() as session: r = session.post(url, json=sign_req, auth=auth) log.debug("Autograph response: {}".format( r.text[:120] if len(r.text) >= 120 else r.text)) r.raise_for_status() return r.json() sign_resp = await retry_async(make_sign_req) signature = base64.b64decode(sign_resp[0]['signature']) # Add a signature to the MAR file (TODO: dedup with mardor.cli do_add_signature) if len(signature) != expected_signature_length: raise SigningScriptError( "signed mar hash signature has invalid length for hash algo {}. Got {} expected {}." .format(hash_algo, len(signature), expected_signature_length)) # use the tmp file in case param `to` is `from_` which causes stream errors tmp_dst = tempfile.NamedTemporaryFile(mode='w+b', delete=False) with open(tmp_dst.name, 'w+b') as dst: with open(from_, 'rb') as src: add_signature_block(src, dst, hash_algo, signature) shutil.copyfile(tmp_dst.name, to) os.unlink(tmp_dst.name) verify_mar_signature(cert_type, fmt, to) log.info("wrote mar with autograph signed hash %s to %s", from_, to) return to
async def sign_file_with_autograph(context, from_, fmt, to=None): """Signs a file with autograph and writes the result to arg `to` or `from_` if `to` is None. Args: context (Context): the signing context from_ (str): the source file to sign fmt (str): the format to sign with to (str, optional): the target path to sign to. If None, overwrite `from_`. Defaults to None. Raises: Requests.RequestException: on failure SigningScriptError: when no suitable signing server is found for fmt Returns: str: the path to the signed file """ if not utils.is_autograph_signing_format(fmt): raise SigningScriptError( "No signing servers found supporting signing format {}".format( fmt)) cert_type = task.task_cert_type(context) servers = get_suitable_signing_servers(context.signing_servers, cert_type, [fmt], raise_on_empty_list=True) s = servers[0] to = to or from_ with open(from_, 'rb') as fin: input_bytes = fin.read() # We need to base64 data for autograph. b64encode() returns bytes, though. We need utf8 strings # to make Python's JSON decoder happy base64_input = base64.b64encode(input_bytes).decode('utf-8') # build and run the signature request sign_req = [{"input": base64_input}] if utils.is_apk_autograph_signing_format(fmt): # We don't want APKs to have their compression changed sign_req[0]['options'] = {'zip': 'passthrough'} if utils.is_sha1_apk_autograph_signing_format(fmt): # We ask for a SHA1 digest from Autograph # https://github.com/mozilla-services/autograph/pull/166/files sign_req[0]['options']['pkcs7_digest'] = "SHA1" log.debug("using the default autograph keyid for %s", s.user) url = "%s/sign/file" % s.server async def make_sign_req(): auth = HawkAuth(id=s.user, key=s.password) with requests.Session() as session: r = session.post(url, json=sign_req, auth=auth) log.debug("Autograph response: {}".format( r.text[:120] if len(r.text) >= 120 else r.text)) r.raise_for_status() return r.json() sign_resp = await retry_async(make_sign_req) with open(to, 'wb') as fout: fout.write(base64.b64decode(sign_resp[0]['signed_file'])) log.info("autograph wrote signed_file %s to %s", from_, to) return to
async def async_main(context): """Sign all the things. Args: context (Context): the signing context. """ connector = _craft_aiohttp_connector(context) # Create a session for talking to the legacy signing servers in order to # generate a signing token async with aiohttp.ClientSession(connector=connector) as session: context.session = session work_dir = context.config["work_dir"] context.signing_servers = load_signing_server_config(context) all_signing_formats = task_signing_formats(context) if "gpg" in all_signing_formats or "autograph_gpg" in all_signing_formats: if not context.config.get("gpg_pubkey"): raise Exception("GPG format is enabled but gpg_pubkey is not defined") if not os.path.exists(context.config["gpg_pubkey"]): raise Exception( "gpg_pubkey ({}) doesn't exist!".format( context.config["gpg_pubkey"] ) ) if "autograph_widevine" in all_signing_formats: if not context.config.get("widevine_cert"): raise Exception( "Widevine format is enabled, but widevine_cert is not defined" ) if not all( is_autograph_signing_format(format_) for format_ in all_signing_formats ): log.info("getting signingserver token") await get_token( context, os.path.join(work_dir, "token"), task_cert_type(context), all_signing_formats, ) # Create a new session to talk to autograph async with aiohttp.ClientSession() as session: context.session = session filelist_dict = build_filelist_dict(context) for path, path_dict in filelist_dict.items(): copy_to_dir(path_dict["full_path"], context.config["work_dir"], target=path) log.info("signing %s", path) output_files = await sign( context, os.path.join(work_dir, path), path_dict["formats"] ) for source in output_files: source = os.path.relpath(source, work_dir) copy_to_dir( os.path.join(work_dir, source), context.config["artifact_dir"], target=source, ) if "gpg" in path_dict["formats"] or "autograph_gpg" in path_dict["formats"]: copy_to_dir( context.config["gpg_pubkey"], context.config["artifact_dir"], target="public/build/KEY", ) log.info("Done!")