async def sign_file_with_autograph(context, from_, fmt, to=None):
"""Signs file with autograph and writes the results to a file.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
to (str, optional): the target path to sign to. If None, overwrite
`from_`. Defaults to None.
Raises:
Requests.RequestException: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
str: the path to the signed file
"""
if not utils.is_autograph_signing_format(fmt):
raise SigningScriptError(f"Not an autograph format: {fmt}")
cert_type = task.task_cert_type(context)
servers = get_suitable_signing_servers(context.signing_servers, cert_type,
[fmt], raise_on_empty_list=True)
s = servers[0]
to = to or from_
input_bytes = open(from_, 'rb').read()
signed_bytes = base64.b64decode(await sign_with_autograph(s, input_bytes, fmt, 'file'))
with open(to, 'wb') as fout:
fout.write(signed_bytes)
return to
async def sign_gpg_with_autograph(context, from_, fmt):
"""Signs file with autograph and writes the results to a file.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
Raises:
Requests.RequestException: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
list: the path to the signed file, and sig.
"""
if not utils.is_autograph_signing_format(fmt):
raise SigningScriptError(f"Not an autograph format: {fmt}")
cert_type = task.task_cert_type(context)
servers = get_suitable_signing_servers(context.signing_servers, cert_type,
[fmt], raise_on_empty_list=True)
s = servers[0]
to = f"{from_}.asc"
input_bytes = open(from_, 'rb').read()
signature = await sign_with_autograph(s, input_bytes, fmt, 'data')
with open(to, 'w') as fout:
fout.write(signature)
return [from_, to]
async def sign_hash_with_autograph(context, hash_, fmt, keyid=None):
"""Signs hash with autograph and returns the result.
Args:
context (Context): the signing context
hash_ (bytes): the input hash to sign
fmt (str): the format to sign with
keyid (str): which key to use on autograph (optional)
Raises:
Requests.RequestException: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
bytes: the signature
"""
if not utils.is_autograph_signing_format(fmt):
raise SigningScriptError(f"Not an autograph format: {fmt}")
cert_type = task.task_cert_type(context)
servers = get_suitable_signing_servers(context.signing_servers, cert_type,
[fmt], raise_on_empty_list=True)
s = servers[0]
signature = base64.b64decode(await sign_with_autograph(s, hash_, fmt, 'hash', keyid))
return signature
async def sign_file(context, from_, fmt, to=None):
"""Send the file to signtool or autograph to be signed.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
to (str, optional): the target path to sign to. If None, overwrite
`from_`. Defaults to None.
Raises:
FailedSubprocess: on subprocess error while signing.
Returns:
str: the path to the signed file
"""
if utils.is_autograph_signing_format(fmt):
log.info("sign_file(): signing %s with %s... using autograph /sign/file", from_, fmt)
await sign_file_with_autograph(context, from_, fmt, to=to)
else:
log.info("sign_file(): signing %s with %s... using signing server", from_, fmt)
cmd = build_signtool_cmd(context, from_, fmt, to=to)
await utils.execute_subprocess(cmd)
return to or from_
async def async_main(context):
"""Sign all the things.
Args:
context (Context): the signing context.
"""
connector = _craft_aiohttp_connector(context)
async with aiohttp.ClientSession(connector=connector) as session:
context.session = session
work_dir = context.config['work_dir']
context.signing_servers = load_signing_server_config(context)
all_signing_formats = task_signing_formats(context)
if 'gpg' in all_signing_formats or 'autograph_gpg' in all_signing_formats:
if not context.config.get('gpg_pubkey'):
raise Exception(
"GPG format is enabled but gpg_pubkey is not defined")
if not os.path.exists(context.config['gpg_pubkey']):
raise Exception("gpg_pubkey ({}) doesn't exist!".format(
context.config['gpg_pubkey']))
if 'autograph_widevine' in all_signing_formats:
if not context.config.get('widevine_cert'):
raise Exception(
"Widevine format is enabled, but widevine_cert is not defined"
)
if not all(
is_autograph_signing_format(format_)
for format_ in all_signing_formats):
log.info("getting signingserver token")
await get_token(context, os.path.join(work_dir, 'token'),
task_cert_type(context), all_signing_formats)
filelist_dict = build_filelist_dict(context)
for path, path_dict in filelist_dict.items():
copy_to_dir(path_dict['full_path'],
context.config['work_dir'],
target=path)
log.info("signing %s", path)
output_files = await sign(context, os.path.join(work_dir, path),
path_dict['formats'])
for source in output_files:
source = os.path.relpath(source, work_dir)
copy_to_dir(os.path.join(work_dir, source),
context.config['artifact_dir'],
target=source)
if 'gpg' in path_dict['formats'] or 'autograph_gpg' in path_dict[
'formats']:
copy_to_dir(context.config['gpg_pubkey'],
context.config['artifact_dir'],
target="public/build/KEY")
log.info("Done!")
async def sign_file_with_autograph(context, from_, fmt, to=None):
"""Signs a file with autograph and writes the result to arg `to` or `from_` if `to` is None.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
to (str, optional): the target path to sign to. If None, overwrite
`from_`. Defaults to None.
Raises:
Requests.RequestException: on failure
Returns:
str: the path to the signed file
"""
if not utils.is_autograph_signing_format(fmt):
raise SigningScriptError(
"No signing servers found supporting signing format {}".format(
fmt))
cert_type = task.task_cert_type(context)
servers = get_suitable_signing_servers(context.signing_servers, cert_type,
[fmt])
if not servers:
raise SigningScriptError(
"No signing servers found with cert type {}".format(cert_type))
s = servers[0]
to = to or from_
with open(from_, 'rb') as fin:
input_bytes = fin.read()
# build and run the signature request
sign_req = [{"input": base64.b64encode(input_bytes)}]
log.debug("using the default autograph keyid for %s", s.user)
url = "%s/sign/file" % s.server
async def make_sign_req():
auth = HawkAuth(id=s.user, key=s.password)
with requests.Session() as session:
r = session.post(url, json=sign_req, auth=auth)
log.debug("Autograph response: {}".format(
r.text[:120] if len(r.text) >= 120 else r.text))
r.raise_for_status()
return r.json()
sign_resp = await retry_async(make_sign_req)
with open(to, 'wb') as fout:
fout.write(base64.b64decode(sign_resp[0]['signed_file']))
log.info("autograph wrote signed_file %s to %s", from_, to)
return to
async def get_token(context, output_file, cert_type, signing_formats):
"""Retrieve a token from the signingserver tied to my ip.
Args:
context (Context): the signing context
output_file (str): the path to write the token to.
cert_type (str): the cert type used to find an appropriate signing server
signing_formats (list): the signing formats used to find an appropriate
signing server
Raises:
SigningServerError: on failure
"""
token = None
data = {
"slave_ip": context.config["my_ip"],
"duration": context.config["token_duration_seconds"],
}
signing_servers = get_suitable_signing_servers(
context.signing_servers,
cert_type,
[
fmt
for fmt in signing_formats if not is_autograph_signing_format(fmt)
],
)
random.shuffle(signing_servers)
for s in signing_servers:
log.info("getting token from %s", s.server)
url = "https://{}/token".format(s.server)
auth = aiohttp.BasicAuth(s.user, password=s.password)
try:
token = await retry_request(context,
url,
method="post",
data=data,
auth=auth,
return_type="text")
if token:
break
except (
ScriptWorkerException,
aiohttp.ClientError,
asyncio.TimeoutError,
) as exc:
log.warning(
"Error retrieving token: {}\nTrying the next server.".format(
str(exc)))
continue
else:
raise SigningServerError(
"Cannot retrieve signing token from any signing server.")
with open(output_file, "w") as fh:
print(token, file=fh, end="")
async def sign_widevine_zip(context, orig_path, fmt):
"""Sign the internals of a zipfile with the widevine key.
Extract the files to sign (see `_WIDEVINE_BLESSED_FILENAMES` and
`_WIDEVINE_UNBLESSED_FILENAMES), skipping already-signed files.
The blessed files should be signed with the `widevine_blessed` format.
Then append the sigfiles to the zipfile.
Args:
context (Context): the signing context
orig_path (str): the source file to sign
fmt (str): the format to sign with
Returns:
str: the path to the signed archive
"""
# This will get cleaned up when we nuke `work_dir`. Clean up at that point
# rather than immediately after `sign_widevine`, to optimize task runtime
# speed over disk space.
tmp_dir = tempfile.mkdtemp(prefix="wvzip", dir=context.config['work_dir'])
# Get file list
all_files = await _get_zipfile_files(orig_path)
files_to_sign = _get_widevine_signing_files(all_files)
is_autograph = utils.is_autograph_signing_format(fmt)
log.debug("Widevine files to sign: %s", files_to_sign)
if files_to_sign:
# Extract all files so we can create `precomplete` with the full
# file list
all_files = await _extract_zipfile(context, orig_path, tmp_dir=tmp_dir)
tasks = []
# Sign the appropriate inner files
for from_, fmt in files_to_sign.items():
from_ = os.path.join(tmp_dir, from_)
to = f"{from_}.sig"
if is_autograph:
tasks.append(asyncio.ensure_future(sign_widevine_with_autograph(
context, from_, "blessed" in fmt, to=to
)))
else:
tasks.append(asyncio.ensure_future(sign_file(
context, from_, fmt, to=to
)))
all_files.append(to)
await raise_future_exceptions(tasks)
remove_extra_files(tmp_dir, all_files)
# Regenerate the `precomplete` file, which is used for cleanup before
# applying a complete mar.
_run_generate_precomplete(context, tmp_dir)
await _create_zipfile(
context, orig_path, all_files, mode='w', tmp_dir=tmp_dir
)
return orig_path
async def sign_widevine_tar(context, orig_path, fmt):
"""Sign the internals of a tarfile with the widevine key.
Extract the entire tarball, but only sign a handful of files (see
`_WIDEVINE_BLESSED_FILENAMES` and `_WIDEVINE_UNBLESSED_FILENAMES).
The blessed files should be signed with the `widevine_blessed` format.
Then recreate the tarball.
Ideally we would be able to append the sigfiles to the original tarball,
but that's not possible with compressed tarballs.
Args:
context (Context): the signing context
orig_path (str): the source file to sign
fmt (str): the format to sign with
Returns:
str: the path to the signed archive
"""
_, compression = os.path.splitext(orig_path)
# This will get cleaned up when we nuke `work_dir`. Clean up at that point
# rather than immediately after `sign_widevine`, to optimize task runtime
# speed over disk space.
tmp_dir = tempfile.mkdtemp(prefix="wvtar", dir=context.config['work_dir'])
# Get file list
all_files = await _get_tarfile_files(orig_path, compression)
files_to_sign = _get_widevine_signing_files(all_files)
is_autograph = utils.is_autograph_signing_format(fmt)
log.debug("Widevine files to sign: %s", files_to_sign)
if files_to_sign:
# Extract all files so we can create `precomplete` with the full
# file list
all_files = await _extract_tarfile(
context, orig_path, compression, tmp_dir=tmp_dir
)
tasks = []
# Sign the appropriate inner files
for from_, fmt in files_to_sign.items():
from_ = os.path.join(tmp_dir, from_)
# Don't try to sign directories
if not os.path.isfile(from_):
continue
# Move the sig location on mac. This should be noop on linux.
to = _get_mac_sigpath(from_)
log.debug("Adding %s to the sigfile paths...", to)
makedirs(os.path.dirname(to))
if is_autograph:
tasks.append(asyncio.ensure_future(sign_widevine_with_autograph(
context, from_, "blessed" in fmt, to=to
)))
else:
tasks.append(asyncio.ensure_future(sign_file(
context, from_, fmt, to=to
)))
all_files.append(to)
await raise_future_exceptions(tasks)
remove_extra_files(tmp_dir, all_files)
# Regenerate the `precomplete` file, which is used for cleanup before
# applying a complete mar.
_run_generate_precomplete(context, tmp_dir)
await _create_tarfile(
context, orig_path, all_files, compression, tmp_dir=tmp_dir
)
return orig_path
async def sign_mar384_with_autograph_hash(context, from_, fmt, to=None):
"""Signs a hash with autograph, injects it into the file, and writes the result to arg `to` or `from_` if `to` is None.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
to (str, optional): the target path to sign to. If None, overwrite
`from_`. Defaults to None.
Raises:
Requests.RequestException: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
str: the path to the signed file
"""
log.info(
"sign_mar384_with_autograph_hash(): signing {} with {}... using autograph /sign/hash"
.format(from_, fmt))
if not utils.is_autograph_signing_format(fmt):
raise SigningScriptError(
"No signing servers found supporting signing format {}".format(
fmt))
cert_type = task.task_cert_type(context)
servers = get_suitable_signing_servers(context.signing_servers,
cert_type, [fmt],
raise_on_empty_list=True)
s = servers[0]
to = to or from_
hash_algo, expected_signature_length = 'sha384', 512
# Add a dummy signature into a temporary file (TODO: dedup with mardor.cli do_hash)
tmp = tempfile.TemporaryFile()
with open(from_, 'rb') as f:
add_signature_block(f, tmp, hash_algo)
tmp.seek(0)
with MarReader(tmp) as m:
hashes = m.calculate_hashes()
h = hashes[0][1]
tmp.close()
# build and run the hash signature request
sign_req = [{"input": base64.b64encode(h).decode('ascii')}]
log.debug(
"signing mar with hash alg %s using the default autograph keyid for %s",
hash_algo, s.user)
url = "%s/sign/hash" % s.server
async def make_sign_req():
auth = HawkAuth(id=s.user, key=s.password)
with requests.Session() as session:
r = session.post(url, json=sign_req, auth=auth)
log.debug("Autograph response: {}".format(
r.text[:120] if len(r.text) >= 120 else r.text))
r.raise_for_status()
return r.json()
sign_resp = await retry_async(make_sign_req)
signature = base64.b64decode(sign_resp[0]['signature'])
# Add a signature to the MAR file (TODO: dedup with mardor.cli do_add_signature)
if len(signature) != expected_signature_length:
raise SigningScriptError(
"signed mar hash signature has invalid length for hash algo {}. Got {} expected {}."
.format(hash_algo, len(signature), expected_signature_length))
# use the tmp file in case param `to` is `from_` which causes stream errors
tmp_dst = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
with open(tmp_dst.name, 'w+b') as dst:
with open(from_, 'rb') as src:
add_signature_block(src, dst, hash_algo, signature)
shutil.copyfile(tmp_dst.name, to)
os.unlink(tmp_dst.name)
verify_mar_signature(cert_type, fmt, to)
log.info("wrote mar with autograph signed hash %s to %s", from_, to)
return to
async def sign_file_with_autograph(context, from_, fmt, to=None):
"""Signs a file with autograph and writes the result to arg `to` or `from_` if `to` is None.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
to (str, optional): the target path to sign to. If None, overwrite
`from_`. Defaults to None.
Raises:
Requests.RequestException: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
str: the path to the signed file
"""
if not utils.is_autograph_signing_format(fmt):
raise SigningScriptError(
"No signing servers found supporting signing format {}".format(
fmt))
cert_type = task.task_cert_type(context)
servers = get_suitable_signing_servers(context.signing_servers,
cert_type, [fmt],
raise_on_empty_list=True)
s = servers[0]
to = to or from_
with open(from_, 'rb') as fin:
input_bytes = fin.read()
# We need to base64 data for autograph. b64encode() returns bytes, though. We need utf8 strings
# to make Python's JSON decoder happy
base64_input = base64.b64encode(input_bytes).decode('utf-8')
# build and run the signature request
sign_req = [{"input": base64_input}]
if utils.is_apk_autograph_signing_format(fmt):
# We don't want APKs to have their compression changed
sign_req[0]['options'] = {'zip': 'passthrough'}
if utils.is_sha1_apk_autograph_signing_format(fmt):
# We ask for a SHA1 digest from Autograph
# https://github.com/mozilla-services/autograph/pull/166/files
sign_req[0]['options']['pkcs7_digest'] = "SHA1"
log.debug("using the default autograph keyid for %s", s.user)
url = "%s/sign/file" % s.server
async def make_sign_req():
auth = HawkAuth(id=s.user, key=s.password)
with requests.Session() as session:
r = session.post(url, json=sign_req, auth=auth)
log.debug("Autograph response: {}".format(
r.text[:120] if len(r.text) >= 120 else r.text))
r.raise_for_status()
return r.json()
sign_resp = await retry_async(make_sign_req)
with open(to, 'wb') as fout:
fout.write(base64.b64decode(sign_resp[0]['signed_file']))
log.info("autograph wrote signed_file %s to %s", from_, to)
return to
async def async_main(context):
"""Sign all the things.
Args:
context (Context): the signing context.
"""
connector = _craft_aiohttp_connector(context)
# Create a session for talking to the legacy signing servers in order to
# generate a signing token
async with aiohttp.ClientSession(connector=connector) as session:
context.session = session
work_dir = context.config["work_dir"]
context.signing_servers = load_signing_server_config(context)
all_signing_formats = task_signing_formats(context)
if "gpg" in all_signing_formats or "autograph_gpg" in all_signing_formats:
if not context.config.get("gpg_pubkey"):
raise Exception("GPG format is enabled but gpg_pubkey is not defined")
if not os.path.exists(context.config["gpg_pubkey"]):
raise Exception(
"gpg_pubkey ({}) doesn't exist!".format(
context.config["gpg_pubkey"]
)
)
if "autograph_widevine" in all_signing_formats:
if not context.config.get("widevine_cert"):
raise Exception(
"Widevine format is enabled, but widevine_cert is not defined"
)
if not all(
is_autograph_signing_format(format_) for format_ in all_signing_formats
):
log.info("getting signingserver token")
await get_token(
context,
os.path.join(work_dir, "token"),
task_cert_type(context),
all_signing_formats,
)
# Create a new session to talk to autograph
async with aiohttp.ClientSession() as session:
context.session = session
filelist_dict = build_filelist_dict(context)
for path, path_dict in filelist_dict.items():
copy_to_dir(path_dict["full_path"], context.config["work_dir"], target=path)
log.info("signing %s", path)
output_files = await sign(
context, os.path.join(work_dir, path), path_dict["formats"]
)
for source in output_files:
source = os.path.relpath(source, work_dir)
copy_to_dir(
os.path.join(work_dir, source),
context.config["artifact_dir"],
target=source,
)
if "gpg" in path_dict["formats"] or "autograph_gpg" in path_dict["formats"]:
copy_to_dir(
context.config["gpg_pubkey"],
context.config["artifact_dir"],
target="public/build/KEY",
)
log.info("Done!")
