def test_task_cert_type_error(context):
context.task = {"scopes": [TEST_CERT_TYPE, "project:releng:signing:cert:notdep"]}
with pytest.raises(ScriptWorkerTaskException):
stask.task_cert_type(context)
context.task = None
with pytest.raises(TaskVerificationError):
stask.task_cert_type(context)
def test_task_cert_type_no_double_cert_scopes(context):
context.task = {
'scopes': [
'project:mobile:focus:releng:signing:cert:release-signing',
'project:mobile:fenix:releng:signing:cert:release-signing',
]
}
with pytest.raises(TaskVerificationError):
stask.task_cert_type(context)
def test_task_cert_type_error(context):
context.task = {
'scopes': [
TEST_CERT_TYPE, 'project:releng:signing:cert:notdep',
'project:releng:signing:format:gpg'
]
}
with pytest.raises(ScriptWorkerTaskException):
stask.task_cert_type(context)
def test_task_cert_errors_when_2_different_projects_are_signed_in_the_same_task(
context):
context.config["taskcluster_scope_prefixes"] = [
"project:mobile:focus:releng:signing:",
"project:mobile:fenix:releng:signing:"
]
context.task = {
"scopes": [
"project:mobile:focus:releng:signing:cert:dep-signing",
"project:mobile:fenix:releng:signing:cert:dep-signing"
]
}
with pytest.raises(TaskVerificationError):
stask.task_cert_type(context)
async def sign_mar384_with_autograph_hash(context, from_, fmt, to=None):
"""Signs a hash with autograph, injects it into the file, and writes the result to arg `to` or `from_` if `to` is None.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
to (str, optional): the target path to sign to. If None, overwrite
`from_`. Defaults to None.
Raises:
Requests.RequestException: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
str: the path to the signed file
"""
cert_type = task.task_cert_type(context)
# Get any key id that the task may have specified
fmt, keyid = utils.split_autograph_format(fmt)
# Call to check that we have a server available
get_suitable_signing_servers(context.signing_servers, cert_type, [fmt],
raise_on_empty_list=True)
hash_algo, expected_signature_length = 'sha384', 512
# Add a dummy signature into a temporary file (TODO: dedup with mardor.cli do_hash)
with tempfile.TemporaryFile() as tmp:
with open(from_, 'rb') as f:
add_signature_block(f, tmp, hash_algo)
tmp.seek(0)
with MarReader(tmp) as m:
hashes = m.calculate_hashes()
h = hashes[0][1]
signature = await sign_hash_with_autograph(context, h, fmt, keyid)
# Add a signature to the MAR file (TODO: dedup with mardor.cli do_add_signature)
if len(signature) != expected_signature_length:
raise SigningScriptError(
"signed mar hash signature has invalid length for hash algo {}. Got {} expected {}.".format(hash_algo, len(signature), expected_signature_length)
)
# use the tmp file in case param `to` is `from_` which causes stream errors
tmp_dst = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
with open(tmp_dst.name, 'w+b') as dst:
with open(from_, 'rb') as src:
add_signature_block(src, dst, hash_algo, signature)
to = to or from_
shutil.copyfile(tmp_dst.name, to)
os.unlink(tmp_dst.name)
verify_mar_signature(cert_type, fmt, to, keyid)
log.info("wrote mar with autograph signed hash %s to %s", from_, to)
return to
async def sign_hash_with_autograph(context, hash_, fmt, keyid=None):
"""Signs hash with autograph and returns the result.
Args:
context (Context): the signing context
hash_ (bytes): the input hash to sign
fmt (str): the format to sign with
keyid (str): which key to use on autograph (optional)
Raises:
Requests.RequestException: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
bytes: the signature
"""
if not utils.is_autograph_signing_format(fmt):
raise SigningScriptError(f"Not an autograph format: {fmt}")
cert_type = task.task_cert_type(context)
servers = get_suitable_signing_servers(context.signing_servers, cert_type,
[fmt], raise_on_empty_list=True)
s = servers[0]
signature = base64.b64decode(await sign_with_autograph(s, hash_, fmt, 'hash', keyid))
return signature
def build_signtool_cmd(context, from_, fmt, to=None, servers=None):
"""Generate a signtool command to run.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
to (str, optional): the target path to sign to. If None, overwrite
`from_`. Defaults to None.
Returns:
list: the signtool command to run.
"""
to = to or from_
work_dir = context.config['work_dir']
token = os.path.join(work_dir, "token")
nonce = os.path.join(work_dir, "nonce")
cert_type = task.task_cert_type(context)
ssl_cert = context.config['ssl_cert']
signtool = context.config['signtool']
if not isinstance(signtool, (list, tuple)):
signtool = [signtool]
cmd = signtool + ["-n", nonce, "-t", token, "-c", ssl_cert]
for s in get_suitable_signing_servers(
context.signing_servers, cert_type, [fmt]
):
cmd.extend(["-H", s.server])
cmd.extend(["-f", fmt])
cmd.extend(["-o", to, from_])
return cmd
async def sign_file_with_autograph(context, from_, fmt, to=None):
"""Signs file with autograph and writes the results to a file.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
to (str, optional): the target path to sign to. If None, overwrite
`from_`. Defaults to None.
Raises:
Requests.RequestException: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
str: the path to the signed file
"""
if not utils.is_autograph_signing_format(fmt):
raise SigningScriptError(f"Not an autograph format: {fmt}")
cert_type = task.task_cert_type(context)
servers = get_suitable_signing_servers(context.signing_servers, cert_type,
[fmt], raise_on_empty_list=True)
s = servers[0]
to = to or from_
input_bytes = open(from_, 'rb').read()
signed_bytes = base64.b64decode(await sign_with_autograph(s, input_bytes, fmt, 'file'))
with open(to, 'wb') as fout:
fout.write(signed_bytes)
return to
async def sign_file_with_autograph(context, from_, fmt, to=None, extension_id=None):
"""Signs file with autograph and writes the results to a file.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
to (str, optional): the target path to sign to. If None, overwrite
`from_`. Defaults to None.
extension_id (str, optional): the extension id to use when signing.
Raises:
aiohttp.ClientError: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
str: the path to the signed file
"""
cert_type = task.task_cert_type(context)
a = get_autograph_config(context.autograph_configs, cert_type, [fmt], raise_on_empty=True)
to = to or from_
input_file = open(from_, "rb")
signed_bytes = base64.b64decode(await sign_with_autograph(context.session, a, input_file, fmt, "file", extension_id=extension_id))
with open(to, "wb") as fout:
fout.write(signed_bytes)
return to
async def async_main(context):
work_dir = context.config['work_dir']
context.task = scriptworker.client.get_task(context.config)
log.info("validating task")
validate_task_schema(context)
context.signing_servers = load_signing_server_config(context)
cert_type = task_cert_type(context.task)
signing_formats = task_signing_formats(context.task)
# TODO scriptworker needs to validate CoT artifact
# Use standard SSL for downloading files.
with aiohttp.ClientSession() as base_ssl_session:
orig_session = context.session
context.session = base_ssl_session
file_urls = context.task["payload"]["unsignedArtifacts"]
filelist = await download_artifacts(context, file_urls)
context.session = orig_session
log.info("getting token")
await get_token(context, os.path.join(work_dir, 'token'), cert_type, signing_formats)
for filepath in filelist:
log.info("signing %s", filepath)
source = os.path.join(work_dir, filepath)
await sign_file(context, source, cert_type, signing_formats,
context.config["ssl_cert"])
sigfiles = detached_sigfiles(filepath, signing_formats)
copy_to_artifact_dir(context, source, target=filepath)
for sigpath in sigfiles:
copy_to_artifact_dir(context, os.path.join(work_dir, sigpath), target=sigpath)
log.info("Done!")
async def sign_gpg_with_autograph(context, from_, fmt):
"""Signs file with autograph and writes the results to a file.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
Raises:
Requests.RequestException: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
list: the path to the signed file, and sig.
"""
if not utils.is_autograph_signing_format(fmt):
raise SigningScriptError(f"Not an autograph format: {fmt}")
cert_type = task.task_cert_type(context)
servers = get_suitable_signing_servers(context.signing_servers, cert_type,
[fmt], raise_on_empty_list=True)
s = servers[0]
to = f"{from_}.asc"
input_bytes = open(from_, 'rb').read()
signature = await sign_with_autograph(s, input_bytes, fmt, 'data')
with open(to, 'w') as fout:
fout.write(signature)
return [from_, to]
def test_task_cert_type(context):
context.task = {
'scopes': [
TEST_CERT_TYPE, "project:releng:signing:format:mar",
"project:releng:signing:format:gpg"
]
}
assert TEST_CERT_TYPE == stask.task_cert_type(context)
async def async_main(context):
"""Sign all the things.
Args:
context (Context): the signing context.
"""
connector = _craft_aiohttp_connector(context)
async with aiohttp.ClientSession(connector=connector) as session:
context.session = session
work_dir = context.config['work_dir']
context.signing_servers = load_signing_server_config(context)
all_signing_formats = task_signing_formats(context)
if 'gpg' in all_signing_formats or 'autograph_gpg' in all_signing_formats:
if not context.config.get('gpg_pubkey'):
raise Exception(
"GPG format is enabled but gpg_pubkey is not defined")
if not os.path.exists(context.config['gpg_pubkey']):
raise Exception("gpg_pubkey ({}) doesn't exist!".format(
context.config['gpg_pubkey']))
if 'autograph_widevine' in all_signing_formats:
if not context.config.get('widevine_cert'):
raise Exception(
"Widevine format is enabled, but widevine_cert is not defined"
)
if not all(
is_autograph_signing_format(format_)
for format_ in all_signing_formats):
log.info("getting signingserver token")
await get_token(context, os.path.join(work_dir, 'token'),
task_cert_type(context), all_signing_formats)
filelist_dict = build_filelist_dict(context)
for path, path_dict in filelist_dict.items():
copy_to_dir(path_dict['full_path'],
context.config['work_dir'],
target=path)
log.info("signing %s", path)
output_files = await sign(context, os.path.join(work_dir, path),
path_dict['formats'])
for source in output_files:
source = os.path.relpath(source, work_dir)
copy_to_dir(os.path.join(work_dir, source),
context.config['artifact_dir'],
target=source)
if 'gpg' in path_dict['formats'] or 'autograph_gpg' in path_dict[
'formats']:
copy_to_dir(context.config['gpg_pubkey'],
context.config['artifact_dir'],
target="public/build/KEY")
log.info("Done!")
async def sign_file_with_autograph(context, from_, fmt, to=None):
"""Signs a file with autograph and writes the result to arg `to` or `from_` if `to` is None.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
to (str, optional): the target path to sign to. If None, overwrite
`from_`. Defaults to None.
Raises:
Requests.RequestException: on failure
Returns:
str: the path to the signed file
"""
if not utils.is_autograph_signing_format(fmt):
raise SigningScriptError(
"No signing servers found supporting signing format {}".format(
fmt))
cert_type = task.task_cert_type(context)
servers = get_suitable_signing_servers(context.signing_servers, cert_type,
[fmt])
if not servers:
raise SigningScriptError(
"No signing servers found with cert type {}".format(cert_type))
s = servers[0]
to = to or from_
with open(from_, 'rb') as fin:
input_bytes = fin.read()
# build and run the signature request
sign_req = [{"input": base64.b64encode(input_bytes)}]
log.debug("using the default autograph keyid for %s", s.user)
url = "%s/sign/file" % s.server
async def make_sign_req():
auth = HawkAuth(id=s.user, key=s.password)
with requests.Session() as session:
r = session.post(url, json=sign_req, auth=auth)
log.debug("Autograph response: {}".format(
r.text[:120] if len(r.text) >= 120 else r.text))
r.raise_for_status()
return r.json()
sign_resp = await retry_async(make_sign_req)
with open(to, 'wb') as fout:
fout.write(base64.b64decode(sign_resp[0]['signed_file']))
log.info("autograph wrote signed_file %s to %s", from_, to)
return to
async def async_main(context):
work_dir = context.config['work_dir']
context.task = scriptworker.client.get_task(context.config)
log.info("validating task")
validate_task_schema(context)
context.signing_servers = load_signing_server_config(context)
cert_type = task_cert_type(context.task)
all_signing_formats = task_signing_formats(context.task)
log.info("getting token")
await get_token(context, os.path.join(work_dir, 'token'), cert_type, all_signing_formats)
filelist_dict = build_filelist_dict(context, all_signing_formats)
for path, path_dict in filelist_dict.items():
copy_to_dir(path_dict['full_path'], context.config['work_dir'], target=path)
log.info("signing %s", path)
source = os.path.join(work_dir, path)
await sign_file(context, source, cert_type, path_dict['formats'], context.config["ssl_cert"])
sigfiles = detached_sigfiles(path, path_dict['formats'])
copy_to_dir(source, context.config['artifact_dir'], target=path)
for sigpath in sigfiles:
copy_to_dir(os.path.join(work_dir, sigpath), context.config['artifact_dir'], target=sigpath)
log.info("Done!")
async def sign_hash_with_autograph(context, hash_, fmt, keyid=None):
"""Signs hash with autograph and returns the result.
Args:
context (Context): the signing context
hash_ (bytes): the input hash to sign
fmt (str): the format to sign with
keyid (str): which key to use on autograph (optional)
Raises:
aiohttp.ClientError: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
bytes: the signature
"""
cert_type = task.task_cert_type(context)
a = get_autograph_config(context.autograph_configs, cert_type, [fmt], raise_on_empty=True)
input_file = BytesIO(hash_)
signature = base64.b64decode(await sign_with_autograph(context.session, a, input_file, fmt, "hash", keyid))
return signature
async def sign_gpg_with_autograph(context, from_, fmt):
"""Signs file with autograph and writes the results to a file.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
Raises:
aiohttp.ClientError: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
list: the path to the signed file, and sig.
"""
cert_type = task.task_cert_type(context)
a = get_autograph_config(context.autograph_configs, cert_type, [fmt], raise_on_empty=True)
to = f"{from_}.asc"
input_file = open(from_, "rb")
signature = await sign_with_autograph(context.session, a, input_file, fmt, "data")
with open(to, "w") as fout:
fout.write(signature)
return [from_, to]
def test_task_cert_type_error():
task = {"scopes": ["project:releng:signing:cert:dep-signing",
"project:releng:signing:cert:notdep",
"project:releng:signing:type:gpg"]}
with pytest.raises(ScriptWorkerTaskException):
task_cert_type(task)
def test_task_cert_type():
task = {"scopes": ["project:releng:signing:cert:dep-signing",
"project:releng:signing:type:mar",
"project:releng:signing:type:gpg"]}
assert "project:releng:signing:cert:dep-signing" == task_cert_type(task)
def test_task_cert_type(context):
context.task = {"scopes": [TEST_CERT_TYPE]}
assert TEST_CERT_TYPE == stask.task_cert_type(context)
async def async_main(context):
"""Sign all the things.
Args:
context (Context): the signing context.
"""
connector = _craft_aiohttp_connector(context)
# Create a session for talking to the legacy signing servers in order to
# generate a signing token
async with aiohttp.ClientSession(connector=connector) as session:
context.session = session
work_dir = context.config["work_dir"]
context.signing_servers = load_signing_server_config(context)
all_signing_formats = task_signing_formats(context)
if "gpg" in all_signing_formats or "autograph_gpg" in all_signing_formats:
if not context.config.get("gpg_pubkey"):
raise Exception("GPG format is enabled but gpg_pubkey is not defined")
if not os.path.exists(context.config["gpg_pubkey"]):
raise Exception(
"gpg_pubkey ({}) doesn't exist!".format(
context.config["gpg_pubkey"]
)
)
if "autograph_widevine" in all_signing_formats:
if not context.config.get("widevine_cert"):
raise Exception(
"Widevine format is enabled, but widevine_cert is not defined"
)
if not all(
is_autograph_signing_format(format_) for format_ in all_signing_formats
):
log.info("getting signingserver token")
await get_token(
context,
os.path.join(work_dir, "token"),
task_cert_type(context),
all_signing_formats,
)
# Create a new session to talk to autograph
async with aiohttp.ClientSession() as session:
context.session = session
filelist_dict = build_filelist_dict(context)
for path, path_dict in filelist_dict.items():
copy_to_dir(path_dict["full_path"], context.config["work_dir"], target=path)
log.info("signing %s", path)
output_files = await sign(
context, os.path.join(work_dir, path), path_dict["formats"]
)
for source in output_files:
source = os.path.relpath(source, work_dir)
copy_to_dir(
os.path.join(work_dir, source),
context.config["artifact_dir"],
target=source,
)
if "gpg" in path_dict["formats"] or "autograph_gpg" in path_dict["formats"]:
copy_to_dir(
context.config["gpg_pubkey"],
context.config["artifact_dir"],
target="public/build/KEY",
)
log.info("Done!")
async def sign_file_with_autograph(context, from_, fmt, to=None):
"""Signs a file with autograph and writes the result to arg `to` or `from_` if `to` is None.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
to (str, optional): the target path to sign to. If None, overwrite
`from_`. Defaults to None.
Raises:
Requests.RequestException: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
str: the path to the signed file
"""
if not utils.is_autograph_signing_format(fmt):
raise SigningScriptError(
"No signing servers found supporting signing format {}".format(
fmt))
cert_type = task.task_cert_type(context)
servers = get_suitable_signing_servers(context.signing_servers,
cert_type, [fmt],
raise_on_empty_list=True)
s = servers[0]
to = to or from_
with open(from_, 'rb') as fin:
input_bytes = fin.read()
# We need to base64 data for autograph. b64encode() returns bytes, though. We need utf8 strings
# to make Python's JSON decoder happy
base64_input = base64.b64encode(input_bytes).decode('utf-8')
# build and run the signature request
sign_req = [{"input": base64_input}]
if utils.is_apk_autograph_signing_format(fmt):
# We don't want APKs to have their compression changed
sign_req[0]['options'] = {'zip': 'passthrough'}
if utils.is_sha1_apk_autograph_signing_format(fmt):
# We ask for a SHA1 digest from Autograph
# https://github.com/mozilla-services/autograph/pull/166/files
sign_req[0]['options']['pkcs7_digest'] = "SHA1"
log.debug("using the default autograph keyid for %s", s.user)
url = "%s/sign/file" % s.server
async def make_sign_req():
auth = HawkAuth(id=s.user, key=s.password)
with requests.Session() as session:
r = session.post(url, json=sign_req, auth=auth)
log.debug("Autograph response: {}".format(
r.text[:120] if len(r.text) >= 120 else r.text))
r.raise_for_status()
return r.json()
sign_resp = await retry_async(make_sign_req)
with open(to, 'wb') as fout:
fout.write(base64.b64decode(sign_resp[0]['signed_file']))
log.info("autograph wrote signed_file %s to %s", from_, to)
return to
async def sign_mar384_with_autograph_hash(context, from_, fmt, to=None):
"""Signs a hash with autograph, injects it into the file, and writes the result to arg `to` or `from_` if `to` is None.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
to (str, optional): the target path to sign to. If None, overwrite
`from_`. Defaults to None.
Raises:
Requests.RequestException: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
str: the path to the signed file
"""
log.info(
"sign_mar384_with_autograph_hash(): signing {} with {}... using autograph /sign/hash"
.format(from_, fmt))
if not utils.is_autograph_signing_format(fmt):
raise SigningScriptError(
"No signing servers found supporting signing format {}".format(
fmt))
cert_type = task.task_cert_type(context)
servers = get_suitable_signing_servers(context.signing_servers,
cert_type, [fmt],
raise_on_empty_list=True)
s = servers[0]
to = to or from_
hash_algo, expected_signature_length = 'sha384', 512
# Add a dummy signature into a temporary file (TODO: dedup with mardor.cli do_hash)
tmp = tempfile.TemporaryFile()
with open(from_, 'rb') as f:
add_signature_block(f, tmp, hash_algo)
tmp.seek(0)
with MarReader(tmp) as m:
hashes = m.calculate_hashes()
h = hashes[0][1]
tmp.close()
# build and run the hash signature request
sign_req = [{"input": base64.b64encode(h).decode('ascii')}]
log.debug(
"signing mar with hash alg %s using the default autograph keyid for %s",
hash_algo, s.user)
url = "%s/sign/hash" % s.server
async def make_sign_req():
auth = HawkAuth(id=s.user, key=s.password)
with requests.Session() as session:
r = session.post(url, json=sign_req, auth=auth)
log.debug("Autograph response: {}".format(
r.text[:120] if len(r.text) >= 120 else r.text))
r.raise_for_status()
return r.json()
sign_resp = await retry_async(make_sign_req)
signature = base64.b64decode(sign_resp[0]['signature'])
# Add a signature to the MAR file (TODO: dedup with mardor.cli do_add_signature)
if len(signature) != expected_signature_length:
raise SigningScriptError(
"signed mar hash signature has invalid length for hash algo {}. Got {} expected {}."
.format(hash_algo, len(signature), expected_signature_length))
# use the tmp file in case param `to` is `from_` which causes stream errors
tmp_dst = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
with open(tmp_dst.name, 'w+b') as dst:
with open(from_, 'rb') as src:
add_signature_block(src, dst, hash_algo, signature)
shutil.copyfile(tmp_dst.name, to)
os.unlink(tmp_dst.name)
verify_mar_signature(cert_type, fmt, to)
log.info("wrote mar with autograph signed hash %s to %s", from_, to)
return to
