def verify_signature(mar, cert):
log.info("Checking %s signature", mar)
with open(mar, "rb") as mar_fh:
m = MarReader(mar_fh)
if not m.verify(verify_key=cert):
raise ValueError("MAR Signature invalid: %s (%s) against %s", mar,
m.signature_type, cert)
def test_check_bad_file_entry_before(mar_sha384, tmpdir):
# Make a copy of mar_sha384
tmpmar = tmpdir.join('test.mar')
mar_sha384.copy(tmpmar)
with tmpmar.open('r+b') as f:
with MarReader(f) as m:
offset = m.mardata.header.index_offset
offset += 4
f.seek(offset)
f.write(b'\x00\x00\x00\x00')
f.seek(0)
with MarReader(f) as m:
assert m.get_errors() == ["Entry 'message.txt' starts before data block"]
def test_check_bad_file_entry_size(mar_sha384, tmpdir):
# Make a copy of mar_sha384
tmpmar = tmpdir.join('test.mar')
mar_sha384.copy(tmpmar)
with tmpmar.open('r+b') as f:
with MarReader(f) as m:
offset = m.mardata.header.index_offset
offset += 8
f.seek(offset)
f.write(b'\x12\x34\x56\x78')
f.seek(0)
with MarReader(f) as m:
assert m.get_errors() == ["Entry 'message.txt' ends past data block"]
def test_verify_unsupportedalgo():
pubkey = open(TEST_PUBKEY, 'rb').read()
with MarReader(open(TEST_MAR_BZ2, 'rb')) as m:
m.mardata.signatures.sigs[0].algorithm_id = 3
with pytest.raises(ValueError) as e:
m.verify(pubkey)
assert "Unsupported signing algorithm: 3" in str(e.value)
async def sign_mar384_with_autograph_hash(context, from_, fmt, to=None):
"""Signs a hash with autograph, injects it into the file, and writes the result to arg `to` or `from_` if `to` is None.
Args:
context (Context): the signing context
from_ (str): the source file to sign
fmt (str): the format to sign with
to (str, optional): the target path to sign to. If None, overwrite
`from_`. Defaults to None.
Raises:
Requests.RequestException: on failure
SigningScriptError: when no suitable signing server is found for fmt
Returns:
str: the path to the signed file
"""
cert_type = task.task_cert_type(context)
# Get any key id that the task may have specified
fmt, keyid = utils.split_autograph_format(fmt)
# Call to check that we have a server available
get_suitable_signing_servers(context.signing_servers, cert_type, [fmt],
raise_on_empty_list=True)
hash_algo, expected_signature_length = 'sha384', 512
# Add a dummy signature into a temporary file (TODO: dedup with mardor.cli do_hash)
with tempfile.TemporaryFile() as tmp:
with open(from_, 'rb') as f:
add_signature_block(f, tmp, hash_algo)
tmp.seek(0)
with MarReader(tmp) as m:
hashes = m.calculate_hashes()
h = hashes[0][1]
signature = await sign_hash_with_autograph(context, h, fmt, keyid)
# Add a signature to the MAR file (TODO: dedup with mardor.cli do_add_signature)
if len(signature) != expected_signature_length:
raise SigningScriptError(
"signed mar hash signature has invalid length for hash algo {}. Got {} expected {}.".format(hash_algo, len(signature), expected_signature_length)
)
# use the tmp file in case param `to` is `from_` which causes stream errors
tmp_dst = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
with open(tmp_dst.name, 'w+b') as dst:
with open(from_, 'rb') as src:
add_signature_block(src, dst, hash_algo, signature)
to = to or from_
shutil.copyfile(tmp_dst.name, to)
os.unlink(tmp_dst.name)
verify_mar_signature(cert_type, fmt, to, keyid)
log.info("wrote mar with autograph signed hash %s to %s", from_, to)
return to
def test_signing(tmpdir, key_size, algo_id, test_keys):
private_key, public_key = test_keys[key_size]
message_p = tmpdir.join('message.txt')
message_p.write('hello world')
mar_p = tmpdir.join('test.mar')
with mar_p.open('w+b') as f:
with MarWriter(f,
signing_key=private_key,
channel='release',
productversion='99.9',
signing_algorithm=algo_id) as m:
with tmpdir.as_cwd():
m.add('message.txt')
assert mar_p.size() > 0
with mar_p.open('rb') as f:
with MarReader(f) as m:
assert m.mardata.additional.count == 1
assert m.mardata.signatures.count == 1
assert len(m.mardata.index.entries) == 1
assert m.mardata.index.entries[0].name == 'message.txt'
m.extract(str(tmpdir.join('extracted')))
assert (tmpdir.join('extracted',
'message.txt').read('rb') == b'hello world')
assert m.verify(public_key)
def test_extract_badpath(tmpdir):
with MarReader(open(TEST_MAR_BZ2, 'rb')) as m:
# Mess with the name
e = m.mardata.index.entries[0]
e.name = "../" + e.name
with pytest.raises(ValueError):
m.extract(str(tmpdir))
def do_list(marfile, detailed=False):
"""
List the MAR file.
Yields lines of text to output
"""
with open(marfile, 'rb') as f:
with MarReader(f) as m:
if detailed:
if m.compression_type:
yield "Compression type: {}".format(m.compression_type)
if m.signature_type:
yield "Signature type: {}".format(m.signature_type)
if m.mardata.signatures:
plural = "s" if (m.mardata.signatures.count == 0 or m.mardata.signatures.count > 1) else ""
yield "Signature block found with {} signature{}".format(m.mardata.signatures.count, plural)
for s in m.mardata.signatures.sigs:
yield "- Signature {} size {}".format(s.algorithm_id, s.size)
yield ""
if m.mardata.additional:
yield "{} additional block found:".format(len(m.mardata.additional.sections))
for s in m.mardata.additional.sections:
if s.id == 1:
yield (" - Product Information Block:")
yield (" - MAR channel name: {}".format(s.channel))
yield (" - Product version: {}".format(s.productversion))
yield ""
else:
yield ("Unknown additional data")
yield ("{:7s} {:7s} {:7s}".format("SIZE", "MODE", "NAME"))
for e in m.mardata.index.entries:
yield ("{:<7d} {:04o} {}".format(e.size, e.flags, e.name))
def do_verify(marfile, keyfiles=None):
"""Verify the MAR file."""
try:
with open(marfile, 'rb') as f:
with MarReader(f) as m:
# Check various parts of the mar file
# e.g. signature algorithms and additional block sections
errors = m.get_errors()
if errors:
print("File is not well formed: {}".format(errors))
sys.exit(1)
if keyfiles:
try:
keys = get_keys(keyfiles, m.signature_type)
except ValueError as e:
print(e)
sys.exit(1)
if any(m.verify(key) for key in keys):
print("Verification OK")
return True
else:
print("Verification failed")
sys.exit(1)
else:
print("Verification OK")
return True
except Exception as e:
print("Error opening or parsing file: {}".format(e))
sys.exit(1)
def test_add_signature_sha384(tmpdir, test_keys):
tmpmar = tmpdir.join('test.mar')
with open(TEST_MAR_XZ, 'rb') as f:
with tmpmar.open('wb') as dst:
add_signature_block(f, dst, 'sha384')
with MarReader(tmpmar.open('rb')) as m:
hashes = m.calculate_hashes()
assert hashes == [(2, b'\x08>\x82\x8d$\xbb\xa6Cg\xca\x15L\x9c\xf1\xde\x170\xbe\xeb8]\x17\xb9\xfdB\xa9\xd6\xf1(y\'\xf44\x1f\x01c%\xd4\x92\x1avm!\t\xd9\xc4\xfbv')]
h = hashes[0][1]
priv, pub = test_keys[4096]
sig = sign_hash(priv, h, 'sha384')
sigfile = tmpdir.join('signature')
with sigfile.open('wb') as f:
f.write(sig)
tmpmar = tmpdir.join('output.mar')
cli.do_add_signature(TEST_MAR_XZ, str(tmpmar), str(sigfile))
pubkey = tmpdir.join('pubkey')
with pubkey.open('wb') as f:
f.write(pub)
assert cli.do_verify(str(tmpmar), [str(pubkey)])
def test_padding(tmpdir):
"""Check that adding a signature preserves the original padding"""
message_p = tmpdir.join('message.txt')
message_p.write('hello world')
def padded_write(self, productversion, channel):
self.fileobj.seek(self.additional_offset)
extras = extras_header.build(
dict(
count=1,
sections=[
dict(
channel=six.u(channel),
productversion=six.u(productversion),
size=len(channel) + len(productversion) + 2 + 8 + 10,
padding=b'\x00' * 10,
)
],
))
self.fileobj.write(extras)
self.last_offset = self.fileobj.tell()
with patch.object(MarWriter, 'write_additional', padded_write):
mar_p = tmpdir.join('test.mar')
with mar_p.open('w+b') as f:
with MarWriter(f, productversion='99.0', channel='1') as m:
with tmpdir.as_cwd():
m.add('message.txt', compress='bz2')
with mar_p.open('rb') as f:
with MarReader(f) as m:
assert m.mardata.additional.sections[0].padding == b'\x00' * 10
def test_add_signature(tmpdir, mar_cue, test_keys):
dest_mar = tmpdir.join('test.mar')
# Add a dummy signature
with mar_cue.open('rb') as s, dest_mar.open('w+b') as f:
add_signature_block(s, f, 'sha384')
with mar_cue.open('rb') as s, MarReader(s) as m, dest_mar.open(
'rb') as f, MarReader(f) as m1:
assert m.productinfo == m1.productinfo
assert m.mardata.additional.sections == m1.mardata.additional.sections
assert len(m.mardata.index.entries) == len(m1.mardata.index.entries)
assert m1.mardata.signatures.count == 1
hashes = m1.calculate_hashes()
assert len(hashes) == 1
assert hashes[0][
1][:20] == b"\r\xa9x\x7f#\xf2m\x93a\xcc\xafJ=\x85\xa3Ss\xb43;"
# Now sign the hash using the test keys, and add the signature back into the file
private_key, public_key = test_keys[4096]
sig = sign_hash(private_key, hashes[0][1], 'sha384')
# Add the signature back into the file
with mar_cue.open('rb') as s, dest_mar.open('w+b') as f:
add_signature_block(s, f, 'sha384', sig)
with dest_mar.open('rb') as f, MarReader(f) as m1:
assert m1.verify(public_key)
# Assert file contents are the same
with dest_mar.open('rb') as f, MarReader(f) as m1:
with MarReader(mar_cue.open('rb')) as m:
offset_delta = m1.mardata.data_offset - m.mardata.data_offset
for (e, e1) in zip(m.mardata.index.entries,
m1.mardata.index.entries):
assert e.name == e1.name
assert e.flags == e1.flags
assert e.size == e1.size
assert e.offset == e1.offset - offset_delta
s = b''.join(m.extract_entry(e, decompress=None))
s1 = b''.join(m1.extract_entry(e1, decompress=None))
assert len(s) == e.size
assert len(s1) == e1.size
assert s == s1
def test_main_create_chdir(tmpdir):
tmpdir.join('hello.txt').write('hello world')
tmpmar = tmpdir.join('test.mar')
cli.main(['-C', str(tmpdir), '-c', str(tmpmar), 'hello.txt'])
with MarReader(tmpmar.open('rb')) as m:
assert len(m.mardata.index.entries) == 1
assert m.mardata.index.entries[0].name == 'hello.txt'
async def download_and_verify_mars(partials_config, allowed_url_prefixes,
signing_cert):
"""Download, check signature, channel ID and unpack MAR files."""
# Separate these categories so we can opt to perform checks on only 'to' downloads.
from_urls = extract_download_urls(partials_config, mar_type="from")
to_urls = extract_download_urls(partials_config, mar_type="to")
tasks = list()
downloads = dict()
semaphore = asyncio.Semaphore(
2) # Magic 2 to reduce network timeout errors.
for url in from_urls.union(to_urls):
verify_allowed_url(url, allowed_url_prefixes)
downloads[url] = {
"download_path": Path(tempfile.mkdtemp()) / Path(url).name,
}
tasks.append(
retry_download(url,
downloads[url]["download_path"],
semaphore=semaphore))
await asyncio.gather(*tasks)
for url in downloads:
# Verify signature, but not from an artifact as we don't
# depend on the signing task
if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"
) and not url.startswith(QUEUE_PREFIX):
verify_signature(downloads[url]["download_path"], signing_cert)
# Only validate the target channel ID, as we update from beta->release
if url in to_urls:
validate_mar_channel_id(downloads[url]["download_path"],
os.environ["MAR_CHANNEL_ID"])
downloads[url]["extracted_path"] = tempfile.mkdtemp()
with open(downloads[url]["download_path"], "rb") as mar_fh:
log.info(
"Unpacking %s into %s",
downloads[url]["download_path"],
downloads[url]["extracted_path"],
)
m = MarReader(mar_fh)
m.extract(downloads[url]["extracted_path"])
return downloads
def is_lzma_compressed_mar(mar):
log.info("Checking %s for lzma compression", mar)
result = MarReader(open(mar, 'rb')).compression_type == 'xz'
if result:
log.info("%s is lzma compressed", mar)
else:
log.info("%s is not lzma compressed", mar)
return result
def test_check_bad_signature_algorithm(mar_sha384, tmpdir):
# Make a copy of mar_sha384
tmpmar = tmpdir.join('test.mar')
mar_sha384.copy(tmpmar)
with tmpmar.open('r+b') as f:
with MarReader(f) as m:
assert m.mardata.signatures.count == 1
offset = m.mardata.signatures.offset
offset += 12
f.seek(offset)
f.write(b'\x12\x34\x56\x78')
f.seek(0)
with MarReader(f) as m:
assert m.mardata.signatures.count == 1
assert m.mardata.signatures.sigs[0].algorithm_id == 0x12345678
assert m.get_errors() == ["Unknown signature algorithm: 0x12345678"]
def test_check_bad_extra_section_id(mar_sha384, tmpdir):
# Make a copy of mar_sha384
tmpmar = tmpdir.join('test.mar')
mar_sha384.copy(tmpmar)
with tmpmar.open('r+b') as f:
with MarReader(f) as m:
assert m.mardata.additional.count == 1
offset = m.mardata.additional.offset
offset += 8
f.seek(offset)
f.write(b'\x12\x34\x56\x78')
f.seek(0)
with MarReader(f) as m:
assert m.mardata.additional.count == 1
assert m.mardata.additional.sections[0].id == 0x12345678
assert m.get_errors() == ["Unknown extra section type: 0x12345678"]
def test_calculate_hashes():
with MarReader(open(TEST_MAR_BZ2, 'rb')) as m:
hashes = m.calculate_hashes()
assert len(hashes) == 1
assert hashes[0][0] == 1
assert hashes[0][1][:20] == b'\xcd%\x0e\x82z%7\xdb\x96\xb4^\x063ZFV8\xfa\xe8k'
pubkey = open(TEST_PUBKEY, 'rb').read()
assert verify_signature(pubkey, m.mardata.signatures.sigs[0].signature, hashes[0][1], 'sha1')
def test_empty_mar(tmpdir):
mar_p = tmpdir.join('test.mar')
with mar_p.open('w+b') as f:
with MarWriter(f) as m:
pass
with mar_p.open('rb') as f:
with MarReader(f) as m:
assert len(m.mardata.index.entries) == 0
assert not m.mardata.signatures
def test_extract_bz2(tmpdir):
with MarReader(open(TEST_MAR_BZ2, 'rb')) as m:
m.extract(str(tmpdir))
assert sorted(tmpdir.listdir()) == [
tmpdir.join(f) for f in [
'Contents',
'defaults',
'update-settings.ini',
'update.manifest',
]]
# Check the contents. These should already be uncompressed
assert (tmpdir.join('defaults/pref/channel-prefs.js').read('rb') ==
b'pref("app.update.channel", "release");\n')
def mar_content_sizes(mar_path):
"""Report on the file sizes contained within a MAR.
Allows comparison of newly created compressed artifacts to see which
is worth storing.
"""
# TODO decision path between binary mar and python mar
mar_path = Path(mar_path)
with mar_path.open(mode="rb") as fh:
with MarReader(fh) as m:
return {
entry.name: entry.size
for entry in m.mardata.index.entries
}
def test_list_unknown_extra(mar_sha384, tmpdir):
tmpmar = tmpdir.join('test.mar')
mar_sha384.copy(tmpmar)
with tmpmar.open('r+b') as f:
with MarReader(f) as m:
offset = m.mardata.additional.offset
offset += 8
f.seek(offset)
f.write(b'\x12\x34\x56\x78')
f.seek(0)
text = "\n".join(cli.do_list(str(tmpmar), detailed=True))
assert "Unknown additional data" in text
def test_verify_malformed(mar_sha384, tmpdir):
tmpmar = tmpdir.join('test.mar')
mar_sha384.copy(tmpmar)
with tmpmar.open('r+b') as f:
# Mess with the mar's file offsets
with MarReader(f) as m:
offset = m.mardata.header.index_offset
offset += 8
f.seek(offset)
f.write(b'\x12\x34\x56\x78')
f.seek(0)
with raises(SystemExit):
assert not cli.do_verify(str(tmpmar))
def validate_mar_channel_id(mar, channel_ids):
log.info("Checking %s for MAR_CHANNEL_ID %s", mar, channel_ids)
# We may get a string with a list representation, or a single entry string.
channel_ids = set(channel_ids.split(','))
product_info = MarReader(open(mar, 'rb')).productinfo
if not isinstance(product_info, tuple):
raise ValueError("Malformed product information in mar: {}".format(product_info))
found_channel_ids = set(product_info[1].split(','))
if not found_channel_ids.issubset(channel_ids):
raise ValueError("MAR_CHANNEL_ID mismatch, {} not in {}".format(
product_info[1], channel_ids))
log.info("%s channel %s in %s", mar, product_info[1], channel_ids)
def do_hash(hash_algo, marfile, asn1=False):
"""Output the hash for this MAR file."""
# Add a dummy signature into a temporary file
dst = tempfile.TemporaryFile()
with open(marfile, 'rb') as f:
add_signature_block(f, dst, hash_algo)
dst.seek(0)
with MarReader(dst) as m:
hashes = m.calculate_hashes()
h = hashes[0][1]
if asn1:
h = format_hash(h, hash_algo)
print(base64.b64encode(h).decode('ascii'))
def extract_mar(mar_path, destination):
"""Extract a MAR file into the given destination."""
# TODO decision path between binary mar and python mar
if not destination.exists():
destination.mkdir(parents=True, exist_ok=True)
elif destination.exists() and not destination.is_dir():
log.error("Destination path %s exists and is not a directory",
destination)
raise ValueError("Destination path %s exists and is not a directory",
destination)
log.info("Extracting %s into %s", mar_path, destination)
with mar_path.open(mode="rb") as fh:
with MarReader(fh) as m:
m.extract(str(destination), decompress="auto")
def test_parsing():
with MarReader(open(TEST_MAR_BZ2, 'rb')) as m:
mardata = m.mardata
index = mardata.index
entries = index.entries
assert len(entries) == 5
assert entries[0] == dict(offset=392, size=141, flags=0o664,
name='update.manifest')
assert entries[1] == dict(offset=533, size=76, flags=0o664,
name='defaults/pref/channel-prefs.js')
assert mardata.additional.count == 1
assert mardata.additional.sections[0].channel == 'thunderbird-comm-esr'
assert mardata.additional.sections[0].productversion == '100.0'
assert mardata.signatures.count == 1
assert m.get_errors() is None
def test_writer_uncompressed(tmpdir):
message_p = tmpdir.join('message.txt')
message_p.write('hello world')
mar_p = tmpdir.join('test.mar')
with mar_p.open('wb') as f:
with MarWriter(f) as m:
with tmpdir.as_cwd():
m.add('message.txt', compress=None)
assert mar_p.size() > 0
with mar_p.open('rb') as f:
with MarReader(f) as m:
assert m.mardata.additional is None
assert m.mardata.signatures is None
assert len(m.mardata.index.entries) == 1
assert m.mardata.index.entries[0].name == 'message.txt'
m.extract(str(tmpdir.join('extracted')))
assert (tmpdir.join('extracted',
'message.txt').read('rb') == b'hello world')
def test_add_signature_sha1(tmpdir, test_keys):
with MarReader(open(TEST_MAR_BZ2, 'rb')) as m:
hashes = m.calculate_hashes()
assert hashes == [(1, b'\xcd%\x0e\x82z%7\xdb\x96\xb4^\x063ZFV8\xfa\xe8k')]
h = hashes[0][1]
priv, pub = test_keys[2048]
sig = sign_hash(priv, h, 'sha1')
sigfile = tmpdir.join('signature')
with sigfile.open('wb') as f:
f.write(sig)
tmpmar = tmpdir.join('output.mar')
cli.do_add_signature(TEST_MAR_BZ2, str(tmpmar), str(sigfile))
pubkey = tmpdir.join('pubkey')
with pubkey.open('wb') as f:
f.write(pub)
assert cli.do_verify(str(tmpmar), [str(pubkey)])
def test_additional(tmpdir):
message_p = tmpdir.join('message.txt')
message_p.write('hello world')
mar_p = tmpdir.join('test.mar')
with mar_p.open('w+b') as f:
with MarWriter(f, productversion='99.9', channel='release') as m:
with tmpdir.as_cwd():
m.add('message.txt')
assert mar_p.size() > 0
with mar_p.open('rb') as f:
with MarReader(f) as m:
assert m.mardata.additional.count == 1
assert m.mardata.additional.sections[0].productversion == '99.9'
assert m.mardata.additional.sections[0].channel == 'release'
assert m.mardata.signatures.count == 0
assert len(m.mardata.index.entries) == 1
assert m.mardata.index.entries[0].name == 'message.txt'
m.extract(str(tmpdir.join('extracted')))
assert (tmpdir.join('extracted',
'message.txt').read('rb') == b'hello world')
def verify_signature(mar, certs):
log.info("Checking %s signature", mar)
with open(mar, 'rb') as mar_fh:
m = MarReader(mar_fh)
m.verify(verify_key=certs.get(m.signature_type))
