def verify_signature(mar, cert): log.info("Checking %s signature", mar) with open(mar, "rb") as mar_fh: m = MarReader(mar_fh) if not m.verify(verify_key=cert): raise ValueError("MAR Signature invalid: %s (%s) against %s", mar, m.signature_type, cert)
def test_check_bad_file_entry_before(mar_sha384, tmpdir): # Make a copy of mar_sha384 tmpmar = tmpdir.join('test.mar') mar_sha384.copy(tmpmar) with tmpmar.open('r+b') as f: with MarReader(f) as m: offset = m.mardata.header.index_offset offset += 4 f.seek(offset) f.write(b'\x00\x00\x00\x00') f.seek(0) with MarReader(f) as m: assert m.get_errors() == ["Entry 'message.txt' starts before data block"]
def test_check_bad_file_entry_size(mar_sha384, tmpdir): # Make a copy of mar_sha384 tmpmar = tmpdir.join('test.mar') mar_sha384.copy(tmpmar) with tmpmar.open('r+b') as f: with MarReader(f) as m: offset = m.mardata.header.index_offset offset += 8 f.seek(offset) f.write(b'\x12\x34\x56\x78') f.seek(0) with MarReader(f) as m: assert m.get_errors() == ["Entry 'message.txt' ends past data block"]
def test_verify_unsupportedalgo(): pubkey = open(TEST_PUBKEY, 'rb').read() with MarReader(open(TEST_MAR_BZ2, 'rb')) as m: m.mardata.signatures.sigs[0].algorithm_id = 3 with pytest.raises(ValueError) as e: m.verify(pubkey) assert "Unsupported signing algorithm: 3" in str(e.value)
async def sign_mar384_with_autograph_hash(context, from_, fmt, to=None): """Signs a hash with autograph, injects it into the file, and writes the result to arg `to` or `from_` if `to` is None. Args: context (Context): the signing context from_ (str): the source file to sign fmt (str): the format to sign with to (str, optional): the target path to sign to. If None, overwrite `from_`. Defaults to None. Raises: Requests.RequestException: on failure SigningScriptError: when no suitable signing server is found for fmt Returns: str: the path to the signed file """ cert_type = task.task_cert_type(context) # Get any key id that the task may have specified fmt, keyid = utils.split_autograph_format(fmt) # Call to check that we have a server available get_suitable_signing_servers(context.signing_servers, cert_type, [fmt], raise_on_empty_list=True) hash_algo, expected_signature_length = 'sha384', 512 # Add a dummy signature into a temporary file (TODO: dedup with mardor.cli do_hash) with tempfile.TemporaryFile() as tmp: with open(from_, 'rb') as f: add_signature_block(f, tmp, hash_algo) tmp.seek(0) with MarReader(tmp) as m: hashes = m.calculate_hashes() h = hashes[0][1] signature = await sign_hash_with_autograph(context, h, fmt, keyid) # Add a signature to the MAR file (TODO: dedup with mardor.cli do_add_signature) if len(signature) != expected_signature_length: raise SigningScriptError( "signed mar hash signature has invalid length for hash algo {}. Got {} expected {}.".format(hash_algo, len(signature), expected_signature_length) ) # use the tmp file in case param `to` is `from_` which causes stream errors tmp_dst = tempfile.NamedTemporaryFile(mode='w+b', delete=False) with open(tmp_dst.name, 'w+b') as dst: with open(from_, 'rb') as src: add_signature_block(src, dst, hash_algo, signature) to = to or from_ shutil.copyfile(tmp_dst.name, to) os.unlink(tmp_dst.name) verify_mar_signature(cert_type, fmt, to, keyid) log.info("wrote mar with autograph signed hash %s to %s", from_, to) return to
def test_signing(tmpdir, key_size, algo_id, test_keys): private_key, public_key = test_keys[key_size] message_p = tmpdir.join('message.txt') message_p.write('hello world') mar_p = tmpdir.join('test.mar') with mar_p.open('w+b') as f: with MarWriter(f, signing_key=private_key, channel='release', productversion='99.9', signing_algorithm=algo_id) as m: with tmpdir.as_cwd(): m.add('message.txt') assert mar_p.size() > 0 with mar_p.open('rb') as f: with MarReader(f) as m: assert m.mardata.additional.count == 1 assert m.mardata.signatures.count == 1 assert len(m.mardata.index.entries) == 1 assert m.mardata.index.entries[0].name == 'message.txt' m.extract(str(tmpdir.join('extracted'))) assert (tmpdir.join('extracted', 'message.txt').read('rb') == b'hello world') assert m.verify(public_key)
def test_extract_badpath(tmpdir): with MarReader(open(TEST_MAR_BZ2, 'rb')) as m: # Mess with the name e = m.mardata.index.entries[0] e.name = "../" + e.name with pytest.raises(ValueError): m.extract(str(tmpdir))
def do_list(marfile, detailed=False): """ List the MAR file. Yields lines of text to output """ with open(marfile, 'rb') as f: with MarReader(f) as m: if detailed: if m.compression_type: yield "Compression type: {}".format(m.compression_type) if m.signature_type: yield "Signature type: {}".format(m.signature_type) if m.mardata.signatures: plural = "s" if (m.mardata.signatures.count == 0 or m.mardata.signatures.count > 1) else "" yield "Signature block found with {} signature{}".format(m.mardata.signatures.count, plural) for s in m.mardata.signatures.sigs: yield "- Signature {} size {}".format(s.algorithm_id, s.size) yield "" if m.mardata.additional: yield "{} additional block found:".format(len(m.mardata.additional.sections)) for s in m.mardata.additional.sections: if s.id == 1: yield (" - Product Information Block:") yield (" - MAR channel name: {}".format(s.channel)) yield (" - Product version: {}".format(s.productversion)) yield "" else: yield ("Unknown additional data") yield ("{:7s} {:7s} {:7s}".format("SIZE", "MODE", "NAME")) for e in m.mardata.index.entries: yield ("{:<7d} {:04o} {}".format(e.size, e.flags, e.name))
def do_verify(marfile, keyfiles=None): """Verify the MAR file.""" try: with open(marfile, 'rb') as f: with MarReader(f) as m: # Check various parts of the mar file # e.g. signature algorithms and additional block sections errors = m.get_errors() if errors: print("File is not well formed: {}".format(errors)) sys.exit(1) if keyfiles: try: keys = get_keys(keyfiles, m.signature_type) except ValueError as e: print(e) sys.exit(1) if any(m.verify(key) for key in keys): print("Verification OK") return True else: print("Verification failed") sys.exit(1) else: print("Verification OK") return True except Exception as e: print("Error opening or parsing file: {}".format(e)) sys.exit(1)
def test_add_signature_sha384(tmpdir, test_keys): tmpmar = tmpdir.join('test.mar') with open(TEST_MAR_XZ, 'rb') as f: with tmpmar.open('wb') as dst: add_signature_block(f, dst, 'sha384') with MarReader(tmpmar.open('rb')) as m: hashes = m.calculate_hashes() assert hashes == [(2, b'\x08>\x82\x8d$\xbb\xa6Cg\xca\x15L\x9c\xf1\xde\x170\xbe\xeb8]\x17\xb9\xfdB\xa9\xd6\xf1(y\'\xf44\x1f\x01c%\xd4\x92\x1avm!\t\xd9\xc4\xfbv')] h = hashes[0][1] priv, pub = test_keys[4096] sig = sign_hash(priv, h, 'sha384') sigfile = tmpdir.join('signature') with sigfile.open('wb') as f: f.write(sig) tmpmar = tmpdir.join('output.mar') cli.do_add_signature(TEST_MAR_XZ, str(tmpmar), str(sigfile)) pubkey = tmpdir.join('pubkey') with pubkey.open('wb') as f: f.write(pub) assert cli.do_verify(str(tmpmar), [str(pubkey)])
def test_padding(tmpdir): """Check that adding a signature preserves the original padding""" message_p = tmpdir.join('message.txt') message_p.write('hello world') def padded_write(self, productversion, channel): self.fileobj.seek(self.additional_offset) extras = extras_header.build( dict( count=1, sections=[ dict( channel=six.u(channel), productversion=six.u(productversion), size=len(channel) + len(productversion) + 2 + 8 + 10, padding=b'\x00' * 10, ) ], )) self.fileobj.write(extras) self.last_offset = self.fileobj.tell() with patch.object(MarWriter, 'write_additional', padded_write): mar_p = tmpdir.join('test.mar') with mar_p.open('w+b') as f: with MarWriter(f, productversion='99.0', channel='1') as m: with tmpdir.as_cwd(): m.add('message.txt', compress='bz2') with mar_p.open('rb') as f: with MarReader(f) as m: assert m.mardata.additional.sections[0].padding == b'\x00' * 10
def test_add_signature(tmpdir, mar_cue, test_keys): dest_mar = tmpdir.join('test.mar') # Add a dummy signature with mar_cue.open('rb') as s, dest_mar.open('w+b') as f: add_signature_block(s, f, 'sha384') with mar_cue.open('rb') as s, MarReader(s) as m, dest_mar.open( 'rb') as f, MarReader(f) as m1: assert m.productinfo == m1.productinfo assert m.mardata.additional.sections == m1.mardata.additional.sections assert len(m.mardata.index.entries) == len(m1.mardata.index.entries) assert m1.mardata.signatures.count == 1 hashes = m1.calculate_hashes() assert len(hashes) == 1 assert hashes[0][ 1][:20] == b"\r\xa9x\x7f#\xf2m\x93a\xcc\xafJ=\x85\xa3Ss\xb43;" # Now sign the hash using the test keys, and add the signature back into the file private_key, public_key = test_keys[4096] sig = sign_hash(private_key, hashes[0][1], 'sha384') # Add the signature back into the file with mar_cue.open('rb') as s, dest_mar.open('w+b') as f: add_signature_block(s, f, 'sha384', sig) with dest_mar.open('rb') as f, MarReader(f) as m1: assert m1.verify(public_key) # Assert file contents are the same with dest_mar.open('rb') as f, MarReader(f) as m1: with MarReader(mar_cue.open('rb')) as m: offset_delta = m1.mardata.data_offset - m.mardata.data_offset for (e, e1) in zip(m.mardata.index.entries, m1.mardata.index.entries): assert e.name == e1.name assert e.flags == e1.flags assert e.size == e1.size assert e.offset == e1.offset - offset_delta s = b''.join(m.extract_entry(e, decompress=None)) s1 = b''.join(m1.extract_entry(e1, decompress=None)) assert len(s) == e.size assert len(s1) == e1.size assert s == s1
def test_main_create_chdir(tmpdir): tmpdir.join('hello.txt').write('hello world') tmpmar = tmpdir.join('test.mar') cli.main(['-C', str(tmpdir), '-c', str(tmpmar), 'hello.txt']) with MarReader(tmpmar.open('rb')) as m: assert len(m.mardata.index.entries) == 1 assert m.mardata.index.entries[0].name == 'hello.txt'
async def download_and_verify_mars(partials_config, allowed_url_prefixes, signing_cert): """Download, check signature, channel ID and unpack MAR files.""" # Separate these categories so we can opt to perform checks on only 'to' downloads. from_urls = extract_download_urls(partials_config, mar_type="from") to_urls = extract_download_urls(partials_config, mar_type="to") tasks = list() downloads = dict() semaphore = asyncio.Semaphore( 2) # Magic 2 to reduce network timeout errors. for url in from_urls.union(to_urls): verify_allowed_url(url, allowed_url_prefixes) downloads[url] = { "download_path": Path(tempfile.mkdtemp()) / Path(url).name, } tasks.append( retry_download(url, downloads[url]["download_path"], semaphore=semaphore)) await asyncio.gather(*tasks) for url in downloads: # Verify signature, but not from an artifact as we don't # depend on the signing task if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION" ) and not url.startswith(QUEUE_PREFIX): verify_signature(downloads[url]["download_path"], signing_cert) # Only validate the target channel ID, as we update from beta->release if url in to_urls: validate_mar_channel_id(downloads[url]["download_path"], os.environ["MAR_CHANNEL_ID"]) downloads[url]["extracted_path"] = tempfile.mkdtemp() with open(downloads[url]["download_path"], "rb") as mar_fh: log.info( "Unpacking %s into %s", downloads[url]["download_path"], downloads[url]["extracted_path"], ) m = MarReader(mar_fh) m.extract(downloads[url]["extracted_path"]) return downloads
def is_lzma_compressed_mar(mar): log.info("Checking %s for lzma compression", mar) result = MarReader(open(mar, 'rb')).compression_type == 'xz' if result: log.info("%s is lzma compressed", mar) else: log.info("%s is not lzma compressed", mar) return result
def test_check_bad_signature_algorithm(mar_sha384, tmpdir): # Make a copy of mar_sha384 tmpmar = tmpdir.join('test.mar') mar_sha384.copy(tmpmar) with tmpmar.open('r+b') as f: with MarReader(f) as m: assert m.mardata.signatures.count == 1 offset = m.mardata.signatures.offset offset += 12 f.seek(offset) f.write(b'\x12\x34\x56\x78') f.seek(0) with MarReader(f) as m: assert m.mardata.signatures.count == 1 assert m.mardata.signatures.sigs[0].algorithm_id == 0x12345678 assert m.get_errors() == ["Unknown signature algorithm: 0x12345678"]
def test_check_bad_extra_section_id(mar_sha384, tmpdir): # Make a copy of mar_sha384 tmpmar = tmpdir.join('test.mar') mar_sha384.copy(tmpmar) with tmpmar.open('r+b') as f: with MarReader(f) as m: assert m.mardata.additional.count == 1 offset = m.mardata.additional.offset offset += 8 f.seek(offset) f.write(b'\x12\x34\x56\x78') f.seek(0) with MarReader(f) as m: assert m.mardata.additional.count == 1 assert m.mardata.additional.sections[0].id == 0x12345678 assert m.get_errors() == ["Unknown extra section type: 0x12345678"]
def test_calculate_hashes(): with MarReader(open(TEST_MAR_BZ2, 'rb')) as m: hashes = m.calculate_hashes() assert len(hashes) == 1 assert hashes[0][0] == 1 assert hashes[0][1][:20] == b'\xcd%\x0e\x82z%7\xdb\x96\xb4^\x063ZFV8\xfa\xe8k' pubkey = open(TEST_PUBKEY, 'rb').read() assert verify_signature(pubkey, m.mardata.signatures.sigs[0].signature, hashes[0][1], 'sha1')
def test_empty_mar(tmpdir): mar_p = tmpdir.join('test.mar') with mar_p.open('w+b') as f: with MarWriter(f) as m: pass with mar_p.open('rb') as f: with MarReader(f) as m: assert len(m.mardata.index.entries) == 0 assert not m.mardata.signatures
def test_extract_bz2(tmpdir): with MarReader(open(TEST_MAR_BZ2, 'rb')) as m: m.extract(str(tmpdir)) assert sorted(tmpdir.listdir()) == [ tmpdir.join(f) for f in [ 'Contents', 'defaults', 'update-settings.ini', 'update.manifest', ]] # Check the contents. These should already be uncompressed assert (tmpdir.join('defaults/pref/channel-prefs.js').read('rb') == b'pref("app.update.channel", "release");\n')
def mar_content_sizes(mar_path): """Report on the file sizes contained within a MAR. Allows comparison of newly created compressed artifacts to see which is worth storing. """ # TODO decision path between binary mar and python mar mar_path = Path(mar_path) with mar_path.open(mode="rb") as fh: with MarReader(fh) as m: return { entry.name: entry.size for entry in m.mardata.index.entries }
def test_list_unknown_extra(mar_sha384, tmpdir): tmpmar = tmpdir.join('test.mar') mar_sha384.copy(tmpmar) with tmpmar.open('r+b') as f: with MarReader(f) as m: offset = m.mardata.additional.offset offset += 8 f.seek(offset) f.write(b'\x12\x34\x56\x78') f.seek(0) text = "\n".join(cli.do_list(str(tmpmar), detailed=True)) assert "Unknown additional data" in text
def test_verify_malformed(mar_sha384, tmpdir): tmpmar = tmpdir.join('test.mar') mar_sha384.copy(tmpmar) with tmpmar.open('r+b') as f: # Mess with the mar's file offsets with MarReader(f) as m: offset = m.mardata.header.index_offset offset += 8 f.seek(offset) f.write(b'\x12\x34\x56\x78') f.seek(0) with raises(SystemExit): assert not cli.do_verify(str(tmpmar))
def validate_mar_channel_id(mar, channel_ids): log.info("Checking %s for MAR_CHANNEL_ID %s", mar, channel_ids) # We may get a string with a list representation, or a single entry string. channel_ids = set(channel_ids.split(',')) product_info = MarReader(open(mar, 'rb')).productinfo if not isinstance(product_info, tuple): raise ValueError("Malformed product information in mar: {}".format(product_info)) found_channel_ids = set(product_info[1].split(',')) if not found_channel_ids.issubset(channel_ids): raise ValueError("MAR_CHANNEL_ID mismatch, {} not in {}".format( product_info[1], channel_ids)) log.info("%s channel %s in %s", mar, product_info[1], channel_ids)
def do_hash(hash_algo, marfile, asn1=False): """Output the hash for this MAR file.""" # Add a dummy signature into a temporary file dst = tempfile.TemporaryFile() with open(marfile, 'rb') as f: add_signature_block(f, dst, hash_algo) dst.seek(0) with MarReader(dst) as m: hashes = m.calculate_hashes() h = hashes[0][1] if asn1: h = format_hash(h, hash_algo) print(base64.b64encode(h).decode('ascii'))
def extract_mar(mar_path, destination): """Extract a MAR file into the given destination.""" # TODO decision path between binary mar and python mar if not destination.exists(): destination.mkdir(parents=True, exist_ok=True) elif destination.exists() and not destination.is_dir(): log.error("Destination path %s exists and is not a directory", destination) raise ValueError("Destination path %s exists and is not a directory", destination) log.info("Extracting %s into %s", mar_path, destination) with mar_path.open(mode="rb") as fh: with MarReader(fh) as m: m.extract(str(destination), decompress="auto")
def test_parsing(): with MarReader(open(TEST_MAR_BZ2, 'rb')) as m: mardata = m.mardata index = mardata.index entries = index.entries assert len(entries) == 5 assert entries[0] == dict(offset=392, size=141, flags=0o664, name='update.manifest') assert entries[1] == dict(offset=533, size=76, flags=0o664, name='defaults/pref/channel-prefs.js') assert mardata.additional.count == 1 assert mardata.additional.sections[0].channel == 'thunderbird-comm-esr' assert mardata.additional.sections[0].productversion == '100.0' assert mardata.signatures.count == 1 assert m.get_errors() is None
def test_writer_uncompressed(tmpdir): message_p = tmpdir.join('message.txt') message_p.write('hello world') mar_p = tmpdir.join('test.mar') with mar_p.open('wb') as f: with MarWriter(f) as m: with tmpdir.as_cwd(): m.add('message.txt', compress=None) assert mar_p.size() > 0 with mar_p.open('rb') as f: with MarReader(f) as m: assert m.mardata.additional is None assert m.mardata.signatures is None assert len(m.mardata.index.entries) == 1 assert m.mardata.index.entries[0].name == 'message.txt' m.extract(str(tmpdir.join('extracted'))) assert (tmpdir.join('extracted', 'message.txt').read('rb') == b'hello world')
def test_add_signature_sha1(tmpdir, test_keys): with MarReader(open(TEST_MAR_BZ2, 'rb')) as m: hashes = m.calculate_hashes() assert hashes == [(1, b'\xcd%\x0e\x82z%7\xdb\x96\xb4^\x063ZFV8\xfa\xe8k')] h = hashes[0][1] priv, pub = test_keys[2048] sig = sign_hash(priv, h, 'sha1') sigfile = tmpdir.join('signature') with sigfile.open('wb') as f: f.write(sig) tmpmar = tmpdir.join('output.mar') cli.do_add_signature(TEST_MAR_BZ2, str(tmpmar), str(sigfile)) pubkey = tmpdir.join('pubkey') with pubkey.open('wb') as f: f.write(pub) assert cli.do_verify(str(tmpmar), [str(pubkey)])
def test_additional(tmpdir): message_p = tmpdir.join('message.txt') message_p.write('hello world') mar_p = tmpdir.join('test.mar') with mar_p.open('w+b') as f: with MarWriter(f, productversion='99.9', channel='release') as m: with tmpdir.as_cwd(): m.add('message.txt') assert mar_p.size() > 0 with mar_p.open('rb') as f: with MarReader(f) as m: assert m.mardata.additional.count == 1 assert m.mardata.additional.sections[0].productversion == '99.9' assert m.mardata.additional.sections[0].channel == 'release' assert m.mardata.signatures.count == 0 assert len(m.mardata.index.entries) == 1 assert m.mardata.index.entries[0].name == 'message.txt' m.extract(str(tmpdir.join('extracted'))) assert (tmpdir.join('extracted', 'message.txt').read('rb') == b'hello world')
def verify_signature(mar, certs): log.info("Checking %s signature", mar) with open(mar, 'rb') as mar_fh: m = MarReader(mar_fh) m.verify(verify_key=certs.get(m.signature_type))