def test_read(self): with open('contrib/firmware.jcat', 'rb') as f: jcatfile = JcatFile(f.read()) self.assertEqual(len(jcatfile.items), 2) jcatitem = jcatfile.get_item('firmware.bin') self.assertEqual(len(jcatitem.blobs), 2) jcatblob = jcatitem.blobs[1] self.assertEqual(jcatblob.kind, JcatBlobKind.SHA256) self.assertEqual(jcatblob.data, b'2577281a88fe9e2a21c7dedbf844f546158ca568f1440eef430f9b6dca499a60')
def test_write(self): jcatfile = JcatFile() jcatitem = JcatItem('filename.bin') jcatfile.add_item(jcatitem) jcatitem.add_blob(JcatBlob(JcatBlobKind.SHA1, b'deadbeef')) jcatitem.add_blob(JcatBlob(JcatBlobKind.SHA256, b'deadbeef')) jcatitem.add_blob(JcatBlob(JcatBlobKind.GPG, b'beefdeaf')) with open('/tmp/firmware.jcat', 'wb') as f: f.write(jcatfile.save()) self.assertEqual(jcatfile.items, [jcatitem])
def main(): for fn in sys.argv[1:]: with open(fn, 'rb') as f: for jcatitem in JcatFile(f.read()).items: print(jcatitem) for jcatblob in jcatitem.blobs: print(jcatblob)
def _regenerate_and_sign_metadata(only_embargo=False): # get list of dirty remotes remotes = [] for r in db.session.query(Remote): if not r.is_signed: continue # fix up any remotes that are not dirty, but have firmware that is dirty # -- which shouldn't happen, but did... if not r.is_dirty: for fw in r.fws: if not fw.is_dirty: continue print('Marking remote %s as dirty due to %u' % (r.name, fw.firmware_id)) r.is_dirty = True if r.is_dirty: if r.is_public and only_embargo: continue remotes.append(r) # nothing to do if not remotes: return # set destination path from app config download_dir = app.config['DOWNLOAD_DIR'] if not os.path.exists(download_dir): os.mkdir(download_dir) # update everything required invalid_fns = [] for r in remotes: print('Updating: %s' % r.name) for r, blob_xmlgz in _metadata_update_targets(remotes): # write metadata-?????.xml.gz fn_xmlgz = os.path.join(download_dir, r.filename) with open(fn_xmlgz, 'wb') as f: f.write(blob_xmlgz) invalid_fns.append(fn_xmlgz) # write metadata.xml.gz fn_xmlgz = os.path.join(download_dir, r.filename_newest) with open(fn_xmlgz, 'wb') as f: f.write(blob_xmlgz) invalid_fns.append(fn_xmlgz) # create Jcat item with SHA256 checksum blob jcatfile = JcatFile() jcatitem = jcatfile.get_item(r.filename) jcatitem.add_alias_id(r.filename_newest) jcatitem.add_blob(JcatBlobSha1(blob_xmlgz)) jcatitem.add_blob(JcatBlobSha256(blob_xmlgz)) # write each signed file for blob in ploader.metadata_sign(blob_xmlgz): # add GPG only to archive for backwards compat with older fwupd if blob.kind == JcatBlobKind.GPG: fn_xmlgz_asc = fn_xmlgz + '.' + blob.filename_ext with open(fn_xmlgz_asc, 'wb') as f: f.write(blob.data) invalid_fns.append(fn_xmlgz_asc) # add to Jcat file too jcatitem.add_blob(blob) # write jcat file fn_xmlgz_jcat = fn_xmlgz + '.jcat' with open(fn_xmlgz_jcat, 'wb') as f: f.write(jcatfile.save()) invalid_fns.append(fn_xmlgz_jcat) # update PULP for r in remotes: if r.name == 'stable': _metadata_update_pulp(download_dir) # do this all at once right at the end of all the I/O for fn in invalid_fns: print('Invalidating {}'.format(fn)) ploader.file_modified(fn) # mark as no longer dirty for r in remotes: if not r.build_cnt: r.build_cnt = 0 r.build_cnt += 1 r.is_dirty = False db.session.commit() # drop caches in other sessions db.session.expire_all() # log what we did for r in remotes: _event_log('Signed metadata {} build {}'.format(r.name, r.build_cnt)) # only keep the last 6 metadata builds (24h / stable refresh every 4h) for r in remotes: if not r.filename: continue suffix = r.filename.split('-')[2] fns = glob.glob( os.path.join(download_dir, 'firmware-*-{}'.format(suffix))) for fn in sorted(fns): build_cnt = int(fn.split('-')[1]) if build_cnt + 6 > r.build_cnt: continue os.remove(fn) _event_log('Deleted metadata {} build {}'.format( r.name, build_cnt))
def _sign_fw(fw): # load the .cab file download_dir = app.config['DOWNLOAD_DIR'] fn = os.path.join(download_dir, fw.filename) try: with open(fn, 'rb') as f: cabarchive = CabArchive(f.read()) except IOError as e: raise NotImplementedError('cannot read %s: %s' % (fn, str(e))) # create Jcat file jcatfile = JcatFile() # sign each component in the archive print('Signing: %s' % fn) for md in fw.mds: try: # create Jcat item with SHA1 and SHA256 checksum blob cabfile = cabarchive[md.filename_contents] jcatitem = jcatfile.get_item(md.filename_contents) jcatitem.add_blob(JcatBlobSha1(cabfile.buf)) jcatitem.add_blob(JcatBlobSha256(cabfile.buf)) # sign using plugins for blob in ploader.archive_sign(cabfile.buf): # add GPG only to archive for backwards compat with older fwupd if blob.kind == JcatBlobKind.GPG: fn_blob = md.filename_contents + '.' + blob.filename_ext cabarchive[fn_blob] = CabFile(blob.data) # add to Jcat file too jcatitem.add_blob(blob) except KeyError as _: raise NotImplementedError('no {} firmware found'.format( md.filename_contents)) # rewrite the metainfo.xml file to reflect latest changes and sign it for md in fw.mds: # write new metainfo.xml file component = _generate_metadata_mds([md], metainfo=True) blob_xml = b'<?xml version="1.0" encoding="UTF-8"?>\n' + \ ET.tostring(component, encoding='UTF-8', xml_declaration=False, pretty_print=True) _show_diff(cabarchive[md.filename_xml].buf, blob_xml) cabarchive[md.filename_xml].buf = blob_xml # sign it jcatitem = jcatfile.get_item(md.filename_xml) jcatitem.add_blob(JcatBlobSha1(blob_xml)) jcatitem.add_blob(JcatBlobSha256(blob_xml)) for blob in ploader.archive_sign(blob_xml): jcatitem.add_blob(blob) # write jcat file if jcatfile.items: cabarchive['firmware.jcat'] = CabFile(jcatfile.save()) # overwrite old file cab_data = cabarchive.save() with open(fn, 'wb') as f: f.write(cab_data) # inform the plugin loader ploader.file_modified(fn) # update the download size for md in fw.mds: md.release_download_size = len(cab_data) # update the database fw.checksum_signed_sha1 = hashlib.sha1(cab_data).hexdigest() fw.checksum_signed_sha256 = hashlib.sha256(cab_data).hexdigest() fw.signed_timestamp = datetime.datetime.utcnow() db.session.commit()
def _regenerate_and_sign_metadata_remote(r): # not required */ if not r.is_signed: return # fix up any remotes that are not dirty, but have firmware that is dirty # -- which shouldn't happen, but did... if not r.is_dirty: for fw in r.fws: if not fw.is_dirty: continue print('Marking remote %s as dirty due to %u' % (r.name, fw.firmware_id)) r.is_dirty = True fw.is_dirty = False # not needed if not r.is_dirty: return # set destination path from app config download_dir = app.config['DOWNLOAD_DIR'] if not os.path.exists(download_dir): os.mkdir(download_dir) invalid_fns = [] print('Updating: %s' % r.name) # create metadata for each remote fws_filtered = [] for fw in db.session.query(Firmware): if fw.remote.name in ['private', 'deleted']: continue if not fw.signed_timestamp: continue if r.check_fw(fw): fws_filtered.append(fw) settings = _get_settings() blob_xmlgz = _generate_metadata_kind(fws_filtered, firmware_baseuri=settings['firmware_baseuri']) # write metadata-?????.xml.gz fn_xmlgz = os.path.join(download_dir, r.filename) with open(fn_xmlgz, 'wb') as f: f.write(blob_xmlgz) invalid_fns.append(fn_xmlgz) # write metadata.xml.gz fn_xmlgz = os.path.join(download_dir, r.filename_newest) with open(fn_xmlgz, 'wb') as f: f.write(blob_xmlgz) invalid_fns.append(fn_xmlgz) # create Jcat item with SHA256 checksum blob jcatfile = JcatFile() jcatitem = jcatfile.get_item(r.filename) jcatitem.add_alias_id(r.filename_newest) jcatitem.add_blob(JcatBlobSha1(blob_xmlgz)) jcatitem.add_blob(JcatBlobSha256(blob_xmlgz)) # write each signed file for blob in ploader.metadata_sign(blob_xmlgz): # add GPG only to archive for backwards compat with older fwupd if blob.kind == JcatBlobKind.GPG: fn_xmlgz_asc = fn_xmlgz + '.' + blob.filename_ext with open(fn_xmlgz_asc, 'wb') as f: f.write(blob.data) invalid_fns.append(fn_xmlgz_asc) # add to Jcat file too jcatitem.add_blob(blob) # write jcat file fn_xmlgz_jcat = fn_xmlgz + '.jcat' with open(fn_xmlgz_jcat, 'wb') as f: f.write(jcatfile.save()) invalid_fns.append(fn_xmlgz_jcat) # update PULP if r.name == 'stable': _metadata_update_pulp(download_dir) # do this all at once right at the end of all the I/O for fn in invalid_fns: print('Invalidating {}'.format(fn)) ploader.file_modified(fn) # mark as no longer dirty if not r.build_cnt: r.build_cnt = 0 r.build_cnt += 1 r.is_dirty = False # log what we did _event_log('Signed metadata {} build {}'.format(r.name, r.build_cnt)) # only keep the last 6 metadata builds (24h / stable refresh every 4h) suffix = r.filename.split('-')[2] fns = glob.glob(os.path.join(download_dir, 'firmware-*-{}'.format(suffix))) for fn in sorted(fns): build_cnt = int(fn.split('-')[1]) if build_cnt + 6 > r.build_cnt: continue os.remove(fn) _event_log('Deleted metadata {} build {}'.format(r.name, build_cnt)) # all firmwares are contained in the correct metadata now for fw in fws_filtered: fw.is_dirty = False db.session.commit()