def _update_metadata_from_fn(fwobj, fn): """ Re-parses the .cab file and updates the database version. """ # load cab file arc = cabarchive.CabArchive() try: cabextract_cmd = app.config['CABEXTRACT_CMD'] if os.path.exists(cabextract_cmd): arc.set_decompressor(cabextract_cmd) arc.parse_file(fn) except cabarchive.CorruptionError as e: return _error_internal('Invalid file type: %s' % str(e)) # parse the MetaInfo file cf = arc.find_file("*.metainfo.xml") if not cf: return _error_internal('The firmware file had no valid metadata') component = appstream.Component() try: component.parse(str(cf.contents)) except appstream.ParseError as e: return _error_internal('The metadata could not be parsed: ' + str(e)) # parse the inf file cf = arc.find_file("*.inf") if not cf: return _error_internal('The firmware file had no valid inf file') cfg = InfParser() cfg.read_data(cf.contents) try: tmp = cfg.get('Version', 'DriverVer') driver_ver = tmp.split(',') if len(driver_ver) != 2: return _error_internal( 'The inf file Version:DriverVer was invalid') except ConfigParser.NoOptionError as e: driver_ver = None # get the contents fw_data = arc.find_file('*.bin') if not fw_data: fw_data = arc.find_file('*.rom') if not fw_data: fw_data = arc.find_file('*.cap') if not fw_data: return _error_internal('No firmware found in the archive') # update sizes fwobj.mds[0].release_installed_size = len(fw_data.contents) fwobj.mds[0].release_download_size = os.path.getsize(fn) # update the descriptions fwobj.mds[0].release_description = component.releases[0].description fwobj.mds[0].description = component.description if driver_ver: fwobj.version_display = driver_ver[1] db.firmware.update(fwobj) return None
def main(): # parse junk app = appstream.Component() try: app.parse('junk') except appstream.ParseError: pass data = """<?xml version="1.0" encoding="UTF-8"?> <!-- Copyright 2015 Richard Hughes <*****@*****.**> --> <component type="firmware"> <id>com.hughski.ColorHug.firmware</id> <name>ColorHug Device Update</name> <summary> Firmware for the Hughski ColorHug Colorimeter </summary> <description> <p> Updating adds new features. </p> <p> 2nd para. </p> </description> <provides> <firmware type="flashed">40338ceb-b966-4eae-adae-9c32edfcc484</firmware> </provides> <url type="homepage">http://www.hughski.com/</url> <metadata_license>CC0-1.0</metadata_license> <project_license>GPL-2.0+</project_license> <updatecontact>richard_at_hughsie.com</updatecontact> <developer_name>Hughski Limited</developer_name> <releases> <release version="1.2.4" timestamp="1438454314"> <size type="installed">123456</size> <size type="download">654321</size> <checksum target="content" filename="firmware.bin" type="sha1">deadbeef</checksum> <description> <p>Fixes bugs:</p> <ul> <li>Fix the RC</li> <li>Scale the output</li> </ul> </description> </release> </releases> </component> """ app = appstream.Component() app.parse(data) app.validate() assert app.id == 'com.hughski.ColorHug.firmware', app.id assert app.name == 'ColorHug Device Update', app.name assert app.summary == 'Firmware for the Hughski ColorHug Colorimeter', app.summary assert app.description == '<p>Updating adds new features.</p><p>2nd para.</p>', app.description assert app.urls['homepage'] == 'http://www.hughski.com/', app.urls[ 'homepage'] assert app.metadata_license == 'CC0-1.0', app.metadata_license assert app.project_license == 'GPL-2.0+', app.project_license assert app.developer_name == 'Hughski Limited', app.developer_name tmp = app.get_provides_by_kind('firmware-flashed')[0].value assert tmp == '40338ceb-b966-4eae-adae-9c32edfcc484', tmp assert len(app.releases) == 1 for rel in app.releases: assert rel.version == '1.2.4', rel.version assert rel.timestamp == 1438454314, rel.timestamp assert rel.size_installed == 123456, rel.size_installed assert rel.size_download == 654321, rel.size_download assert rel.description == '<p>Fixes bugs:</p><ul><li>Fix the RC</li><li>Scale the output</li></ul>', rel.description assert len(rel.checksums) == 1, len(rel.checksums) for csum in rel.checksums: assert csum.kind == 'sha1', csum.kind assert csum.target == 'content', csum.target assert csum.value == 'deadbeef', csum.value assert csum.filename == 'firmware.bin', csum.filename # add extra information for AppStream file rel = app.releases[0] rel.location = 'http://localhost:8051/hughski-colorhug-als-3.0.2.cab' csum = appstream.Checksum() csum.value = 'deadbeef' csum.target = 'container' csum.filename = 'hughski-colorhug-als-3.0.2.cab' rel.add_checksum(csum) csum = appstream.Checksum() csum.value = 'beefdead' csum.target = 'content' csum.filename = 'firmware.bin' rel.add_checksum(csum) # add to store store = appstream.Store() store.add(app) # add new release data = """<?xml version="1.0" encoding="UTF-8"?> <!-- Copyright 2015 Richard Hughes <*****@*****.**> --> <component type="firmware"> <id>com.hughski.ColorHug.firmware</id> <releases> <release version="1.2.5" timestamp="1500000000"> <description><p>This release adds magic®.</p></description> </release> </releases> </component> """ app = appstream.Component() app.parse(data) store.add(app) print(store.to_xml().encode('utf-8')) store.to_file('/tmp/firmware.xml.gz')
def main(): # test import ss = appstream.Screenshot() print(ss) test_data = """ Fixes: - BIOS boot successfully with special food. - No related beep codes displayed when HDD disabled. Enhancemets: - Update Microcode to dave. - Update TGC function WINS test. """ xml = appstream.utils.import_description(test_data) print(xml) assert appstream.utils.validate_description(xml) test_data = """ 1. First version to support Win7 OS. 2. First version to support dock. """ xml = appstream.utils.import_description(test_data) print(xml) assert appstream.utils.validate_description(xml) # parse junk app = appstream.Component() try: app.parse('junk') except appstream.ParseError: pass data = """<?xml version="1.0" encoding="UTF-8"?> <!-- Copyright 2015 Richard Hughes <*****@*****.**> --> <component type="firmware"> <id>com.hughski.ColorHug.firmware</id> <name>ColorHug Device Update</name> <summary> Firmware for the Hughski ColorHug Colorimeter </summary> <description> <p> Updating adds new features. </p> <p> 2nd para. </p> </description> <provides> <firmware type="flashed">40338ceb-b966-4eae-adae-9c32edfcc484</firmware> </provides> <requires> <id compare="ge" version="0.8.2">org.freedesktop.fwupd</id> <firmware compare="regex" version="BOT03.0[0-1]_*">bootloader</firmware> <firmware compare="eq" version="USB:0x046X">vendor-id</firmware> </requires> <keywords> <keyword>one</keyword> <keyword>two</keyword> </keywords> <url type="homepage">http://www.hughski.com/</url> <metadata_license>CC0-1.0</metadata_license> <project_license>GPL-2.0+</project_license> <updatecontact>richard_at_hughsie.com</updatecontact> <developer_name>Hughski Limited</developer_name> <releases> <release version="1.2.4" timestamp="1438454314" date="2016-02-25" urgency="high"> <size type="installed">123456</size> <size type="download">654321</size> <checksum target="content" filename="firmware.bin" type="sha1">deadbeef</checksum> <description> <p>Fixes bugs:</p> <ul> <li>Fix the RC</li> <li>Scale the output</li> </ul> </description> </release> </releases> <reviews> <review date="2016-09-15" rating="80" score="5" karma="-1" id="17"> <summary>Hello world</summary> <description><p>Mighty Fine</p></description> <version>1.2.3</version> <reviewer_id>deadbeef</reviewer_id> <reviewer_name>Richard Hughes</reviewer_name> <lang>en_GB</lang> <metadata> <value key="foo">bar</value> </metadata> </review> </reviews> <screenshots> <screenshot type="default"> <image type="source">http://a.png</image> <image type="thumbnail" height="351" width="624">http://b.png</image> <caption><p>This is a caption</p></caption> </screenshot> <screenshot> <image>http://c.png</image> <caption>No markup</caption> </screenshot> </screenshots> <custom> <value key="foo">bar</value> </custom> </component> """ app = appstream.Component() app.parse(data) app.validate() assert app.id == 'com.hughski.ColorHug.firmware', app.id assert app.name == 'ColorHug Device Update', app.name assert app.summary == 'Firmware for the Hughski ColorHug Colorimeter', app.summary assert app.description == '<p>Updating adds new features.</p><p>2nd para.</p>', app.description assert app.urls['homepage'] == 'http://www.hughski.com/', app.urls[ 'homepage'] assert app.metadata_license == 'CC0-1.0', app.metadata_license assert app.project_license == 'GPL-2.0+', app.project_license assert app.developer_name == 'Hughski Limited', app.developer_name tmp = app.get_provides_by_kind('firmware-flashed')[0].value assert tmp == '40338ceb-b966-4eae-adae-9c32edfcc484', tmp req = app.get_require_by_kind('id', 'org.freedesktop.fwupd') assert req.kind == 'id', req.kind assert req.compare == 'ge', req.compare assert req.version == '0.8.2', req.version assert req.value == 'org.freedesktop.fwupd', req.value assert len(app.releases) == 1 assert len(app.keywords) == 2 for rel in app.releases: assert rel.version == '1.2.4', rel.version assert rel.timestamp == 1456358400, rel.timestamp assert rel.size_installed == 123456, rel.size_installed assert rel.size_download == 654321, rel.size_download assert rel.description == '<p>Fixes bugs:</p><ul><li>Fix the RC</li><li>Scale the output</li></ul>', rel.description assert rel.urgency == 'high', rel.urgency assert len(rel.checksums) == 1, len(rel.checksums) for csum in rel.checksums: assert csum.kind == 'sha1', csum.kind assert csum.target == 'content', csum.target assert csum.value == 'deadbeef', csum.value assert csum.filename == 'firmware.bin', csum.filename assert len(app.reviews) == 1 for rev in app.reviews: assert rev.id == '17', rev.id assert rev.summary == 'Hello world', rev.summary assert rev.description == '<p>Mighty Fine</p>', rev.description assert rev.version == '1.2.3', rev.version assert rev.reviewer_id == 'deadbeef', rev.reviewer_id assert rev.reviewer_name == 'Richard Hughes', rev.reviewer_name assert rev.locale == 'en_GB', rev.locale assert rev.karma == -1, rev.karma assert rev.score == 5, rev.score assert rev.rating == 80, rev.rating assert rev.date == 1473894000, rev.date assert len(rev.metadata) == 1 assert rev.metadata['foo'] == 'bar', rev.metadata # screenshots assert len(app.screenshots) == 2, app.screenshots ss = app.screenshots[0] assert ss.kind == 'default' assert ss.caption == '<p>This is a caption</p>', ss.caption assert len(ss.images) == 2, ss.images im = ss.images[0] assert im.kind == 'source', im.kind assert im.height == 0, im.height assert im.width == 0, im.width assert im.url == 'http://a.png', im.url im = ss.images[1] assert im.kind == 'thumbnail', im.kind assert im.height == 351, im.height assert im.width == 624, im.width assert im.url == 'http://b.png', im.url ss = app.screenshots[1] assert ss.caption == '<p>No markup</p>', ss.caption # custom metadata assert 'foo' in app.custom, app.custom assert app.custom['foo'] == 'bar', app.custom # add extra information for AppStream file rel = app.releases[0] rel.location = 'http://localhost:8051/hughski-colorhug-als-3.0.2.cab' csum = appstream.Checksum() csum.value = 'deadbeef' csum.target = 'container' csum.filename = 'hughski-colorhug-als-3.0.2.cab' rel.add_checksum(csum) csum = appstream.Checksum() csum.value = 'beefdead' csum.target = 'content' csum.filename = 'firmware.bin' rel.add_checksum(csum) # add to store store = appstream.Store() store.add(app) # add new release data = """<?xml version="1.0" encoding="UTF-8"?> <!-- Copyright 2015 Richard Hughes <*****@*****.**> --> <component type="firmware"> <id>com.hughski.ColorHug.firmware</id> <releases> <release version="1.2.5" timestamp="1500000000"> <description><p>This release adds magic®.</p></description> </release> </releases> </component> """ app = appstream.Component() app.parse(data) store.add(app) print(store.to_xml().encode('utf-8')) store.to_file('/tmp/firmware.xml.gz')
def upload(): """ Upload a .cab file to the LVFS service """ # only accept form data if request.method != 'POST': if 'username' not in session: return redirect(url_for('.index')) vendor_ids = [] try: item = db.groups.get_item(session['group_id']) except CursorError as e: return _error_internal(str(e)) if item: vendor_ids.extend(item.vendor_ids) return render_template('upload.html', vendor_ids=vendor_ids) # not correct parameters if not 'target' in request.form: return _error_internal('No target') if not 'file' in request.files: return _error_internal('No file') # can the user upload directly to stable if request.form['target'] in ['stable', 'testing']: if not session['qa_capability']: return _error_permission_denied( 'Unable to upload to this target as not QA user') # check size < 50Mb fileitem = request.files['file'] if not fileitem: return _error_internal('No file object') data = fileitem.read() if len(data) > 50000000: return _error_internal('File too large, limit is 50Mb', 413) if len(data) == 0: return _error_internal('File has no content') if len(data) < 1024: return _error_internal('File too small, mimimum is 1k') # check the file does not already exist firmware_id = hashlib.sha1(data).hexdigest() try: item = db.firmware.get_item(firmware_id) except CursorError as e: return _error_internal(str(e)) if item: return _error_internal( "A firmware file with hash %s already exists" % firmware_id, 422) # parse the file arc = cabarchive.CabArchive() try: cabextract_cmd = app.config['CABEXTRACT_CMD'] if os.path.exists(cabextract_cmd): arc.set_decompressor(cabextract_cmd) arc.parse(data) except cabarchive.CorruptionError as e: return _error_internal('Invalid file type: %s' % str(e), 415) except cabarchive.NotSupportedError as e: return _error_internal('The file is unsupported: %s' % str(e), 415) # check .inf exists fw_version_inf = None fw_version_display_inf = None cf = arc.find_file("*.inf") if cf: if cf.contents.find('FIXME') != -1: return _error_internal( "The inf file was not complete; " "Any FIXME text must be replaced with the correct values.") # check .inf file is valid cfg = InfParser() cfg.read_data(cf.contents) try: tmp = cfg.get('Version', 'Class') except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e: return _error_internal('The inf file Version:Class was missing') if tmp != 'Firmware': return _error_internal('The inf file Version:Class was invalid') try: tmp = cfg.get('Version', 'ClassGuid') except ConfigParser.NoOptionError as e: return _error_internal( 'The inf file Version:ClassGuid was missing') if tmp != '{f2e7dd72-6468-4e36-b6f1-6488f42c1b52}': return _error_internal( 'The inf file Version:ClassGuid was invalid') try: tmp = cfg.get('Version', 'DriverVer') fw_version_display_inf = tmp.split(',') if len(fw_version_display_inf) != 2: return _error_internal( 'The inf file Version:DriverVer was invalid') except ConfigParser.NoOptionError as e: pass # this is optional, but if supplied must match the version in the XML # -- also note this will not work with multi-firmware .cab files try: fw_version_inf = cfg.get('Firmware_AddReg', 'HKR->FirmwareVersion') if fw_version_inf.startswith('0x'): fw_version_inf = str(int(fw_version_inf[2:], 16)) if fw_version_inf == '0': fw_version_inf = None except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e: pass # check metainfo exists cfs = arc.find_files("*.metainfo.xml") if len(cfs) == 0: return _error_internal('The firmware file had no .metadata.xml files') # parse each MetaInfo file apps = [] for cf in cfs: component = appstream.Component() try: component.parse(str(cf.contents)) component.validate() except appstream.ParseError as e: return _error_internal('The metadata %s could not be parsed: %s' % (cf, str(e))) except appstream.ValidationError as e: return _error_internal( 'The metadata %s file did not validate: %s' % (cf, str(e))) # get the metadata ID component.custom['metainfo_id'] = hashlib.sha1(cf.contents).hexdigest() # check the file does not have any missing request.form if cf.contents.find('FIXME') != -1: return _error_internal( "The metadata file was not complete; " "Any FIXME text must be replaced with the correct values.") # check the firmware provides something if len(component.provides) == 0: return _error_internal( "The metadata file did not provide any GUID.") if len(component.releases) == 0: return _error_internal( "The metadata file did not provide any releases.") # check the inf file matches up with the .xml file if fw_version_inf and fw_version_inf != component.releases[0].version: return _error_internal( "The inf Firmware_AddReg[HKR->FirmwareVersion] " "'%s' did not match the metainfo.xml value '%s'." % (fw_version_inf, component.releases[0].version)) # check the guid and version does not already exist try: items = db.firmware.get_all() except CursorError as e: return _error_internal(str(e)) for item in items: for md in item.mds: for guid in md.guids: if guid == component.provides[ 0].value and md.version == component.releases[ 0].version: return _error_internal( "A firmware file for this version already exists", 422) # check if the file dropped a GUID previously supported new_guids = [] for prov in component.provides: new_guids.append(prov.value) for item in items: for md in item.mds: if md.cid != component.id: continue for old_guid in md.guids: if not old_guid in new_guids: return _error_internal( "Firmware %s dropped a GUID previously supported %s" % (md.cid, old_guid), 422) # check the file didn't try to add it's own <require> on vendor-id # to work around the vendor-id security checks in fwupd req = component.get_require_by_kind('firmware', 'vendor-id') if req: return _error_internal("Firmware cannot specify vendor-id", 422) # add to array apps.append(component) # only save if we passed all tests basename = os.path.basename(fileitem.filename) new_filename = firmware_id + '-' + basename # add these after parsing in case multiple components use the same file asc_files = {} # fix up the checksums and add the detached signature for component in apps: # ensure there's always a container checksum release = component.releases[0] csum = release.get_checksum_by_target('content') if not csum: csum = appstream.Checksum() csum.target = 'content' csum.filename = 'firmware.bin' component.releases[0].add_checksum(csum) # get the contents checksum fw_data = arc.find_file(csum.filename) if not fw_data: return _error_internal('No %s found in the archive' % csum.filename) csum.kind = 'sha1' csum.value = hashlib.sha1(fw_data.contents).hexdigest() # set the sizes release.size_installed = len(fw_data.contents) release.size_download = len(data) # add the detached signature if not already signed sig_data = arc.find_file(csum.filename + ".asc") if not sig_data: if csum.filename not in asc_files: try: affidavit = _create_affidavit() except NoKeyError as e: return _error_internal('Failed to sign archive: ' + str(e)) cff = cabarchive.CabFile(fw_data.filename + '.asc', affidavit.create(fw_data.contents)) asc_files[csum.filename] = cff else: # check this file is signed by something we trust try: affidavit = _create_affidavit() affidavit.verify(fw_data.contents) except NoKeyError as e: return _error_internal('Failed to verify archive: ' + str(e)) # add all the .asc files to the archive for key in asc_files: arc.add_file(asc_files[key]) # export the new archive and get the checksum cab_data = arc.save(compressed=True) checksum_container = hashlib.sha1(cab_data).hexdigest() # dump to a file download_dir = app.config['DOWNLOAD_DIR'] if not os.path.exists(download_dir): os.mkdir(download_dir) fn = os.path.join(download_dir, new_filename) open(fn, 'wb').write(cab_data) # dump to the CDN _upload_to_cdn(new_filename, StringIO(cab_data)) # create parent firmware object target = request.form['target'] fwobj = Firmware() fwobj.group_id = session['group_id'] fwobj.addr = _get_client_address() fwobj.filename = new_filename fwobj.firmware_id = firmware_id fwobj.target = target if fw_version_display_inf: fwobj.version_display = fw_version_display_inf[1] # create child metadata object for the component for component in apps: md = FirmwareMd() md.firmware_id = firmware_id md.metainfo_id = component.custom['metainfo_id'] md.cid = component.id md.name = component.name md.summary = component.summary md.developer_name = component.developer_name md.metadata_license = component.metadata_license md.project_license = component.project_license md.url_homepage = component.urls['homepage'] md.description = component.description md.checksum_container = checksum_container # from the provide for prov in component.provides: md.guids.append(prov.value) # from the release rel = component.releases[0] md.version = rel.version md.release_description = rel.description md.release_timestamp = rel.timestamp md.release_installed_size = rel.size_installed md.release_download_size = rel.size_download md.release_urgency = rel.urgency # from requires for req in component.requires: req_txt = "%s/%s/%s/%s" % (req.kind, req.value, req.compare, req.version) md.requirements.append(req_txt) # from the first screenshot if len(component.screenshots) > 0: ss = component.screenshots[0] if ss.caption: md.screenshot_caption = ss.caption if len(ss.images) > 0: im = ss.images[0] if im.url: md.screenshot_url = im.url # from the content checksum csum = component.releases[0].get_checksum_by_target('content') md.checksum_contents = csum.value md.filename_contents = csum.filename fwobj.mds.append(md) # add to database try: db.firmware.add(fwobj) except CursorError as e: return _error_internal(str(e)) # set correct response code _event_log("Uploaded file %s to %s" % (new_filename, target)) # ensure up to date try: if target == 'embargo': _metadata_update_group(fwobj.group_id) if target == 'stable': _metadata_update_targets(['stable', 'testing']) elif target == 'testing': _metadata_update_targets(['testing']) except NoKeyError as e: return _error_internal('Failed to sign metadata: ' + str(e)) except CursorError as e: return _error_internal('Failed to generate metadata: ' + str(e)) return redirect(url_for('.firmware_show', firmware_id=firmware_id))
def _generate_metadata_kind(filename, items, affidavit=None): """ Generates AppStream metadata of a specific kind """ store = appstream.Store('lvfs') for item in items: # add each component for md in item.mds: component = appstream.Component() component.id = md.cid component.kind = 'firmware' component.name = md.name component.summary = md.summary component.description = md.description if md.url_homepage: component.urls['homepage'] = md.url_homepage component.metadata_license = md.metadata_license component.project_license = md.project_license component.developer_name = md.developer_name # add provide for guid in md.guids: prov = appstream.Provide() prov.kind = 'firmware-flashed' prov.value = guid component.add_provide(prov) # add release if md.version: rel = appstream.Release() rel.version = md.version rel.description = md.release_description if md.release_timestamp: rel.timestamp = md.release_timestamp rel.checksums = [] rel.location = app.config['FIRMWARE_BASEURL'] + item.filename rel.size_installed = md.release_installed_size rel.size_download = md.release_download_size rel.urgency = md.release_urgency component.add_release(rel) # add container checksum if md.checksum_container: csum = appstream.Checksum() csum.target = 'container' csum.value = md.checksum_container csum.filename = item.filename rel.add_checksum(csum) # add content checksum if md.checksum_contents: csum = appstream.Checksum() csum.target = 'content' csum.value = md.checksum_contents csum.filename = md.filename_contents rel.add_checksum(csum) # add screenshot if md.screenshot_caption: ss = appstream.Screenshot() ss.caption = md.screenshot_caption if md.screenshot_url: im = appstream.Image() im.url = md.screenshot_url ss.add_image(im) component.add_screenshot(ss) # add requires for each allowed vendor_ids group = db.groups.get_item(item.group_id) if group.vendor_ids: req = appstream.Require() req.kind = 'firmware' req.value = 'vendor-id' if len(group.vendor_ids) == 1: req.compare = 'eq' else: req.compare = 'regex' req.version = '|'.join(group.vendor_ids) component.add_require(req) # add manual firmware or fwupd version requires for req_txt in md.requirements: split = req_txt.split('/', 4) req = appstream.Require() req.kind = split[0] req.value = split[1] req.compare = split[2] req.version = split[3] component.add_require(req) # add component store.add(component) # dump to file download_dir = app.config['DOWNLOAD_DIR'] if not os.path.exists(download_dir): os.mkdir(download_dir) filename = os.path.join(download_dir, filename) store.to_file(filename) # upload to the CDN blob = open(filename, 'rb').read() _upload_to_cdn(filename, blob) # generate and upload the detached signature if affidavit: blob_asc = affidavit.create(blob) _upload_to_cdn(filename + '.asc', blob_asc)