def _update_metadata_from_fn(fwobj, fn):
"""
Re-parses the .cab file and updates the database version.
"""
# load cab file
arc = cabarchive.CabArchive()
try:
if os.path.exists(CABEXTRACT_CMD):
arc.set_decompressor(CABEXTRACT_CMD)
arc.parse_file(fn)
except cabarchive.CorruptionError as e:
return error_internal('Invalid file type: %s' % str(e))
# parse the MetaInfo file
cf = arc.find_file("*.metainfo.xml")
if not cf:
return error_internal('The firmware file had no valid metadata')
component = appstream.Component()
try:
component.parse(str(cf.contents))
except appstream.ParseError as e:
return error_internal('The metadata could not be parsed: ' + str(e))
# parse the inf file
cf = arc.find_file("*.inf")
if not cf:
return error_internal('The firmware file had no valid inf file')
cfg = InfParser()
cfg.read_data(cf.contents)
try:
tmp = cfg.get('Version', 'DriverVer')
driver_ver = tmp.split(',')
if len(driver_ver) != 2:
return error_internal('The inf file Version:DriverVer was invalid')
except ConfigParser.NoOptionError as e:
driver_ver = None
# get the contents
fw_data = arc.find_file('*.bin')
if not fw_data:
fw_data = arc.find_file('*.rom')
if not fw_data:
fw_data = arc.find_file('*.cap')
if not fw_data:
return error_internal('No firmware found in the archive')
# update sizes
fwobj.mds[0].release_installed_size = len(fw_data.contents)
fwobj.mds[0].release_download_size = os.path.getsize(fn)
# update the descriptions
fwobj.mds[0].release_description = component.releases[0].description
fwobj.mds[0].description = component.description
if driver_ver:
fwobj.version_display = driver_ver[1]
db = LvfsDatabase(os.environ)
db_firmware = LvfsDatabaseFirmware(db)
db_firmware.update(fwobj)
return None
def device():
"""
Show all devices -- probably only useful for the admin user.
"""
# security check
if session['username'] != 'admin':
return error_permission_denied('Unable to view devices')
# get all firmware
try:
db = LvfsDatabase(os.environ)
db_firmware = LvfsDatabaseFirmware(db)
items = db_firmware.get_items()
except CursorError as e:
return error_internal(str(e))
# get all the guids we can target
devices = []
seen_guid = {}
for item in items:
for md in item.mds:
if md.guid in seen_guid:
continue
seen_guid[md.guid] = 1
devices.append(md.guid)
return render_template('devices.html', devices=devices)
def firmware_delete_force(fwid):
""" Delete a firmware entry and also delete the file from disk """
# check firmware exists in database
db = LvfsDatabase(os.environ)
db_firmware = LvfsDatabaseFirmware(db)
try:
item = db_firmware.get_item(fwid)
except CursorError as e:
return error_internal(str(e))
if not item:
return error_internal("No firmware file with hash %s exists" % fwid)
if session['username'] != 'admin' and item.qa_group != session['qa_group']:
return error_permission_denied("No QA access to %s" % fwid)
# only QA users can delete once the firmware has gone stable
if not session['qa_capability'] and item.target == 'stable':
return error_permission_denied('Unable to delete stable firmware as not QA')
# delete id from database
try:
db_firmware.remove(fwid)
except CursorError as e:
return error_internal(str(e))
# delete file(s)
for loc in [DOWNLOAD_DIR]:
path = os.path.join(loc, item.filename)
if os.path.exists(path):
os.remove(path)
# update everything
try:
metadata_update_qa_group(item.qa_group)
if item.target == 'stable':
metadata_update_targets(targets=['stable', 'testing'])
elif item.target == 'testing':
metadata_update_targets(targets=['testing'])
except NoKeyError as e:
return error_internal('Failed to sign metadata: ' + str(e))
except CursorError as e:
return error_internal('Failed to generate metadata: ' + str(e))
_event_log("Deleted firmware %s" % fwid)
return redirect(url_for('.firmware'))
def device_list():
# add devices in stable or testing
try:
db = LvfsDatabase(os.environ)
db_firmware = LvfsDatabaseFirmware(db)
items = db_firmware.get_items()
except CursorError as e:
return error_internal(str(e))
# get a sorted list of vendors
vendors = []
for item in items:
if item.target != 'stable':
continue
vendor = item.mds[0].developer_name
if vendor in vendors:
continue
vendors.append(vendor)
seen_guid = {}
mds_by_vendor = {}
for vendor in sorted(vendors):
for item in items:
if item.target != 'stable':
continue
for md in item.mds:
# only show correct vendor
if vendor != md.developer_name:
continue
# only show the newest version
if md.guid in seen_guid:
continue
seen_guid[md.guid] = 1
# add
if not vendor in mds_by_vendor:
mds_by_vendor[vendor] = []
mds_by_vendor[vendor].append(md)
return render_template('devicelist.html',
vendors=sorted(vendors),
mds_by_vendor=mds_by_vendor)
def firmware_promote(fwid, target):
"""
Promote or demote a firmware file from one target to another,
for example from testing to stable, or stable to testing.
"""
# check is QA
if not session['qa_capability']:
return error_permission_denied('Unable to promote as not QA')
# check valid
if target not in ['stable', 'testing', 'private', 'embargo']:
return error_internal("Target %s invalid" % target)
# check firmware exists in database
db = LvfsDatabase(os.environ)
db_firmware = LvfsDatabaseFirmware(db)
try:
item = db_firmware.get_item(fwid)
except CursorError as e:
return error_internal(str(e))
if session['username'] != 'admin' and item.qa_group != session['qa_group']:
return error_permission_denied("No QA access to %s" % fwid)
try:
db_firmware.set_target(fwid, target)
except CursorError as e:
return error_internal(str(e))
# set correct response code
_event_log("Moved firmware %s to %s" % (fwid, target))
# update everything
try:
metadata_update_qa_group(item.qa_group)
if target == 'stable':
metadata_update_targets(['stable', 'testing'])
elif target == 'testing':
metadata_update_targets(['testing'])
except NoKeyError as e:
return error_internal('Failed to sign metadata: ' + str(e))
except CursorError as e:
return error_internal('Failed to generate metadata: ' + str(e))
return redirect(url_for('.firmware_id', fwid=fwid))
def metadata_update_qa_group(qa_group):
""" updates metadata for a specific qa_group """
# explicit
affidavit = create_affidavit()
if qa_group:
filename = 'firmware-%s.xml.gz' % _qa_hash(qa_group)
_generate_metadata_kind(filename,
qa_group=qa_group,
affidavit=affidavit)
return
# do for all
db = LvfsDatabase(os.environ)
db_firmware = LvfsDatabaseFirmware(db)
qa_groups = db_firmware.get_qa_groups()
for qa_group in qa_groups:
filename_qa = 'firmware-%s.xml.gz' % _qa_hash(qa_group)
_generate_metadata_kind(filename_qa,
qa_group=qa_group,
affidavit=affidavit)
def metadata_update_qa_group(qa_group):
""" updates metadata for a specific qa_group """
# explicit
if qa_group:
filename = 'firmware-%s.xml.gz' % _qa_hash(qa_group)
_generate_metadata_kind(filename, qa_group=qa_group)
return [os.path.join(DOWNLOAD_DIR, filename)]
# do for all
db = LvfsDatabase(os.environ)
db_firmware = LvfsDatabaseFirmware(db)
qa_groups = db_firmware.get_qa_groups()
filenames = []
for qa_group in qa_groups:
filename_qa = 'firmware-%s.xml.gz' % _qa_hash(qa_group)
filename = _generate_metadata_kind(filename_qa, qa_group=qa_group)
filenames.append(filename)
# return all the files we have to sign
return filenames
def firmware_modify(fwid):
""" Modifies the update urgency and release notes for the update """
if request.method != 'POST':
return redirect(url_for('.firmware'))
# find firmware
try:
db = LvfsDatabase(os.environ)
db_firmware = LvfsDatabaseFirmware(db)
fwobj = db_firmware.get_item(fwid)
except CursorError as e:
return error_internal(str(e))
if not fwobj:
return error_internal("No firmware %s" % fwid)
# set new metadata values
for md in fwobj.mds:
if 'urgency' in request.form:
md.release_urgency = request.form['urgency']
if 'description' in request.form:
txt = request.form['description']
if txt.find('<p>') == -1:
txt = appstream.utils.import_description(txt)
try:
appstream.utils.validate_description(txt)
except appstream.ParseError as e:
return error_internal("Failed to parse %s: %s" % (txt, str(e)))
md.release_description = txt
# modify
try:
db_firmware.update(fwobj)
except CursorError as e:
return error_internal(str(e))
# log
_event_log('Changed update description on %s' % fwid)
return redirect(url_for('.firmware_id', fwid=fwid))
def firmware(show_all=False):
"""
Show all previsouly uploaded firmware for this user.
"""
# get all firmware
try:
db = LvfsDatabase(os.environ)
db_firmware = LvfsDatabaseFirmware(db)
items = db_firmware.get_items()
except CursorError as e:
return error_internal(str(e))
# group by the firmware name
names = {}
for item in items:
# admin can see everything
if session['username'] != 'admin':
if item.qa_group != session['qa_group']:
continue
name = item.mds[0].name
if not name in names:
names[name] = []
names[name].append(item)
# only show one version in each state
for name in sorted(names):
targets_seen = {}
for item in names[name]:
if item.target in targets_seen:
item.is_newest_in_state = False
else:
item.is_newest_in_state = True
targets_seen[item.target] = item
return render_template('firmware.html',
fw_by_name=names,
names_sorted=sorted(names),
qa_group=session['qa_group'],
show_all=show_all)
def device_guid(guid):
"""
Show information for one device, which can be seen without a valid login
"""
# get all firmware
try:
db = LvfsDatabase(os.environ)
db_firmware = LvfsDatabaseFirmware(db)
items = db_firmware.get_items()
except CursorError as e:
return error_internal(str(e))
# get all the guids we can target
firmware_items = []
for item in items:
for md in item.mds:
if md.guid != guid:
continue
firmware_items.append(item)
break
return render_template('device.html', items=firmware_items)
def firmware_id(fwid):
""" Show firmware information """
# get details about the firmware
db = LvfsDatabase(os.environ)
db_firmware = LvfsDatabaseFirmware(db)
try:
item = db_firmware.get_item(fwid)
except CursorError as e:
return error_internal(str(e))
if not item:
return error_internal('No firmware matched!')
# we can only view our own firmware, unless admin
qa_group = item.qa_group
if qa_group != session['qa_group'] and session['username'] != 'admin':
return error_permission_denied('Unable to view other vendor firmware')
if not qa_group:
embargo_url = '/downloads/firmware.xml.gz'
qa_group = 'None'
else:
embargo_url = '/downloads/firmware-%s.xml.gz' % _qa_hash(qa_group)
db = LvfsDatabase(os.environ)
db_clients = LvfsDatabaseClients(db)
cnt_fn = db_clients.get_firmware_count_filename(item.filename)
data_fw = db_clients.get_stats_for_fn(12, 30, item.filename)
return render_template('firmware-details.html',
fw=item,
qa_capability=session['qa_capability'],
orig_filename='-'.join(item.filename.split('-')[1:]),
embargo_url=embargo_url,
qa_group=qa_group,
cnt_fn=cnt_fn,
fwid=fwid,
graph_labels=_get_chart_labels_months()[::-1],
graph_data=data_fw[::-1])
def _generate_metadata_kind(filename, targets=None, qa_group=None, affidavit=None):
""" Generates AppStream metadata of a specific kind """
db = LvfsDatabase(os.environ)
db_firmware = LvfsDatabaseFirmware(db)
items = db_firmware.get_items()
store = appstream.Store('lvfs')
for item in items:
# filter
if item.target == 'private':
continue
if targets and item.target not in targets:
continue
if qa_group and qa_group != item.qa_group:
continue
# add each component
for md in item.mds:
component = appstream.Component()
component.id = md.cid
component.kind = 'firmware'
component.name = md.name
component.summary = md.summary
component.description = md.description
if md.url_homepage:
component.urls['homepage'] = md.url_homepage
component.metadata_license = md.metadata_license
component.project_license = md.project_license
component.developer_name = md.developer_name
# add provide
if md.guid:
prov = appstream.Provide()
prov.kind = 'firmware-flashed'
prov.value = md.guid
component.add_provide(prov)
# add release
if md.version:
rel = appstream.Release()
rel.version = md.version
rel.description = md.release_description
if md.release_timestamp:
rel.timestamp = md.release_timestamp
rel.checksums = []
rel.location = 'https://secure-lvfs.rhcloud.com/downloads/' + item.filename
rel.size_installed = md.release_installed_size
rel.size_download = md.release_download_size
rel.urgency = md.release_urgency
component.add_release(rel)
# add container checksum
if md.checksum_container:
csum = appstream.Checksum()
csum.target = 'container'
csum.value = md.checksum_container
csum.filename = item.filename
rel.add_checksum(csum)
# add content checksum
if md.checksum_contents:
csum = appstream.Checksum()
csum.target = 'content'
csum.value = md.checksum_contents
csum.filename = md.filename_contents
rel.add_checksum(csum)
# add screenshot
if md.screenshot_caption:
ss = appstream.Screenshot()
ss.caption = md.screenshot_caption
if md.screenshot_url:
im = appstream.Image()
im.url = md.screenshot_url
ss.add_image(im)
component.add_screenshot(ss)
# add component
store.add(component)
# dump to file
if not os.path.exists(DOWNLOAD_DIR):
os.mkdir(DOWNLOAD_DIR)
filename = os.path.join(DOWNLOAD_DIR, filename)
store.to_file(filename)
# upload to the CDN
blob = open(filename, 'rb').read()
_upload_to_cdn(filename, blob)
# generate and upload the detached signature
if affidavit:
blob_asc = affidavit.create(blob)
_upload_to_cdn(filename + '.asc', blob_asc)
def upload():
""" Upload a .cab file to the LVFS service """
# only accept form data
if request.method != 'POST':
return redirect(url_for('.index'))
# not correct parameters
if not 'target' in request.form:
return error_internal('No target')
if not 'file' in request.files:
return error_internal('No file')
# can the user upload directly to stable
if request.form['target'] in ['stable', 'testing']:
if not session['qa_capability']:
return error_permission_denied('Unable to upload to this target as not QA user')
# check size < 50Mb
fileitem = request.files['file']
if not fileitem:
return error_internal('No file object')
data = fileitem.read()
if len(data) > 50000000:
return error_internal('File too large, limit is 50Mb', 413)
if len(data) == 0:
return error_internal('File has no content')
if len(data) < 1024:
return error_internal('File too small, mimimum is 1k')
# check the file does not already exist
db = LvfsDatabase(os.environ)
db_firmware = LvfsDatabaseFirmware(db)
fwid = hashlib.sha1(data).hexdigest()
try:
item = db_firmware.get_item(fwid)
except CursorError as e:
return error_internal(str(e))
if item:
return error_internal("A firmware file with hash %s already exists" % fwid, 422)
# parse the file
arc = cabarchive.CabArchive()
try:
if os.path.exists(CABEXTRACT_CMD):
arc.set_decompressor(CABEXTRACT_CMD)
arc.parse(data)
except cabarchive.CorruptionError as e:
return error_internal('Invalid file type: %s' % str(e), 415)
except cabarchive.NotSupportedError as e:
return error_internal('The file is unsupported: %s' % str(e), 415)
# check .inf exists
fw_version_inf = None
fw_version_display_inf = None
cf = arc.find_file("*.inf")
if cf:
if cf.contents.find('FIXME') != -1:
return error_internal("The inf file was not complete; "
"Any FIXME text must be replaced with the correct values.")
# check .inf file is valid
cfg = InfParser()
cfg.read_data(cf.contents)
try:
tmp = cfg.get('Version', 'Class')
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e:
return error_internal('The inf file Version:Class was missing')
if tmp != 'Firmware':
return error_internal('The inf file Version:Class was invalid')
try:
tmp = cfg.get('Version', 'ClassGuid')
except ConfigParser.NoOptionError as e:
return error_internal('The inf file Version:ClassGuid was missing')
if tmp != '{f2e7dd72-6468-4e36-b6f1-6488f42c1b52}':
return error_internal('The inf file Version:ClassGuid was invalid')
try:
tmp = cfg.get('Version', 'DriverVer')
fw_version_display_inf = tmp.split(',')
if len(fw_version_display_inf) != 2:
return error_internal('The inf file Version:DriverVer was invalid')
except ConfigParser.NoOptionError as e:
pass
# this is optional, but if supplied must match the version in the XML
# -- also note this will not work with multi-firmware .cab files
try:
fw_version_inf = cfg.get('Firmware_AddReg', 'HKR->FirmwareVersion')
if fw_version_inf.startswith('0x'):
fw_version_inf = str(int(fw_version_inf[2:], 16))
if fw_version_inf == '0':
fw_version_inf = None
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e:
pass
# check metainfo exists
cfs = arc.find_files("*.metainfo.xml")
if len(cfs) == 0:
return error_internal('The firmware file had no .metadata.xml files')
# parse each MetaInfo file
apps = []
for cf in cfs:
component = appstream.Component()
try:
component.parse(str(cf.contents))
component.validate()
except appstream.ParseError as e:
return error_internal('The metadata %s could not be parsed: %s' % (cf, str(e)))
except appstream.ValidationError as e:
return error_internal('The metadata %s file did not validate: %s' % (cf, str(e)))
# check the file does not have any missing request.form
if cf.contents.find('FIXME') != -1:
return error_internal("The metadata file was not complete; "
"Any FIXME text must be replaced with the correct values.")
# check the firmware provides something
if len(component.provides) == 0:
return error_internal("The metadata file did not provide any GUID.")
if len(component.releases) == 0:
return error_internal("The metadata file did not provide any releases.")
# check the inf file matches up with the .xml file
if fw_version_inf and fw_version_inf != component.releases[0].version:
return error_internal("The inf Firmware_AddReg[HKR->FirmwareVersion] "
"'%s' did not match the metainfo.xml value '%s'."
% (fw_version_inf, component.releases[0].version))
# check the guid and version does not already exist
try:
items = db_firmware.get_items()
except CursorError as e:
return error_internal(str(e))
for item in items:
for md in item.mds:
if md.guid == component.provides[0].value and md.version == component.releases[0].version:
return error_internal("A firmware file for this version already exists", 422)
# check the ID hasn't been reused by a different GUID
for item in items:
for md in item.mds:
if md.cid == component.id and not md.guid == component.provides[0].value:
return error_internal("The %s ID has already been used by GUID %s" % (md.cid, md.guid), 422)
# add to array
apps.append(component)
# only save if we passed all tests
basename = os.path.basename(fileitem.filename)
new_filename = fwid + '-' + basename
# add these after parsing in case multiple components use the same file
asc_files = {}
# fix up the checksums and add the detached signature
for component in apps:
# ensure there's always a container checksum
release = component.releases[0]
csum = release.get_checksum_by_target('content')
if not csum:
csum = appstream.Checksum()
csum.target = 'content'
csum.filename = 'firmware.bin'
component.releases[0].add_checksum(csum)
# get the contents checksum
fw_data = arc.find_file(csum.filename)
if not fw_data:
return error_internal('No %s found in the archive' % csum.filename)
csum.kind = 'sha1'
csum.value = hashlib.sha1(fw_data.contents).hexdigest()
# set the sizes
release.size_installed = len(fw_data.contents)
release.size_download = len(data)
# add the detached signature if not already signed
sig_data = arc.find_file(csum.filename + ".asc")
if not sig_data:
if not csum.filename in asc_files:
try:
affidavit = create_affidavit()
except NoKeyError as e:
return error_internal('Failed to sign archive: ' + str(e))
cff = cabarchive.CabFile(fw_data.filename + '.asc',
affidavit.create(fw_data.contents))
asc_files[csum.filename] = cff
else:
# check this file is signed by something we trust
try:
affidavit = create_affidavit()
affidavit.verify(fw_data.contents)
except NoKeyError as e:
return error_internal('Failed to verify archive: ' + str(e))
# add all the .asc files to the archive
for key in asc_files:
arc.add_file(asc_files[key])
# export the new archive and get the checksum
cab_data = arc.save(compressed=True)
checksum_container = hashlib.sha1(cab_data).hexdigest()
# dump to a file
if not os.path.exists(DOWNLOAD_DIR):
os.mkdir(DOWNLOAD_DIR)
fn = os.path.join(DOWNLOAD_DIR, new_filename)
open(fn, 'wb').write(cab_data)
# dump to the CDN
_upload_to_cdn(new_filename, StringIO(cab_data))
# create parent firmware object
target = request.form['target']
fwobj = LvfsFirmware()
fwobj.qa_group = session['qa_group']
fwobj.addr = _get_client_address()
fwobj.filename = new_filename
fwobj.fwid = fwid
fwobj.target = target
if fw_version_display_inf:
fwobj.version_display = fw_version_display_inf[1]
# create child metadata object for the component
for component in apps:
md = LvfsFirmwareMd()
md.fwid = fwid
md.cid = component.id
md.name = component.name
md.summary = component.summary
md.developer_name = component.developer_name
md.metadata_license = component.metadata_license
md.project_license = component.project_license
md.url_homepage = component.urls['homepage']
md.description = component.description
md.checksum_container = checksum_container
# from the provide
prov = component.provides[0]
md.guid = prov.value
# from the release
rel = component.releases[0]
md.version = rel.version
md.release_description = rel.description
md.release_timestamp = rel.timestamp
md.release_installed_size = rel.size_installed
md.release_download_size = rel.size_download
md.release_urgency = rel.urgency
# from the first screenshot
if len(component.screenshots) > 0:
ss = component.screenshots[0]
if ss.caption:
md.screenshot_caption = ss.caption
if len(ss.images) > 0:
im = ss.images[0]
if im.url:
md.screenshot_url = im.url
# from the content checksum
csum = component.releases[0].get_checksum_by_target('content')
md.checksum_contents = csum.value
md.filename_contents = csum.filename
fwobj.mds.append(md)
# add to database
try:
db_firmware.add(fwobj)
except CursorError as e:
return error_internal(str(e))
# set correct response code
_event_log("Uploaded file %s to %s" % (new_filename, target))
# ensure up to date
try:
if target != 'private':
metadata_update_qa_group(fwobj.qa_group)
if target == 'stable':
metadata_update_targets(['stable', 'testing'])
elif target == 'testing':
metadata_update_targets(['testing'])
except NoKeyError as e:
return error_internal('Failed to sign metadata: ' + str(e))
except CursorError as e:
return error_internal('Failed to generate metadata: ' + str(e))
return redirect(url_for('.firmware_id', fwid=fwid))