def _repair_ts():
# fix any timestamps that are incorrect
for md in db.session.query(Component).filter(
Component.release_timestamp < 1980):
fn = _get_absolute_path(md.fw)
if not os.path.exists(fn):
continue
print(fn, md.release_timestamp)
try:
ufile = UploadedFile(is_strict=False)
for cat in db.session.query(Category):
ufile.category_map[cat.value] = cat.category_id
for pro in db.session.query(Protocol):
ufile.protocol_map[pro.value] = pro.protocol_id
with open(fn, 'rb') as f:
ufile.parse(os.path.basename(fn), f.read())
except MetadataInvalid as e:
print('failed to parse file: {}'.format(str(e)))
continue
for md_local in ufile.fw.mds:
if md_local.appstream_id == md.appstream_id:
print('repairing timestamp from {} to {}'.format(
md.release_timestamp, md_local.release_timestamp))
md.release_timestamp = md_local.release_timestamp
md.fw.mark_dirty()
# all done
db.session.commit()
def create_metadata(archive_dir, basename, metadata_fn):
# process all archives in directory
fws = []
print('Searching %s' % archive_dir)
for root, dirs, files in os.walk(archive_dir): #pylint: disable=unused-variable
for filename in files:
if not filename.endswith('.cab'):
continue
print('Processing %s...' % filename)
ufile = UploadedFile(is_strict=False)
try:
with open(os.path.join(root, filename), 'rb') as f:
ufile.parse(filename, f.read(), use_hashed_prefix=False)
except MetadataInvalid as e:
print('Failed to parse {}: {}'.format(filename, e))
else:
fws.append(ufile.fw)
# write metadata
print('Writing %s' % metadata_fn)
xml = _generate_metadata_kind(fws,
firmware_baseuri="%s/" % basename,
local=True)
with open(metadata_fn, 'wb') as f:
f.write(xml)
def test_metainfo_missing(self):
cabarchive = CabArchive()
cabarchive['firmware.bin'] = _get_valid_firmware()
with self.assertRaises(MetadataInvalid):
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', cabarchive.save())
def test_invalid_type(self):
cabarchive = CabArchive()
cabarchive['firmware.bin'] = _get_valid_firmware()
with self.assertRaises(FileNotSupported):
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.doc', cabarchive.save())
def test_autogenerated(self):
cabarchive = CabArchive()
cabarchive['0x0962_nonsecure.bin'] = _get_valid_firmware()
cabarchive['NVM0.metainfo.xml'] = _get_generated_metainfo()
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', cabarchive.save())
def _repair():
# fix any timestamps that are incorrect
for md in db.session.query(Component).filter(
Component.release_timestamp < 1980):
fn = _get_absolute_path(md.fw)
if not os.path.exists(fn):
continue
print(fn, md.release_timestamp)
try:
ufile = UploadedFile(is_strict=False)
for cat in db.session.query(Category):
ufile.category_map[cat.value] = cat.category_id
for pro in db.session.query(Protocol):
ufile.protocol_map[pro.value] = pro.protocol_id
with open(fn, 'rb') as f:
ufile.parse(os.path.basename(fn), f.read())
except MetadataInvalid as e:
print('failed to parse file: {}'.format(str(e)))
continue
for md_local in ufile.fw.mds:
if md_local.appstream_id == md.appstream_id:
print('repairing timestamp from {} to {}'.format(
md.release_timestamp, md_local.release_timestamp))
md.release_timestamp = md_local.release_timestamp
md.fw.mark_dirty()
# fix all the checksums and file sizes
for fw in db.session.query(Firmware):
try:
with open(fw.filename_absolute, 'rb') as f:
checksum_signed_sha1 = hashlib.sha1(f.read()).hexdigest()
if checksum_signed_sha1 != fw.checksum_signed_sha1:
print('repairing checksum from {} to {}'.format(
fw.checksum_signed_sha1, checksum_signed_sha1))
fw.checksum_signed_sha1 = checksum_signed_sha1
fw.mark_dirty()
checksum_signed_sha256 = hashlib.sha256(f.read()).hexdigest()
if checksum_signed_sha256 != fw.checksum_signed_sha256:
print('repairing checksum from {} to {}'.format(
fw.checksum_signed_sha256, checksum_signed_sha256))
fw.checksum_signed_sha256 = checksum_signed_sha256
fw.mark_dirty()
for md in fw.mds:
sz = os.path.getsize(fw.filename_absolute)
if sz != md.release_download_size:
print('repairing size from {} to {}'.format(
md.release_download_size, sz))
md.release_download_size = sz
md.fw.mark_dirty()
except FileNotFoundError as _:
pass
# ensure the test has been added for the firmware type
if not fw.is_deleted:
ploader.ensure_test_for_fw(fw)
# all done
db.session.commit()
def test_release_date(self):
cabarchive = CabArchive()
cabarchive['firmware.bin'] = _get_valid_firmware()
cabarchive['firmware.metainfo.xml'] = _get_alternate_metainfo()
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', cabarchive.save())
self.assertEqual(ufile.fw.mds[0].release_timestamp, 1562025600)
def test_invalid_release_tag(self):
cabarchive = CabArchive()
cabarchive['firmware.bin'] = _get_valid_firmware()
cabarchive['firmware.metainfo.xml'] = _get_valid_metainfo(release_tag='foo')
with self.assertRaises(MetadataInvalid):
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', cabarchive.save())
def test_metainfo_invalid(self):
cabarchive = CabArchive()
cabarchive['firmware.bin'] = _get_valid_firmware()
cabarchive['firmware.metainfo.xml'] = CabFile(b'<compoXXXXnent/>')
with self.assertRaises(MetadataInvalid):
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', cabarchive.save())
def test_metadata(self):
cabarchive = CabArchive()
cabarchive['firmware.bin'] = _get_valid_firmware()
cabarchive['firmware.metainfo.xml'] = _get_valid_metainfo()
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', cabarchive.save())
self.assertTrue(ufile.fw.mds[0].inhibit_download)
self.assertTrue(ufile.fw.mds[0].verfmt.value == 'quad')
def test_invalid_bom(self):
cabarchive = CabArchive()
cabarchive['firmware.bin'] = _get_valid_firmware()
cabarchive['firmware.metainfo.xml'] = CabFile(b'\xEF\xBB\xBF<?xml version="1.0" encoding="UTF-8"?>\n'
b'<component type="firmware"/>\n')
with self.assertRaises(MetadataInvalid):
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', cabarchive.save())
def test_valid_path_back(self):
cabarchive = CabArchive()
cabarchive['DriverPackage\\firmware.bin'] = _get_valid_firmware()
cabarchive['DriverPackage\\firmware.metainfo.xml'] = _get_valid_metainfo()
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', cabarchive.save())
cabarchive2 = ufile.cabarchive_repacked
self.assertIsNotNone(cabarchive2['firmware.bin'])
self.assertIsNotNone(cabarchive2['firmware.metainfo.xml'])
def test_invalid_xml_header(self):
cabarchive = CabArchive()
cabarchive['firmware.bin'] = _get_valid_firmware()
cabarchive['firmware.metainfo.xml'] = CabFile(b'<!-- Copyright 2015 Richard Hughes <*****@*****.**> -->\n'
b'<?xml version="1.0" encoding="UTF-8"?>\n'
b'<component type="firmware"/>\n')
with self.assertRaises(MetadataInvalid):
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', cabarchive.save())
def test_release_mentions_file(self):
cabarchive = CabArchive()
cabarchive['firmware.bin'] = _get_valid_firmware()
cabarchive['README.txt'] = _get_valid_firmware()
cabarchive['firmware.metainfo.xml'] = \
_get_valid_metainfo(release_description='See README.txt for details.')
with self.assertRaises(MetadataInvalid):
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', cabarchive.save())
def test_valid_zipfile(self):
imz = InMemoryZip()
imz.append('DriverPackage\\firmware.bin', _get_valid_firmware().buf)
imz.append('DriverPackage\\firmware.metainfo.xml', _get_valid_metainfo().buf)
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.zip', imz.read())
cabarchive2 = ufile.cabarchive_repacked
self.assertIsNotNone(cabarchive2['firmware.bin'])
self.assertIsNotNone(cabarchive2['firmware.metainfo.xml'])
def test_valid_with_ignored_inf(self):
cabarchive = CabArchive()
cabarchive['firmware.bin'] = _get_valid_firmware()
cabarchive['firmware.metainfo.xml'] = _get_valid_metainfo(enable_inf_parsing=False)
cabarchive['firmware.inf'] = CabFile(b'fubar')
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', cabarchive.save())
cabarchive2 = ufile.cabarchive_repacked
self.assertIsNotNone(cabarchive2['firmware.bin'])
self.assertIsNotNone(cabarchive2['firmware.metainfo.xml'])
self.assertIsNotNone(cabarchive2['firmware.inf'])
def test_extra_files(self):
cabarchive = CabArchive()
cabarchive['firmware.bin'] = _get_valid_firmware()
cabarchive['firmware.metainfo.xml'] = _get_valid_metainfo()
cabarchive['README.txt'] = CabFile(b'fubar')
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', cabarchive.save())
cabarchive2 = ufile.cabarchive_repacked
self.assertIsNotNone(cabarchive2['firmware.bin'])
self.assertIsNotNone(cabarchive2['firmware.metainfo.xml'])
with self.assertRaises(KeyError):
self.assertIsNotNone(cabarchive2['README.txt'])
def test_multiple_metainfo_same_firmware(self):
cabarchive = CabArchive()
cabarchive['firmware.bin'] = _get_valid_firmware()
cabarchive['firmware1.metainfo.xml'] = _get_valid_metainfo()
cabarchive['firmware2.metainfo.xml'] = _get_valid_metainfo()
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', cabarchive.save())
cabarchive2 = ufile.cabarchive_repacked
self.assertIsNotNone(cabarchive2['firmware.bin'])
self.assertIsNotNone(cabarchive2['firmware1.metainfo.xml'])
self.assertIsNotNone(cabarchive2['firmware2.metainfo.xml'])
def create_metadata(archive_dir, basename, metadata_fn):
# process all archives in directory
fws = []
print('Searching %s' % archive_dir)
for root, dirs, files in os.walk(archive_dir): #pylint: disable=unused-variable
for filename in files:
if not filename.endswith('.cab'):
continue
print('Processing %s...' % filename)
ufile = UploadedFile()
with open(os.path.join(root, filename), 'r') as f:
ufile.parse(filename, f.read(), use_hashed_prefix=False)
fws.append(ufile.fw)
# write metadata
print('Writing %s' % metadata_fn)
_generate_metadata_kind(metadata_fn,
fws,
firmware_baseuri="%s/" % basename,
local=True)
def test_src_empty(self):
with self.assertRaises(FileTooSmall):
ufile = UploadedFile()
_add_version_formats(ufile)
ufile.parse('foo.cab', '')
self.assertEqual(ufile.fwupd_min_version, '0.8.0')
def _upload_firmware():
# verify the user can upload
if not _user_can_upload(g.user):
flash('User has not signed legal agreement', 'danger')
return redirect(url_for('main.route_dashboard'))
# used a custom vendor_id
if 'vendor_id' in request.form:
try:
vendor_id = int(request.form['vendor_id'])
except ValueError as e:
flash('Failed to upload file: Specified vendor ID %s invalid' % request.form['vendor_id'], 'warning')
return redirect(url_for('upload.route_firmware'))
vendor = db.session.query(Vendor).filter(Vendor.vendor_id == vendor_id).first()
if not vendor:
flash('Failed to upload file: Specified vendor ID not found', 'warning')
return redirect(url_for('upload.route_firmware'))
else:
vendor = g.user.vendor
# security check
if not vendor.check_acl('@upload'):
flash('Permission denied: Failed to upload file for vendor: '
'User with vendor %s cannot upload to vendor %s' %
(g.user.vendor.group_id, vendor.group_id), 'warning')
return redirect(url_for('upload.route_firmware'))
# not correct parameters
if not 'target' in request.form:
return _error_internal('No target')
if not 'file' in request.files:
return _error_internal('No file')
if request.form['target'] not in ['private', 'embargo', 'testing']:
return _error_internal('Target not valid')
# find remote, creating if required
remote_name = request.form['target']
if remote_name == 'embargo':
remote = vendor.remote
else:
remote = db.session.query(Remote).filter(Remote.name == remote_name).first()
if not remote:
return _error_internal('No remote for target %s' % remote_name)
# if the vendor has uploaded a lot of firmware don't start changing the rules
is_strict = len(vendor.fws) < 500
# load in the archive
fileitem = request.files['file']
if not fileitem:
return _error_internal('No file object')
try:
ufile = UploadedFile(is_strict=is_strict)
for cat in db.session.query(Category):
ufile.category_map[cat.value] = cat.category_id
for pro in db.session.query(Protocol):
ufile.protocol_map[pro.value] = pro.protocol_id
for verfmt in db.session.query(Verfmt):
ufile.version_formats[verfmt.value] = verfmt
ufile.parse(os.path.basename(fileitem.filename), fileitem.read())
except (FileTooLarge, FileTooSmall, FileNotSupported, MetadataInvalid) as e:
flash('Failed to upload file: ' + str(e), 'danger')
return redirect(request.url)
# check the file does not already exist
fw = db.session.query(Firmware)\
.filter(or_(Firmware.checksum_upload_sha1 == ufile.fw.checksum_upload_sha1,
Firmware.checksum_upload_sha256 == ufile.fw.checksum_upload_sha256)).first()
if fw:
if fw.check_acl('@view'):
flash('Failed to upload file: A file with hash %s already exists' % fw.checksum_upload_sha1, 'warning')
return redirect('/lvfs/firmware/%s' % fw.firmware_id)
flash('Failed to upload file: Another user has already uploaded this firmware', 'warning')
return redirect(url_for('upload.route_firmware'))
# check the guid and version does not already exist
fws = db.session.query(Firmware).all()
fws_already_exist = []
for md in ufile.fw.mds:
provides_value = md.guids[0].value
fw = _filter_fw_by_id_guid_version(fws,
md.appstream_id,
provides_value,
md.version)
if fw:
fws_already_exist.append(fw)
# all the components existed, so build an error out of all the versions
if len(fws_already_exist) == len(ufile.fw.mds):
if g.user.check_acl('@robot') and 'auto-delete' in request.form:
for fw in fws_already_exist:
if fw.remote.is_public:
flash('Firmware {} cannot be autodeleted as is in remote {}'.format(
fw.firmware_id, fw.remote.name), 'danger')
return redirect(url_for('upload.route_firmware'))
if fw.user.user_id != g.user.user_id:
flash('Firmware was not uploaded by this user', 'danger')
return redirect(url_for('upload.route_firmware'))
for fw in fws_already_exist:
flash('Firmware %i was auto-deleted due to robot upload' % fw.firmware_id)
_firmware_delete(fw)
else:
versions_for_display = []
for fw in fws_already_exist:
for md in fw.mds:
if not md.version_display in versions_for_display:
versions_for_display.append(md.version_display)
flash('Failed to upload file: A firmware file for this device with '
'version %s already exists' % ','.join(versions_for_display), 'danger')
return redirect('/lvfs/firmware/%s' % fw.firmware_id)
# check if the file dropped a GUID previously supported
for umd in ufile.fw.mds:
new_guids = [guid.value for guid in umd.guids]
for md in db.session.query(Component).\
filter(Component.appstream_id == umd.appstream_id):
if md.fw.is_deleted:
continue
for old_guid in [guid.value for guid in md.guids]:
if old_guid in new_guids:
continue
fw_str = str(md.fw.firmware_id)
if g.user.check_acl('@qa') or g.user.check_acl('@robot'):
flash('Firmware drops GUID {} previously supported '
'in firmware {}'.format(old_guid, fw_str), 'warning')
else:
flash('Firmware would drop GUID {} previously supported '
'in firmware {}'.format(old_guid, fw_str), 'danger')
return redirect(request.url)
# allow plugins to copy any extra files from the source archive
for cffile in ufile.cabarchive_upload.values():
ploader.archive_copy(ufile.cabarchive_repacked, cffile)
# allow plugins to add files
ploader.archive_finalize(ufile.cabarchive_repacked,
_get_plugin_metadata_for_uploaded_file(ufile))
# dump to a file
download_dir = app.config['DOWNLOAD_DIR']
if not os.path.exists(download_dir):
os.mkdir(download_dir)
fn = os.path.join(download_dir, ufile.fw.filename)
cab_data = ufile.cabarchive_repacked.save(compress=True)
with open(fn, 'wb') as f:
f.write(cab_data)
# create parent firmware object
settings = _get_settings()
target = request.form['target']
fw = ufile.fw
fw.vendor = vendor
fw.user = g.user
fw.addr = _get_client_address()
fw.remote = remote
fw.checksum_signed_sha1 = hashlib.sha1(cab_data).hexdigest()
fw.checksum_signed_sha256 = hashlib.sha256(cab_data).hexdigest()
fw.is_dirty = True
fw.failure_minimum = settings['default_failure_minimum']
fw.failure_percentage = settings['default_failure_percentage']
# fix name
for md in fw.mds:
name_fixed = _fix_component_name(md.name, md.developer_name_display)
if name_fixed != md.name:
flash('Fixed component name from "%s" to "%s"' % (md.name, name_fixed), 'warning')
md.name = name_fixed
# verify each component has a version format
for md in fw.mds:
if not md.verfmt_with_fallback:
flash('Component {} does not have required LVFS::VersionFormat'.\
format(md.appstream_id), 'warning')
# add to database
fw.events.append(FirmwareEvent(remote_id=remote.remote_id, user_id=g.user.user_id))
db.session.add(fw)
db.session.commit()
# ensure the test has been added for the firmware type
ploader.ensure_test_for_fw(fw)
# send out emails to anyone interested
for u in fw.get_possible_users_to_email:
if u == g.user:
continue
if u.get_action('notify-upload-vendor') and u.vendor == fw.vendor:
send_email("[LVFS] Firmware has been uploaded",
u.email_address,
render_template('email-firmware-uploaded.txt',
user=u, user_upload=g.user, fw=fw))
elif u.get_action('notify-upload-affiliate'):
send_email("[LVFS] Firmware has been uploaded by affiliate",
u.email_address,
render_template('email-firmware-uploaded.txt',
user=u, user_upload=g.user, fw=fw))
flash('Uploaded file %s to %s' % (ufile.fw.filename, target), 'info')
# invalidate
if target == 'embargo':
remote.is_dirty = True
g.user.vendor.remote.is_dirty = True
db.session.commit()
return redirect(url_for('firmware.route_show', firmware_id=fw.firmware_id))
