def _repair_ts(): # fix any timestamps that are incorrect for md in db.session.query(Component).filter( Component.release_timestamp < 1980): fn = _get_absolute_path(md.fw) if not os.path.exists(fn): continue print(fn, md.release_timestamp) try: ufile = UploadedFile(is_strict=False) for cat in db.session.query(Category): ufile.category_map[cat.value] = cat.category_id for pro in db.session.query(Protocol): ufile.protocol_map[pro.value] = pro.protocol_id with open(fn, 'rb') as f: ufile.parse(os.path.basename(fn), f.read()) except MetadataInvalid as e: print('failed to parse file: {}'.format(str(e))) continue for md_local in ufile.fw.mds: if md_local.appstream_id == md.appstream_id: print('repairing timestamp from {} to {}'.format( md.release_timestamp, md_local.release_timestamp)) md.release_timestamp = md_local.release_timestamp md.fw.mark_dirty() # all done db.session.commit()
def create_metadata(archive_dir, basename, metadata_fn): # process all archives in directory fws = [] print('Searching %s' % archive_dir) for root, dirs, files in os.walk(archive_dir): #pylint: disable=unused-variable for filename in files: if not filename.endswith('.cab'): continue print('Processing %s...' % filename) ufile = UploadedFile(is_strict=False) try: with open(os.path.join(root, filename), 'rb') as f: ufile.parse(filename, f.read(), use_hashed_prefix=False) except MetadataInvalid as e: print('Failed to parse {}: {}'.format(filename, e)) else: fws.append(ufile.fw) # write metadata print('Writing %s' % metadata_fn) xml = _generate_metadata_kind(fws, firmware_baseuri="%s/" % basename, local=True) with open(metadata_fn, 'wb') as f: f.write(xml)
def test_metainfo_missing(self): cabarchive = CabArchive() cabarchive['firmware.bin'] = _get_valid_firmware() with self.assertRaises(MetadataInvalid): ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', cabarchive.save())
def test_invalid_type(self): cabarchive = CabArchive() cabarchive['firmware.bin'] = _get_valid_firmware() with self.assertRaises(FileNotSupported): ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.doc', cabarchive.save())
def test_autogenerated(self): cabarchive = CabArchive() cabarchive['0x0962_nonsecure.bin'] = _get_valid_firmware() cabarchive['NVM0.metainfo.xml'] = _get_generated_metainfo() ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', cabarchive.save())
def _repair(): # fix any timestamps that are incorrect for md in db.session.query(Component).filter( Component.release_timestamp < 1980): fn = _get_absolute_path(md.fw) if not os.path.exists(fn): continue print(fn, md.release_timestamp) try: ufile = UploadedFile(is_strict=False) for cat in db.session.query(Category): ufile.category_map[cat.value] = cat.category_id for pro in db.session.query(Protocol): ufile.protocol_map[pro.value] = pro.protocol_id with open(fn, 'rb') as f: ufile.parse(os.path.basename(fn), f.read()) except MetadataInvalid as e: print('failed to parse file: {}'.format(str(e))) continue for md_local in ufile.fw.mds: if md_local.appstream_id == md.appstream_id: print('repairing timestamp from {} to {}'.format( md.release_timestamp, md_local.release_timestamp)) md.release_timestamp = md_local.release_timestamp md.fw.mark_dirty() # fix all the checksums and file sizes for fw in db.session.query(Firmware): try: with open(fw.filename_absolute, 'rb') as f: checksum_signed_sha1 = hashlib.sha1(f.read()).hexdigest() if checksum_signed_sha1 != fw.checksum_signed_sha1: print('repairing checksum from {} to {}'.format( fw.checksum_signed_sha1, checksum_signed_sha1)) fw.checksum_signed_sha1 = checksum_signed_sha1 fw.mark_dirty() checksum_signed_sha256 = hashlib.sha256(f.read()).hexdigest() if checksum_signed_sha256 != fw.checksum_signed_sha256: print('repairing checksum from {} to {}'.format( fw.checksum_signed_sha256, checksum_signed_sha256)) fw.checksum_signed_sha256 = checksum_signed_sha256 fw.mark_dirty() for md in fw.mds: sz = os.path.getsize(fw.filename_absolute) if sz != md.release_download_size: print('repairing size from {} to {}'.format( md.release_download_size, sz)) md.release_download_size = sz md.fw.mark_dirty() except FileNotFoundError as _: pass # ensure the test has been added for the firmware type if not fw.is_deleted: ploader.ensure_test_for_fw(fw) # all done db.session.commit()
def test_release_date(self): cabarchive = CabArchive() cabarchive['firmware.bin'] = _get_valid_firmware() cabarchive['firmware.metainfo.xml'] = _get_alternate_metainfo() ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', cabarchive.save()) self.assertEqual(ufile.fw.mds[0].release_timestamp, 1562025600)
def test_invalid_release_tag(self): cabarchive = CabArchive() cabarchive['firmware.bin'] = _get_valid_firmware() cabarchive['firmware.metainfo.xml'] = _get_valid_metainfo(release_tag='foo') with self.assertRaises(MetadataInvalid): ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', cabarchive.save())
def test_metainfo_invalid(self): cabarchive = CabArchive() cabarchive['firmware.bin'] = _get_valid_firmware() cabarchive['firmware.metainfo.xml'] = CabFile(b'<compoXXXXnent/>') with self.assertRaises(MetadataInvalid): ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', cabarchive.save())
def test_metadata(self): cabarchive = CabArchive() cabarchive['firmware.bin'] = _get_valid_firmware() cabarchive['firmware.metainfo.xml'] = _get_valid_metainfo() ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', cabarchive.save()) self.assertTrue(ufile.fw.mds[0].inhibit_download) self.assertTrue(ufile.fw.mds[0].verfmt.value == 'quad')
def test_invalid_bom(self): cabarchive = CabArchive() cabarchive['firmware.bin'] = _get_valid_firmware() cabarchive['firmware.metainfo.xml'] = CabFile(b'\xEF\xBB\xBF<?xml version="1.0" encoding="UTF-8"?>\n' b'<component type="firmware"/>\n') with self.assertRaises(MetadataInvalid): ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', cabarchive.save())
def test_valid_path_back(self): cabarchive = CabArchive() cabarchive['DriverPackage\\firmware.bin'] = _get_valid_firmware() cabarchive['DriverPackage\\firmware.metainfo.xml'] = _get_valid_metainfo() ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', cabarchive.save()) cabarchive2 = ufile.cabarchive_repacked self.assertIsNotNone(cabarchive2['firmware.bin']) self.assertIsNotNone(cabarchive2['firmware.metainfo.xml'])
def test_invalid_xml_header(self): cabarchive = CabArchive() cabarchive['firmware.bin'] = _get_valid_firmware() cabarchive['firmware.metainfo.xml'] = CabFile(b'<!-- Copyright 2015 Richard Hughes <*****@*****.**> -->\n' b'<?xml version="1.0" encoding="UTF-8"?>\n' b'<component type="firmware"/>\n') with self.assertRaises(MetadataInvalid): ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', cabarchive.save())
def test_release_mentions_file(self): cabarchive = CabArchive() cabarchive['firmware.bin'] = _get_valid_firmware() cabarchive['README.txt'] = _get_valid_firmware() cabarchive['firmware.metainfo.xml'] = \ _get_valid_metainfo(release_description='See README.txt for details.') with self.assertRaises(MetadataInvalid): ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', cabarchive.save())
def test_valid_zipfile(self): imz = InMemoryZip() imz.append('DriverPackage\\firmware.bin', _get_valid_firmware().buf) imz.append('DriverPackage\\firmware.metainfo.xml', _get_valid_metainfo().buf) ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.zip', imz.read()) cabarchive2 = ufile.cabarchive_repacked self.assertIsNotNone(cabarchive2['firmware.bin']) self.assertIsNotNone(cabarchive2['firmware.metainfo.xml'])
def test_valid_with_ignored_inf(self): cabarchive = CabArchive() cabarchive['firmware.bin'] = _get_valid_firmware() cabarchive['firmware.metainfo.xml'] = _get_valid_metainfo(enable_inf_parsing=False) cabarchive['firmware.inf'] = CabFile(b'fubar') ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', cabarchive.save()) cabarchive2 = ufile.cabarchive_repacked self.assertIsNotNone(cabarchive2['firmware.bin']) self.assertIsNotNone(cabarchive2['firmware.metainfo.xml']) self.assertIsNotNone(cabarchive2['firmware.inf'])
def test_extra_files(self): cabarchive = CabArchive() cabarchive['firmware.bin'] = _get_valid_firmware() cabarchive['firmware.metainfo.xml'] = _get_valid_metainfo() cabarchive['README.txt'] = CabFile(b'fubar') ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', cabarchive.save()) cabarchive2 = ufile.cabarchive_repacked self.assertIsNotNone(cabarchive2['firmware.bin']) self.assertIsNotNone(cabarchive2['firmware.metainfo.xml']) with self.assertRaises(KeyError): self.assertIsNotNone(cabarchive2['README.txt'])
def test_multiple_metainfo_same_firmware(self): cabarchive = CabArchive() cabarchive['firmware.bin'] = _get_valid_firmware() cabarchive['firmware1.metainfo.xml'] = _get_valid_metainfo() cabarchive['firmware2.metainfo.xml'] = _get_valid_metainfo() ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', cabarchive.save()) cabarchive2 = ufile.cabarchive_repacked self.assertIsNotNone(cabarchive2['firmware.bin']) self.assertIsNotNone(cabarchive2['firmware1.metainfo.xml']) self.assertIsNotNone(cabarchive2['firmware2.metainfo.xml'])
def create_metadata(archive_dir, basename, metadata_fn): # process all archives in directory fws = [] print('Searching %s' % archive_dir) for root, dirs, files in os.walk(archive_dir): #pylint: disable=unused-variable for filename in files: if not filename.endswith('.cab'): continue print('Processing %s...' % filename) ufile = UploadedFile() with open(os.path.join(root, filename), 'r') as f: ufile.parse(filename, f.read(), use_hashed_prefix=False) fws.append(ufile.fw) # write metadata print('Writing %s' % metadata_fn) _generate_metadata_kind(metadata_fn, fws, firmware_baseuri="%s/" % basename, local=True)
def test_src_empty(self): with self.assertRaises(FileTooSmall): ufile = UploadedFile() _add_version_formats(ufile) ufile.parse('foo.cab', '') self.assertEqual(ufile.fwupd_min_version, '0.8.0')
def _upload_firmware(): # verify the user can upload if not _user_can_upload(g.user): flash('User has not signed legal agreement', 'danger') return redirect(url_for('main.route_dashboard')) # used a custom vendor_id if 'vendor_id' in request.form: try: vendor_id = int(request.form['vendor_id']) except ValueError as e: flash('Failed to upload file: Specified vendor ID %s invalid' % request.form['vendor_id'], 'warning') return redirect(url_for('upload.route_firmware')) vendor = db.session.query(Vendor).filter(Vendor.vendor_id == vendor_id).first() if not vendor: flash('Failed to upload file: Specified vendor ID not found', 'warning') return redirect(url_for('upload.route_firmware')) else: vendor = g.user.vendor # security check if not vendor.check_acl('@upload'): flash('Permission denied: Failed to upload file for vendor: ' 'User with vendor %s cannot upload to vendor %s' % (g.user.vendor.group_id, vendor.group_id), 'warning') return redirect(url_for('upload.route_firmware')) # not correct parameters if not 'target' in request.form: return _error_internal('No target') if not 'file' in request.files: return _error_internal('No file') if request.form['target'] not in ['private', 'embargo', 'testing']: return _error_internal('Target not valid') # find remote, creating if required remote_name = request.form['target'] if remote_name == 'embargo': remote = vendor.remote else: remote = db.session.query(Remote).filter(Remote.name == remote_name).first() if not remote: return _error_internal('No remote for target %s' % remote_name) # if the vendor has uploaded a lot of firmware don't start changing the rules is_strict = len(vendor.fws) < 500 # load in the archive fileitem = request.files['file'] if not fileitem: return _error_internal('No file object') try: ufile = UploadedFile(is_strict=is_strict) for cat in db.session.query(Category): ufile.category_map[cat.value] = cat.category_id for pro in db.session.query(Protocol): ufile.protocol_map[pro.value] = pro.protocol_id for verfmt in db.session.query(Verfmt): ufile.version_formats[verfmt.value] = verfmt ufile.parse(os.path.basename(fileitem.filename), fileitem.read()) except (FileTooLarge, FileTooSmall, FileNotSupported, MetadataInvalid) as e: flash('Failed to upload file: ' + str(e), 'danger') return redirect(request.url) # check the file does not already exist fw = db.session.query(Firmware)\ .filter(or_(Firmware.checksum_upload_sha1 == ufile.fw.checksum_upload_sha1, Firmware.checksum_upload_sha256 == ufile.fw.checksum_upload_sha256)).first() if fw: if fw.check_acl('@view'): flash('Failed to upload file: A file with hash %s already exists' % fw.checksum_upload_sha1, 'warning') return redirect('/lvfs/firmware/%s' % fw.firmware_id) flash('Failed to upload file: Another user has already uploaded this firmware', 'warning') return redirect(url_for('upload.route_firmware')) # check the guid and version does not already exist fws = db.session.query(Firmware).all() fws_already_exist = [] for md in ufile.fw.mds: provides_value = md.guids[0].value fw = _filter_fw_by_id_guid_version(fws, md.appstream_id, provides_value, md.version) if fw: fws_already_exist.append(fw) # all the components existed, so build an error out of all the versions if len(fws_already_exist) == len(ufile.fw.mds): if g.user.check_acl('@robot') and 'auto-delete' in request.form: for fw in fws_already_exist: if fw.remote.is_public: flash('Firmware {} cannot be autodeleted as is in remote {}'.format( fw.firmware_id, fw.remote.name), 'danger') return redirect(url_for('upload.route_firmware')) if fw.user.user_id != g.user.user_id: flash('Firmware was not uploaded by this user', 'danger') return redirect(url_for('upload.route_firmware')) for fw in fws_already_exist: flash('Firmware %i was auto-deleted due to robot upload' % fw.firmware_id) _firmware_delete(fw) else: versions_for_display = [] for fw in fws_already_exist: for md in fw.mds: if not md.version_display in versions_for_display: versions_for_display.append(md.version_display) flash('Failed to upload file: A firmware file for this device with ' 'version %s already exists' % ','.join(versions_for_display), 'danger') return redirect('/lvfs/firmware/%s' % fw.firmware_id) # check if the file dropped a GUID previously supported for umd in ufile.fw.mds: new_guids = [guid.value for guid in umd.guids] for md in db.session.query(Component).\ filter(Component.appstream_id == umd.appstream_id): if md.fw.is_deleted: continue for old_guid in [guid.value for guid in md.guids]: if old_guid in new_guids: continue fw_str = str(md.fw.firmware_id) if g.user.check_acl('@qa') or g.user.check_acl('@robot'): flash('Firmware drops GUID {} previously supported ' 'in firmware {}'.format(old_guid, fw_str), 'warning') else: flash('Firmware would drop GUID {} previously supported ' 'in firmware {}'.format(old_guid, fw_str), 'danger') return redirect(request.url) # allow plugins to copy any extra files from the source archive for cffile in ufile.cabarchive_upload.values(): ploader.archive_copy(ufile.cabarchive_repacked, cffile) # allow plugins to add files ploader.archive_finalize(ufile.cabarchive_repacked, _get_plugin_metadata_for_uploaded_file(ufile)) # dump to a file download_dir = app.config['DOWNLOAD_DIR'] if not os.path.exists(download_dir): os.mkdir(download_dir) fn = os.path.join(download_dir, ufile.fw.filename) cab_data = ufile.cabarchive_repacked.save(compress=True) with open(fn, 'wb') as f: f.write(cab_data) # create parent firmware object settings = _get_settings() target = request.form['target'] fw = ufile.fw fw.vendor = vendor fw.user = g.user fw.addr = _get_client_address() fw.remote = remote fw.checksum_signed_sha1 = hashlib.sha1(cab_data).hexdigest() fw.checksum_signed_sha256 = hashlib.sha256(cab_data).hexdigest() fw.is_dirty = True fw.failure_minimum = settings['default_failure_minimum'] fw.failure_percentage = settings['default_failure_percentage'] # fix name for md in fw.mds: name_fixed = _fix_component_name(md.name, md.developer_name_display) if name_fixed != md.name: flash('Fixed component name from "%s" to "%s"' % (md.name, name_fixed), 'warning') md.name = name_fixed # verify each component has a version format for md in fw.mds: if not md.verfmt_with_fallback: flash('Component {} does not have required LVFS::VersionFormat'.\ format(md.appstream_id), 'warning') # add to database fw.events.append(FirmwareEvent(remote_id=remote.remote_id, user_id=g.user.user_id)) db.session.add(fw) db.session.commit() # ensure the test has been added for the firmware type ploader.ensure_test_for_fw(fw) # send out emails to anyone interested for u in fw.get_possible_users_to_email: if u == g.user: continue if u.get_action('notify-upload-vendor') and u.vendor == fw.vendor: send_email("[LVFS] Firmware has been uploaded", u.email_address, render_template('email-firmware-uploaded.txt', user=u, user_upload=g.user, fw=fw)) elif u.get_action('notify-upload-affiliate'): send_email("[LVFS] Firmware has been uploaded by affiliate", u.email_address, render_template('email-firmware-uploaded.txt', user=u, user_upload=g.user, fw=fw)) flash('Uploaded file %s to %s' % (ufile.fw.filename, target), 'info') # invalidate if target == 'embargo': remote.is_dirty = True g.user.vendor.remote.is_dirty = True db.session.commit() return redirect(url_for('firmware.route_show', firmware_id=fw.firmware_id))