def _metadata_update_targets(remotes): """ updates metadata for a specific target """ fws = db.session.query(Firmware).all() settings = _get_settings() # create metadata for each remote targets = [] for r in remotes: fws_filtered = [] for fw in fws: if fw.remote.name in ['private', 'deleted']: continue if not fw.signed_timestamp: continue if r.check_fw(fw): fws_filtered.append(fw) blob = _generate_metadata_kind( fws_filtered, firmware_baseuri=settings['firmware_baseuri']) targets.append((r, blob)) # all firmwares are contained in the correct metadata now for fw in fws_filtered: fw.is_dirty = False # success return targets
def _cdn_mirror_file(self, test, url): # download try: r = requests.get(url) r.raise_for_status() except requests.exceptions.RequestException as e: test.add_fail('Download', str(e)) return None # load as a PNG try: im = Image.open(BytesIO(r.content)) except UnidentifiedImageError as e: test.add_fail('Parse', str(e)) return None if im.width > 800 or im.height > 600: test.add_fail('Size', '{}x{} is too large'.format(im.width, im.height)) elif im.width < 300 or im.height < 100: test.add_fail('Size', '{}x{} is too small'.format(im.width, im.height)) # save to download directory basename = 'img-{}.png'.format(hashlib.sha256(r.content).hexdigest()) fn = os.path.join(app.config['DOWNLOAD_DIR'], basename) if not os.path.isfile(fn): im.save(fn, "PNG") # set the safe URL settings = _get_settings('firmware') return os.path.join(settings['firmware_baseuri_cdn'], basename)
def _metadata_update_targets(remotes): """ updates metadata for a specific target """ fws = db.session.query(Firmware).all() settings = _get_settings() # set destination path from app config download_dir = app.config['DOWNLOAD_DIR'] if not os.path.exists(download_dir): os.mkdir(download_dir) # create metadata for each remote for r in remotes: fws_filtered = [] for fw in fws: if fw.remote.name in ['private', 'deleted']: continue if not fw.signed_timestamp: continue if r.check_fw(fw): fws_filtered.append(fw) _generate_metadata_kind(os.path.join(download_dir, r.filename), fws_filtered, firmware_baseuri=settings['firmware_baseuri']) # all firmwares are contained in the correct metadata now for fw in fws_filtered: fw.is_dirty = False
def _get_plugin_metadata_for_uploaded_file(ufile): settings = _get_settings() metadata = {} metadata['$DATE$'] = datetime.datetime.now().replace(microsecond=0).isoformat() metadata['$FWUPD_MIN_VERSION$'] = ufile.fwupd_min_version metadata['$CAB_FILENAME$'] = ufile.fw.filename metadata['$FIRMWARE_BASEURI$'] = settings['firmware_baseuri'] return metadata
def route_create(): # create all the plugin default keys settings = _get_settings() for plugin in ploader.get_all(): for s in plugin.settings(): if s.key not in settings: db.session.add(Setting(key=s.key, value=s.default)) db.session.commit() return redirect(url_for('settings.route_view'))
def route_view(plugin_id='general'): """ Allows the admin to change details about the LVFS instance """ plugin = ploader.get_by_id(plugin_id) if not plugin: flash('No plugin {}'.format(plugin_id), 'danger') return redirect(url_for('settings.route_view')) tests_by_type = _convert_tests_for_plugin(plugin) return render_template('settings.html', category='settings', settings=_get_settings(), plugin=plugin, tests_by_type=tests_by_type)
def route_dashboard(): user = db.session.query(User).filter( User.username == '*****@*****.**').first() settings = _get_settings() default_admin_password = False if user and user.verify_password('Pa$$w0rd'): default_admin_password = True # get the 10 most recent firmwares fws = db.session.query(Firmware).\ filter(Firmware.user_id == g.user.user_id).\ join(Remote).filter(Remote.name != 'deleted').\ order_by(Firmware.timestamp.desc()).limit(10).all() download_cnt = 0 devices_cnt = 0 appstream_ids = {} for fw in g.user.vendor.fws: download_cnt += fw.download_cnt for md in fw.mds: appstream_ids[md.appstream_id] = fw devices_cnt = len(appstream_ids) # this is somewhat klunky data = [] datestr = _get_datestr_from_datetime(datetime.date.today() - datetime.timedelta(days=31)) for cnt in db.session.query(AnalyticVendor.cnt).\ filter(AnalyticVendor.vendor_id == g.user.vendor.vendor_id).\ filter(AnalyticVendor.datestr > datestr).\ order_by(AnalyticVendor.datestr): data.append(int(cnt[0])) return render_template( 'dashboard.html', fws_recent=fws, devices_cnt=devices_cnt, download_cnt=download_cnt, labels_days=_get_chart_labels_days(limit=len(data))[::-1], data_days=data, server_warning=settings.get('server_warning', None), category='home', default_admin_password=default_admin_password)
def route_modify(plugin_id='general'): """ Change details about the instance """ # only accept form data if request.method != 'POST': return redirect(url_for('settings.route_view', plugin_id=plugin_id)) # save new values settings = _get_settings() for key in request.form: if key == 'csrf_token': continue if settings[key] == request.form[key]: continue setting = db.session.query(Setting).filter(Setting.key == key).first() setting.value = _textarea_string_to_text(request.form[key]) _event_log('Changed server settings %s to %s' % (key, setting.value)) db.session.commit() flash('Updated settings', 'info') return redirect(url_for('settings.route_view', plugin_id=plugin_id), 302)
def route_hwinfo(): """ Upload a hwinfo binary file to the LVFS service without authentication """ # not correct parameters if not 'type' in request.form: return _json_error('no type') if not 'machine_id' in request.form: return _json_error('no machine_id') if not 'file' in request.files: return _json_error('no file') if len(request.form['machine_id']) != 32: return _json_error('machine_id %s not valid' % request.form['machine_id']) try: int(request.form['machine_id'], 16) except ValueError as e: return _json_error(str(e)) # check type against defined list settings = _get_settings() if request.form['type'] not in settings['hwinfo_kinds'].split(','): return _json_error('type not valid') # read in entire file fileitem = request.files['file'] if not fileitem: return _json_error('no file object') filebuf = fileitem.read() if len(filebuf) > 0x40000: return _json_error('file is too large') # dump to a file hwinfo_dir = os.path.join(app.config['HWINFO_DIR'], request.form['type']) if not os.path.exists(hwinfo_dir): os.mkdir(hwinfo_dir) fn = os.path.join(hwinfo_dir, '%s' % request.form['machine_id']) if os.path.exists(fn): return _json_error('already reported from this machine-id') with open(fn, 'wb') as f: f.write(filebuf) return _json_success()
# Copyright (C) 2018 Richard Hughes <*****@*****.**> # # SPDX-License-Identifier: GPL-2.0+ # # pylint: disable=no-self-use,no-member,unexpected-keyword-arg import uuid from flask import session, request from flask_oauthlib.client import OAuthException from lvfs import oauth from lvfs.pluginloader import PluginBase, PluginSettingBool, PluginSettingText, PluginError from lvfs.util import _get_settings settings = _get_settings('auth_azure') if 'auth_azure_consumer_key' in settings and settings[ 'auth_azure_consumer_key']: remote_app = oauth.remote_app( 'microsoft', consumer_key=settings['auth_azure_consumer_key'], consumer_secret=settings['auth_azure_consumer_secret'], request_token_params={'scope': 'offline_access User.Read'}, base_url='https://graph.microsoft.com/v1.0/', request_token_url=None, access_token_method='POST', access_token_url= 'https://login.microsoftonline.com/common/oauth2/v2.0/token', authorize_url= 'https://login.microsoftonline.com/common/oauth2/v2.0/authorize')
def _upload_firmware(): # verify the user can upload if not _user_can_upload(g.user): flash('User has not signed legal agreement', 'danger') return redirect(url_for('main.route_dashboard')) # used a custom vendor_id if 'vendor_id' in request.form: try: vendor_id = int(request.form['vendor_id']) except ValueError as e: flash('Failed to upload file: Specified vendor ID %s invalid' % request.form['vendor_id'], 'warning') return redirect(url_for('upload.route_firmware')) vendor = db.session.query(Vendor).filter(Vendor.vendor_id == vendor_id).first() if not vendor: flash('Failed to upload file: Specified vendor ID not found', 'warning') return redirect(url_for('upload.route_firmware')) else: vendor = g.user.vendor # security check if not vendor.check_acl('@upload'): flash('Permission denied: Failed to upload file for vendor: ' 'User with vendor %s cannot upload to vendor %s' % (g.user.vendor.group_id, vendor.group_id), 'warning') return redirect(url_for('upload.route_firmware')) # not correct parameters if not 'target' in request.form: return _error_internal('No target') if not 'file' in request.files: return _error_internal('No file') if request.form['target'] not in ['private', 'embargo', 'testing']: return _error_internal('Target not valid') # find remote, creating if required remote_name = request.form['target'] if remote_name == 'embargo': remote = vendor.remote else: remote = db.session.query(Remote).filter(Remote.name == remote_name).first() if not remote: return _error_internal('No remote for target %s' % remote_name) # if the vendor has uploaded a lot of firmware don't start changing the rules is_strict = len(vendor.fws) < 500 # load in the archive fileitem = request.files['file'] if not fileitem: return _error_internal('No file object') try: ufile = UploadedFile(is_strict=is_strict) for cat in db.session.query(Category): ufile.category_map[cat.value] = cat.category_id for pro in db.session.query(Protocol): ufile.protocol_map[pro.value] = pro.protocol_id for verfmt in db.session.query(Verfmt): ufile.version_formats[verfmt.value] = verfmt ufile.parse(os.path.basename(fileitem.filename), fileitem.read()) except (FileTooLarge, FileTooSmall, FileNotSupported, MetadataInvalid) as e: flash('Failed to upload file: ' + str(e), 'danger') return redirect(request.url) # check the file does not already exist fw = db.session.query(Firmware)\ .filter(or_(Firmware.checksum_upload_sha1 == ufile.fw.checksum_upload_sha1, Firmware.checksum_upload_sha256 == ufile.fw.checksum_upload_sha256)).first() if fw: if fw.check_acl('@view'): flash('Failed to upload file: A file with hash %s already exists' % fw.checksum_upload_sha1, 'warning') return redirect('/lvfs/firmware/%s' % fw.firmware_id) flash('Failed to upload file: Another user has already uploaded this firmware', 'warning') return redirect(url_for('upload.route_firmware')) # check the guid and version does not already exist fws = db.session.query(Firmware).all() fws_already_exist = [] for md in ufile.fw.mds: provides_value = md.guids[0].value fw = _filter_fw_by_id_guid_version(fws, md.appstream_id, provides_value, md.version) if fw: fws_already_exist.append(fw) # all the components existed, so build an error out of all the versions if len(fws_already_exist) == len(ufile.fw.mds): if g.user.check_acl('@robot') and 'auto-delete' in request.form: for fw in fws_already_exist: if fw.remote.is_public: flash('Firmware {} cannot be autodeleted as is in remote {}'.format( fw.firmware_id, fw.remote.name), 'danger') return redirect(url_for('upload.route_firmware')) if fw.user.user_id != g.user.user_id: flash('Firmware was not uploaded by this user', 'danger') return redirect(url_for('upload.route_firmware')) for fw in fws_already_exist: flash('Firmware %i was auto-deleted due to robot upload' % fw.firmware_id) _firmware_delete(fw) else: versions_for_display = [] for fw in fws_already_exist: for md in fw.mds: if not md.version_display in versions_for_display: versions_for_display.append(md.version_display) flash('Failed to upload file: A firmware file for this device with ' 'version %s already exists' % ','.join(versions_for_display), 'danger') return redirect('/lvfs/firmware/%s' % fw.firmware_id) # check if the file dropped a GUID previously supported for umd in ufile.fw.mds: new_guids = [guid.value for guid in umd.guids] for md in db.session.query(Component).\ filter(Component.appstream_id == umd.appstream_id): if md.fw.is_deleted: continue for old_guid in [guid.value for guid in md.guids]: if old_guid in new_guids: continue fw_str = str(md.fw.firmware_id) if g.user.check_acl('@qa') or g.user.check_acl('@robot'): flash('Firmware drops GUID {} previously supported ' 'in firmware {}'.format(old_guid, fw_str), 'warning') else: flash('Firmware would drop GUID {} previously supported ' 'in firmware {}'.format(old_guid, fw_str), 'danger') return redirect(request.url) # allow plugins to copy any extra files from the source archive for cffile in ufile.cabarchive_upload.values(): ploader.archive_copy(ufile.cabarchive_repacked, cffile) # allow plugins to add files ploader.archive_finalize(ufile.cabarchive_repacked, _get_plugin_metadata_for_uploaded_file(ufile)) # dump to a file download_dir = app.config['DOWNLOAD_DIR'] if not os.path.exists(download_dir): os.mkdir(download_dir) fn = os.path.join(download_dir, ufile.fw.filename) cab_data = ufile.cabarchive_repacked.save(compress=True) with open(fn, 'wb') as f: f.write(cab_data) # create parent firmware object settings = _get_settings() target = request.form['target'] fw = ufile.fw fw.vendor = vendor fw.user = g.user fw.addr = _get_client_address() fw.remote = remote fw.checksum_signed_sha1 = hashlib.sha1(cab_data).hexdigest() fw.checksum_signed_sha256 = hashlib.sha256(cab_data).hexdigest() fw.is_dirty = True fw.failure_minimum = settings['default_failure_minimum'] fw.failure_percentage = settings['default_failure_percentage'] # fix name for md in fw.mds: name_fixed = _fix_component_name(md.name, md.developer_name_display) if name_fixed != md.name: flash('Fixed component name from "%s" to "%s"' % (md.name, name_fixed), 'warning') md.name = name_fixed # verify each component has a version format for md in fw.mds: if not md.verfmt_with_fallback: flash('Component {} does not have required LVFS::VersionFormat'.\ format(md.appstream_id), 'warning') # add to database fw.events.append(FirmwareEvent(remote_id=remote.remote_id, user_id=g.user.user_id)) db.session.add(fw) db.session.commit() # ensure the test has been added for the firmware type ploader.ensure_test_for_fw(fw) # send out emails to anyone interested for u in fw.get_possible_users_to_email: if u == g.user: continue if u.get_action('notify-upload-vendor') and u.vendor == fw.vendor: send_email("[LVFS] Firmware has been uploaded", u.email_address, render_template('email-firmware-uploaded.txt', user=u, user_upload=g.user, fw=fw)) elif u.get_action('notify-upload-affiliate'): send_email("[LVFS] Firmware has been uploaded by affiliate", u.email_address, render_template('email-firmware-uploaded.txt', user=u, user_upload=g.user, fw=fw)) flash('Uploaded file %s to %s' % (ufile.fw.filename, target), 'info') # invalidate if target == 'embargo': remote.is_dirty = True g.user.vendor.remote.is_dirty = True db.session.commit() return redirect(url_for('firmware.route_show', firmware_id=fw.firmware_id))
def _regenerate_and_sign_metadata_remote(r): # not required */ if not r.is_signed: return # fix up any remotes that are not dirty, but have firmware that is dirty # -- which shouldn't happen, but did... if not r.is_dirty: for fw in r.fws: if not fw.is_dirty: continue print('Marking remote %s as dirty due to %u' % (r.name, fw.firmware_id)) r.is_dirty = True fw.is_dirty = False # not needed if not r.is_dirty: return # set destination path from app config download_dir = app.config['DOWNLOAD_DIR'] if not os.path.exists(download_dir): os.mkdir(download_dir) invalid_fns = [] print('Updating: %s' % r.name) # create metadata for each remote fws_filtered = [] for fw in db.session.query(Firmware): if fw.remote.name in ['private', 'deleted']: continue if not fw.signed_timestamp: continue if r.check_fw(fw): fws_filtered.append(fw) settings = _get_settings() blob_xmlgz = _generate_metadata_kind(fws_filtered, firmware_baseuri=settings['firmware_baseuri']) # write metadata-?????.xml.gz fn_xmlgz = os.path.join(download_dir, r.filename) with open(fn_xmlgz, 'wb') as f: f.write(blob_xmlgz) invalid_fns.append(fn_xmlgz) # write metadata.xml.gz fn_xmlgz = os.path.join(download_dir, r.filename_newest) with open(fn_xmlgz, 'wb') as f: f.write(blob_xmlgz) invalid_fns.append(fn_xmlgz) # create Jcat item with SHA256 checksum blob jcatfile = JcatFile() jcatitem = jcatfile.get_item(r.filename) jcatitem.add_alias_id(r.filename_newest) jcatitem.add_blob(JcatBlobSha1(blob_xmlgz)) jcatitem.add_blob(JcatBlobSha256(blob_xmlgz)) # write each signed file for blob in ploader.metadata_sign(blob_xmlgz): # add GPG only to archive for backwards compat with older fwupd if blob.kind == JcatBlobKind.GPG: fn_xmlgz_asc = fn_xmlgz + '.' + blob.filename_ext with open(fn_xmlgz_asc, 'wb') as f: f.write(blob.data) invalid_fns.append(fn_xmlgz_asc) # add to Jcat file too jcatitem.add_blob(blob) # write jcat file fn_xmlgz_jcat = fn_xmlgz + '.jcat' with open(fn_xmlgz_jcat, 'wb') as f: f.write(jcatfile.save()) invalid_fns.append(fn_xmlgz_jcat) # update PULP if r.name == 'stable': _metadata_update_pulp(download_dir) # do this all at once right at the end of all the I/O for fn in invalid_fns: print('Invalidating {}'.format(fn)) ploader.file_modified(fn) # mark as no longer dirty if not r.build_cnt: r.build_cnt = 0 r.build_cnt += 1 r.is_dirty = False # log what we did _event_log('Signed metadata {} build {}'.format(r.name, r.build_cnt)) # only keep the last 6 metadata builds (24h / stable refresh every 4h) suffix = r.filename.split('-')[2] fns = glob.glob(os.path.join(download_dir, 'firmware-*-{}'.format(suffix))) for fn in sorted(fns): build_cnt = int(fn.split('-')[1]) if build_cnt + 6 > r.build_cnt: continue os.remove(fn) _event_log('Deleted metadata {} build {}'.format(r.name, build_cnt)) # all firmwares are contained in the correct metadata now for fw in fws_filtered: fw.is_dirty = False db.session.commit()