def _metadata_update_targets(remotes):
""" updates metadata for a specific target """
fws = db.session.query(Firmware).all()
settings = _get_settings()
# create metadata for each remote
targets = []
for r in remotes:
fws_filtered = []
for fw in fws:
if fw.remote.name in ['private', 'deleted']:
continue
if not fw.signed_timestamp:
continue
if r.check_fw(fw):
fws_filtered.append(fw)
blob = _generate_metadata_kind(
fws_filtered, firmware_baseuri=settings['firmware_baseuri'])
targets.append((r, blob))
# all firmwares are contained in the correct metadata now
for fw in fws_filtered:
fw.is_dirty = False
# success
return targets
def _cdn_mirror_file(self, test, url):
# download
try:
r = requests.get(url)
r.raise_for_status()
except requests.exceptions.RequestException as e:
test.add_fail('Download', str(e))
return None
# load as a PNG
try:
im = Image.open(BytesIO(r.content))
except UnidentifiedImageError as e:
test.add_fail('Parse', str(e))
return None
if im.width > 800 or im.height > 600:
test.add_fail('Size',
'{}x{} is too large'.format(im.width, im.height))
elif im.width < 300 or im.height < 100:
test.add_fail('Size',
'{}x{} is too small'.format(im.width, im.height))
# save to download directory
basename = 'img-{}.png'.format(hashlib.sha256(r.content).hexdigest())
fn = os.path.join(app.config['DOWNLOAD_DIR'], basename)
if not os.path.isfile(fn):
im.save(fn, "PNG")
# set the safe URL
settings = _get_settings('firmware')
return os.path.join(settings['firmware_baseuri_cdn'], basename)
def _metadata_update_targets(remotes):
""" updates metadata for a specific target """
fws = db.session.query(Firmware).all()
settings = _get_settings()
# set destination path from app config
download_dir = app.config['DOWNLOAD_DIR']
if not os.path.exists(download_dir):
os.mkdir(download_dir)
# create metadata for each remote
for r in remotes:
fws_filtered = []
for fw in fws:
if fw.remote.name in ['private', 'deleted']:
continue
if not fw.signed_timestamp:
continue
if r.check_fw(fw):
fws_filtered.append(fw)
_generate_metadata_kind(os.path.join(download_dir, r.filename),
fws_filtered,
firmware_baseuri=settings['firmware_baseuri'])
# all firmwares are contained in the correct metadata now
for fw in fws_filtered:
fw.is_dirty = False
def _get_plugin_metadata_for_uploaded_file(ufile):
settings = _get_settings()
metadata = {}
metadata['$DATE$'] = datetime.datetime.now().replace(microsecond=0).isoformat()
metadata['$FWUPD_MIN_VERSION$'] = ufile.fwupd_min_version
metadata['$CAB_FILENAME$'] = ufile.fw.filename
metadata['$FIRMWARE_BASEURI$'] = settings['firmware_baseuri']
return metadata
def route_create():
# create all the plugin default keys
settings = _get_settings()
for plugin in ploader.get_all():
for s in plugin.settings():
if s.key not in settings:
db.session.add(Setting(key=s.key, value=s.default))
db.session.commit()
return redirect(url_for('settings.route_view'))
def route_view(plugin_id='general'):
"""
Allows the admin to change details about the LVFS instance
"""
plugin = ploader.get_by_id(plugin_id)
if not plugin:
flash('No plugin {}'.format(plugin_id), 'danger')
return redirect(url_for('settings.route_view'))
tests_by_type = _convert_tests_for_plugin(plugin)
return render_template('settings.html',
category='settings',
settings=_get_settings(),
plugin=plugin,
tests_by_type=tests_by_type)
def route_dashboard():
user = db.session.query(User).filter(
User.username == '*****@*****.**').first()
settings = _get_settings()
default_admin_password = False
if user and user.verify_password('Pa$$w0rd'):
default_admin_password = True
# get the 10 most recent firmwares
fws = db.session.query(Firmware).\
filter(Firmware.user_id == g.user.user_id).\
join(Remote).filter(Remote.name != 'deleted').\
order_by(Firmware.timestamp.desc()).limit(10).all()
download_cnt = 0
devices_cnt = 0
appstream_ids = {}
for fw in g.user.vendor.fws:
download_cnt += fw.download_cnt
for md in fw.mds:
appstream_ids[md.appstream_id] = fw
devices_cnt = len(appstream_ids)
# this is somewhat klunky
data = []
datestr = _get_datestr_from_datetime(datetime.date.today() -
datetime.timedelta(days=31))
for cnt in db.session.query(AnalyticVendor.cnt).\
filter(AnalyticVendor.vendor_id == g.user.vendor.vendor_id).\
filter(AnalyticVendor.datestr > datestr).\
order_by(AnalyticVendor.datestr):
data.append(int(cnt[0]))
return render_template(
'dashboard.html',
fws_recent=fws,
devices_cnt=devices_cnt,
download_cnt=download_cnt,
labels_days=_get_chart_labels_days(limit=len(data))[::-1],
data_days=data,
server_warning=settings.get('server_warning', None),
category='home',
default_admin_password=default_admin_password)
def route_modify(plugin_id='general'):
""" Change details about the instance """
# only accept form data
if request.method != 'POST':
return redirect(url_for('settings.route_view', plugin_id=plugin_id))
# save new values
settings = _get_settings()
for key in request.form:
if key == 'csrf_token':
continue
if settings[key] == request.form[key]:
continue
setting = db.session.query(Setting).filter(Setting.key == key).first()
setting.value = _textarea_string_to_text(request.form[key])
_event_log('Changed server settings %s to %s' % (key, setting.value))
db.session.commit()
flash('Updated settings', 'info')
return redirect(url_for('settings.route_view', plugin_id=plugin_id), 302)
def route_hwinfo():
""" Upload a hwinfo binary file to the LVFS service without authentication """
# not correct parameters
if not 'type' in request.form:
return _json_error('no type')
if not 'machine_id' in request.form:
return _json_error('no machine_id')
if not 'file' in request.files:
return _json_error('no file')
if len(request.form['machine_id']) != 32:
return _json_error('machine_id %s not valid' % request.form['machine_id'])
try:
int(request.form['machine_id'], 16)
except ValueError as e:
return _json_error(str(e))
# check type against defined list
settings = _get_settings()
if request.form['type'] not in settings['hwinfo_kinds'].split(','):
return _json_error('type not valid')
# read in entire file
fileitem = request.files['file']
if not fileitem:
return _json_error('no file object')
filebuf = fileitem.read()
if len(filebuf) > 0x40000:
return _json_error('file is too large')
# dump to a file
hwinfo_dir = os.path.join(app.config['HWINFO_DIR'], request.form['type'])
if not os.path.exists(hwinfo_dir):
os.mkdir(hwinfo_dir)
fn = os.path.join(hwinfo_dir, '%s' % request.form['machine_id'])
if os.path.exists(fn):
return _json_error('already reported from this machine-id')
with open(fn, 'wb') as f:
f.write(filebuf)
return _json_success()
# Copyright (C) 2018 Richard Hughes <*****@*****.**>
#
# SPDX-License-Identifier: GPL-2.0+
#
# pylint: disable=no-self-use,no-member,unexpected-keyword-arg
import uuid
from flask import session, request
from flask_oauthlib.client import OAuthException
from lvfs import oauth
from lvfs.pluginloader import PluginBase, PluginSettingBool, PluginSettingText, PluginError
from lvfs.util import _get_settings
settings = _get_settings('auth_azure')
if 'auth_azure_consumer_key' in settings and settings[
'auth_azure_consumer_key']:
remote_app = oauth.remote_app(
'microsoft',
consumer_key=settings['auth_azure_consumer_key'],
consumer_secret=settings['auth_azure_consumer_secret'],
request_token_params={'scope': 'offline_access User.Read'},
base_url='https://graph.microsoft.com/v1.0/',
request_token_url=None,
access_token_method='POST',
access_token_url=
'https://login.microsoftonline.com/common/oauth2/v2.0/token',
authorize_url=
'https://login.microsoftonline.com/common/oauth2/v2.0/authorize')
def _upload_firmware():
# verify the user can upload
if not _user_can_upload(g.user):
flash('User has not signed legal agreement', 'danger')
return redirect(url_for('main.route_dashboard'))
# used a custom vendor_id
if 'vendor_id' in request.form:
try:
vendor_id = int(request.form['vendor_id'])
except ValueError as e:
flash('Failed to upload file: Specified vendor ID %s invalid' % request.form['vendor_id'], 'warning')
return redirect(url_for('upload.route_firmware'))
vendor = db.session.query(Vendor).filter(Vendor.vendor_id == vendor_id).first()
if not vendor:
flash('Failed to upload file: Specified vendor ID not found', 'warning')
return redirect(url_for('upload.route_firmware'))
else:
vendor = g.user.vendor
# security check
if not vendor.check_acl('@upload'):
flash('Permission denied: Failed to upload file for vendor: '
'User with vendor %s cannot upload to vendor %s' %
(g.user.vendor.group_id, vendor.group_id), 'warning')
return redirect(url_for('upload.route_firmware'))
# not correct parameters
if not 'target' in request.form:
return _error_internal('No target')
if not 'file' in request.files:
return _error_internal('No file')
if request.form['target'] not in ['private', 'embargo', 'testing']:
return _error_internal('Target not valid')
# find remote, creating if required
remote_name = request.form['target']
if remote_name == 'embargo':
remote = vendor.remote
else:
remote = db.session.query(Remote).filter(Remote.name == remote_name).first()
if not remote:
return _error_internal('No remote for target %s' % remote_name)
# if the vendor has uploaded a lot of firmware don't start changing the rules
is_strict = len(vendor.fws) < 500
# load in the archive
fileitem = request.files['file']
if not fileitem:
return _error_internal('No file object')
try:
ufile = UploadedFile(is_strict=is_strict)
for cat in db.session.query(Category):
ufile.category_map[cat.value] = cat.category_id
for pro in db.session.query(Protocol):
ufile.protocol_map[pro.value] = pro.protocol_id
for verfmt in db.session.query(Verfmt):
ufile.version_formats[verfmt.value] = verfmt
ufile.parse(os.path.basename(fileitem.filename), fileitem.read())
except (FileTooLarge, FileTooSmall, FileNotSupported, MetadataInvalid) as e:
flash('Failed to upload file: ' + str(e), 'danger')
return redirect(request.url)
# check the file does not already exist
fw = db.session.query(Firmware)\
.filter(or_(Firmware.checksum_upload_sha1 == ufile.fw.checksum_upload_sha1,
Firmware.checksum_upload_sha256 == ufile.fw.checksum_upload_sha256)).first()
if fw:
if fw.check_acl('@view'):
flash('Failed to upload file: A file with hash %s already exists' % fw.checksum_upload_sha1, 'warning')
return redirect('/lvfs/firmware/%s' % fw.firmware_id)
flash('Failed to upload file: Another user has already uploaded this firmware', 'warning')
return redirect(url_for('upload.route_firmware'))
# check the guid and version does not already exist
fws = db.session.query(Firmware).all()
fws_already_exist = []
for md in ufile.fw.mds:
provides_value = md.guids[0].value
fw = _filter_fw_by_id_guid_version(fws,
md.appstream_id,
provides_value,
md.version)
if fw:
fws_already_exist.append(fw)
# all the components existed, so build an error out of all the versions
if len(fws_already_exist) == len(ufile.fw.mds):
if g.user.check_acl('@robot') and 'auto-delete' in request.form:
for fw in fws_already_exist:
if fw.remote.is_public:
flash('Firmware {} cannot be autodeleted as is in remote {}'.format(
fw.firmware_id, fw.remote.name), 'danger')
return redirect(url_for('upload.route_firmware'))
if fw.user.user_id != g.user.user_id:
flash('Firmware was not uploaded by this user', 'danger')
return redirect(url_for('upload.route_firmware'))
for fw in fws_already_exist:
flash('Firmware %i was auto-deleted due to robot upload' % fw.firmware_id)
_firmware_delete(fw)
else:
versions_for_display = []
for fw in fws_already_exist:
for md in fw.mds:
if not md.version_display in versions_for_display:
versions_for_display.append(md.version_display)
flash('Failed to upload file: A firmware file for this device with '
'version %s already exists' % ','.join(versions_for_display), 'danger')
return redirect('/lvfs/firmware/%s' % fw.firmware_id)
# check if the file dropped a GUID previously supported
for umd in ufile.fw.mds:
new_guids = [guid.value for guid in umd.guids]
for md in db.session.query(Component).\
filter(Component.appstream_id == umd.appstream_id):
if md.fw.is_deleted:
continue
for old_guid in [guid.value for guid in md.guids]:
if old_guid in new_guids:
continue
fw_str = str(md.fw.firmware_id)
if g.user.check_acl('@qa') or g.user.check_acl('@robot'):
flash('Firmware drops GUID {} previously supported '
'in firmware {}'.format(old_guid, fw_str), 'warning')
else:
flash('Firmware would drop GUID {} previously supported '
'in firmware {}'.format(old_guid, fw_str), 'danger')
return redirect(request.url)
# allow plugins to copy any extra files from the source archive
for cffile in ufile.cabarchive_upload.values():
ploader.archive_copy(ufile.cabarchive_repacked, cffile)
# allow plugins to add files
ploader.archive_finalize(ufile.cabarchive_repacked,
_get_plugin_metadata_for_uploaded_file(ufile))
# dump to a file
download_dir = app.config['DOWNLOAD_DIR']
if not os.path.exists(download_dir):
os.mkdir(download_dir)
fn = os.path.join(download_dir, ufile.fw.filename)
cab_data = ufile.cabarchive_repacked.save(compress=True)
with open(fn, 'wb') as f:
f.write(cab_data)
# create parent firmware object
settings = _get_settings()
target = request.form['target']
fw = ufile.fw
fw.vendor = vendor
fw.user = g.user
fw.addr = _get_client_address()
fw.remote = remote
fw.checksum_signed_sha1 = hashlib.sha1(cab_data).hexdigest()
fw.checksum_signed_sha256 = hashlib.sha256(cab_data).hexdigest()
fw.is_dirty = True
fw.failure_minimum = settings['default_failure_minimum']
fw.failure_percentage = settings['default_failure_percentage']
# fix name
for md in fw.mds:
name_fixed = _fix_component_name(md.name, md.developer_name_display)
if name_fixed != md.name:
flash('Fixed component name from "%s" to "%s"' % (md.name, name_fixed), 'warning')
md.name = name_fixed
# verify each component has a version format
for md in fw.mds:
if not md.verfmt_with_fallback:
flash('Component {} does not have required LVFS::VersionFormat'.\
format(md.appstream_id), 'warning')
# add to database
fw.events.append(FirmwareEvent(remote_id=remote.remote_id, user_id=g.user.user_id))
db.session.add(fw)
db.session.commit()
# ensure the test has been added for the firmware type
ploader.ensure_test_for_fw(fw)
# send out emails to anyone interested
for u in fw.get_possible_users_to_email:
if u == g.user:
continue
if u.get_action('notify-upload-vendor') and u.vendor == fw.vendor:
send_email("[LVFS] Firmware has been uploaded",
u.email_address,
render_template('email-firmware-uploaded.txt',
user=u, user_upload=g.user, fw=fw))
elif u.get_action('notify-upload-affiliate'):
send_email("[LVFS] Firmware has been uploaded by affiliate",
u.email_address,
render_template('email-firmware-uploaded.txt',
user=u, user_upload=g.user, fw=fw))
flash('Uploaded file %s to %s' % (ufile.fw.filename, target), 'info')
# invalidate
if target == 'embargo':
remote.is_dirty = True
g.user.vendor.remote.is_dirty = True
db.session.commit()
return redirect(url_for('firmware.route_show', firmware_id=fw.firmware_id))
def _regenerate_and_sign_metadata_remote(r):
# not required */
if not r.is_signed:
return
# fix up any remotes that are not dirty, but have firmware that is dirty
# -- which shouldn't happen, but did...
if not r.is_dirty:
for fw in r.fws:
if not fw.is_dirty:
continue
print('Marking remote %s as dirty due to %u' % (r.name, fw.firmware_id))
r.is_dirty = True
fw.is_dirty = False
# not needed
if not r.is_dirty:
return
# set destination path from app config
download_dir = app.config['DOWNLOAD_DIR']
if not os.path.exists(download_dir):
os.mkdir(download_dir)
invalid_fns = []
print('Updating: %s' % r.name)
# create metadata for each remote
fws_filtered = []
for fw in db.session.query(Firmware):
if fw.remote.name in ['private', 'deleted']:
continue
if not fw.signed_timestamp:
continue
if r.check_fw(fw):
fws_filtered.append(fw)
settings = _get_settings()
blob_xmlgz = _generate_metadata_kind(fws_filtered,
firmware_baseuri=settings['firmware_baseuri'])
# write metadata-?????.xml.gz
fn_xmlgz = os.path.join(download_dir, r.filename)
with open(fn_xmlgz, 'wb') as f:
f.write(blob_xmlgz)
invalid_fns.append(fn_xmlgz)
# write metadata.xml.gz
fn_xmlgz = os.path.join(download_dir, r.filename_newest)
with open(fn_xmlgz, 'wb') as f:
f.write(blob_xmlgz)
invalid_fns.append(fn_xmlgz)
# create Jcat item with SHA256 checksum blob
jcatfile = JcatFile()
jcatitem = jcatfile.get_item(r.filename)
jcatitem.add_alias_id(r.filename_newest)
jcatitem.add_blob(JcatBlobSha1(blob_xmlgz))
jcatitem.add_blob(JcatBlobSha256(blob_xmlgz))
# write each signed file
for blob in ploader.metadata_sign(blob_xmlgz):
# add GPG only to archive for backwards compat with older fwupd
if blob.kind == JcatBlobKind.GPG:
fn_xmlgz_asc = fn_xmlgz + '.' + blob.filename_ext
with open(fn_xmlgz_asc, 'wb') as f:
f.write(blob.data)
invalid_fns.append(fn_xmlgz_asc)
# add to Jcat file too
jcatitem.add_blob(blob)
# write jcat file
fn_xmlgz_jcat = fn_xmlgz + '.jcat'
with open(fn_xmlgz_jcat, 'wb') as f:
f.write(jcatfile.save())
invalid_fns.append(fn_xmlgz_jcat)
# update PULP
if r.name == 'stable':
_metadata_update_pulp(download_dir)
# do this all at once right at the end of all the I/O
for fn in invalid_fns:
print('Invalidating {}'.format(fn))
ploader.file_modified(fn)
# mark as no longer dirty
if not r.build_cnt:
r.build_cnt = 0
r.build_cnt += 1
r.is_dirty = False
# log what we did
_event_log('Signed metadata {} build {}'.format(r.name, r.build_cnt))
# only keep the last 6 metadata builds (24h / stable refresh every 4h)
suffix = r.filename.split('-')[2]
fns = glob.glob(os.path.join(download_dir, 'firmware-*-{}'.format(suffix)))
for fn in sorted(fns):
build_cnt = int(fn.split('-')[1])
if build_cnt + 6 > r.build_cnt:
continue
os.remove(fn)
_event_log('Deleted metadata {} build {}'.format(r.name, build_cnt))
# all firmwares are contained in the correct metadata now
for fw in fws_filtered:
fw.is_dirty = False
db.session.commit()
