def sign_file(file_obj):
"""Sign a File if necessary.
If it's not necessary (file exists but it's a mozilla signed one) then
return the file directly.
If there's no endpoint (signing is not enabled) or isn't reviewed yet,
or there was an error while signing, raise an exception - it
shouldn't happen.
Otherwise proceed with signing and return the signed file.
"""
from olympia.git.utils import create_git_extraction_entry
if not settings.ENABLE_ADDON_SIGNING:
raise SigningError('Not signing file {0}: no active endpoint'.format(
file_obj.pk))
# No file? No signature.
if not os.path.exists(file_obj.current_file_path):
raise SigningError("File {0} doesn't exist on disk".format(
file_obj.current_file_path))
# Don't sign Mozilla signed extensions (they're already signed).
if file_obj.is_mozilla_signed_extension:
# Don't raise an exception here, just log and return file_obj even
# though we didn't sign, it's not an error - we just don't need to do
# anything in this case.
log.info('Not signing file {0}: mozilla signed extension is already '
'signed'.format(file_obj.pk))
return file_obj
# We only sign files that are compatible with Firefox.
if not supports_firefox(file_obj):
raise SigningError(
'Not signing version {0}: not for a Firefox version we support'.
format(file_obj.version.pk))
# Sign the file. If there's any exception, we skip the rest.
cert_serial_num = str(call_signing(file_obj))
size = storage.size(file_obj.current_file_path)
# Save the certificate serial number for revocation if needed, and re-hash
# the file now that it's been signed.
file_obj.update(
cert_serial_num=cert_serial_num,
hash=file_obj.generate_hash(),
is_signed=True,
size=size,
)
log.info('Signing complete for file {0}'.format(file_obj.pk))
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Schedule this version for git extraction.
transaction.on_commit(
lambda: create_git_extraction_entry(version=file_obj.version))
return file_obj
def from_upload(cls,
upload,
addon,
selected_apps,
channel,
parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of compatible app ids, a channel id and
the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
from olympia.addons.models import AddonReviewerFlags
from olympia.addons.utils import RestrictionChecker
from olympia.git.utils import create_git_extraction_entry
assert parsed_data is not None
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
if upload.addon and upload.addon != addon:
raise VersionCreateError(
'FileUpload was made for a different Addon')
if not upload.user or not upload.ip_address or not upload.source:
raise VersionCreateError(
'FileUpload does not have some required fields')
if not upload.user.last_login_ip or not upload.user.email:
raise VersionCreateError(
'FileUpload user does not have some required fields')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(channel=channel,
exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
approval_notes = None
if parsed_data.get('is_mozilla_signed_extension'):
approval_notes = (
'This version has been signed with Mozilla internal certificate.'
)
version = cls.objects.create(
addon=addon,
approval_notes=approval_notes,
version=parsed_data['version'],
license_id=license_id,
channel=channel,
)
email = upload.user.email if upload.user and upload.user.email else ''
with core.override_remote_addr(upload.ip_address):
# The following log statement is used by foxsec-pipeline.
# We override the IP because it might be called from a task and we
# want the original IP from the submitter.
log.info(
f'New version: {version!r} ({version.id}) from {upload!r}',
extra={
'email': email,
'guid': addon.guid,
'upload': upload.uuid.hex,
'user_id': upload.user_id,
'from_api': upload.source == amo.UPLOAD_SOURCE_API,
},
)
activity.log_create(amo.LOG.ADD_VERSION,
version,
addon,
user=upload.user or get_task_user())
if addon.type == amo.ADDON_STATICTHEME:
# We don't let developers select apps for static themes
selected_apps = [app.id for app in amo.APP_USAGE]
compatible_apps = {}
for app in parsed_data.get('apps', []):
if app.id not in selected_apps:
# If the user chose to explicitly deselect Firefox for Android
# we're not creating the respective `ApplicationsVersions`
# which will have this add-on then be listed only for
# Firefox specifically.
continue
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# Pre-generate _compatible_apps property to avoid accidentally
# triggering queries with that instance later.
version._compatible_apps = compatible_apps
# Create relevant file and update the all_files cached property on the
# Version, because we might need it afterwards.
version.all_files = [
File.from_upload(
upload=upload,
version=version,
parsed_data=parsed_data,
)
]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to its permanent location, delete it
# from storage. Keep the FileUpload instance (it gets cleaned up by a
# cron eventually some time after its creation, in amo.cron.gc()),
# making sure it's associated with the add-on instance.
storage.delete(upload.path)
upload.path = ''
if upload.addon is None:
upload.addon = addon
upload.save()
version_uploaded.send(instance=version, sender=Version)
if version.is_webextension:
if (waffle.switch_is_active('enable-yara')
or waffle.switch_is_active('enable-customs')
or waffle.switch_is_active('enable-wat')):
ScannerResult.objects.filter(upload_id=upload.id).update(
version=version)
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Schedule this version for git extraction.
transaction.on_commit(
lambda: create_git_extraction_entry(version=version))
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME
and channel == amo.RELEASE_CHANNEL_LISTED):
theme_data = parsed_data.get('theme', {})
generate_static_theme_preview(theme_data, version.pk)
# Reset add-on reviewer flags to disable auto-approval and require
# admin code review if the package has already been signed by mozilla.
reviewer_flags_defaults = {}
is_mozilla_signed = parsed_data.get('is_mozilla_signed_extension')
if upload.validation_timeout:
reviewer_flags_defaults['needs_admin_code_review'] = True
if is_mozilla_signed and addon.type != amo.ADDON_LPAPP:
reviewer_flags_defaults['needs_admin_code_review'] = True
reviewer_flags_defaults['auto_approval_disabled'] = True
# Check if the approval should be restricted
if not RestrictionChecker(upload=upload).is_auto_approval_allowed():
flag = ('auto_approval_disabled'
if channel == amo.RELEASE_CHANNEL_LISTED else
'auto_approval_disabled_unlisted')
reviewer_flags_defaults[flag] = True
if reviewer_flags_defaults:
AddonReviewerFlags.objects.update_or_create(
addon=addon, defaults=reviewer_flags_defaults)
# Authors need to be notified about auto-approval delay again since
# they are submitting a new version.
addon.reset_notified_about_auto_approval_delay()
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'.format(delta=upload_time,
version=version,
created=upload.created,
now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
def from_upload(cls, upload, addon, selected_apps, channel,
parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of compatible app ids, a channel id and
the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
from olympia.git.utils import create_git_extraction_entry
assert parsed_data is not None
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(
channel=channel, exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
approval_notes = None
if parsed_data.get('is_mozilla_signed_extension'):
approval_notes = (u'This version has been signed with '
u'Mozilla internal certificate.')
version = cls.objects.create(
addon=addon,
approval_notes=approval_notes,
version=parsed_data['version'],
license_id=license_id,
channel=channel,
)
email = upload.user.email if upload.user and upload.user.email else ''
with core.override_remote_addr(upload.ip_address):
log.info(
'New version: %r (%s) from %r' % (version, version.id, upload),
extra={
'email': email,
'guid': addon.guid,
'upload': upload.uuid.hex,
'user_id': upload.user_id,
'from_api': upload.source == amo.UPLOAD_SOURCE_API,
}
)
activity.log_create(
amo.LOG.ADD_VERSION, version, addon,
user=upload.user or get_task_user())
if addon.type == amo.ADDON_STATICTHEME:
# We don't let developers select apps for static themes
selected_apps = [app.id for app in amo.APP_USAGE]
compatible_apps = {}
for app in parsed_data.get('apps', []):
if app.id not in selected_apps:
# If the user chose to explicitly deselect Firefox for Android
# we're not creating the respective `ApplicationsVersions`
# which will have this add-on then be listed only for
# Firefox specifically.
continue
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# See #2828: sometimes when we generate the filename(s) below, in
# File.from_upload(), cache-machine is confused and has trouble
# fetching the ApplicationsVersions that were just created. To work
# around this we pre-generate version.compatible_apps and avoid the
# queries completely.
version._compatible_apps = compatible_apps
# For backwards compatibility. We removed specific platform
# support during submission but we don't handle it any different
# beyond that yet. That means, we're going to simply set it
# to `PLATFORM_ALL` and also have the backend create separate
# files for each platform. Cleaning that up is another step.
# Given the timing on this, we don't care about updates to legacy
# add-ons as well.
# Create relevant file and update the all_files cached property on the
# Version, because we might need it afterwards.
version.all_files = [File.from_upload(
upload=upload, version=version, platform=amo.PLATFORM_ALL.id,
parsed_data=parsed_data
)]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to all platforms, remove the upload.
storage.delete(upload.path)
upload.path = ''
upload.save()
version_uploaded.send(instance=version, sender=Version)
if version.is_webextension:
if (
waffle.switch_is_active('enable-yara') or
waffle.switch_is_active('enable-customs') or
waffle.switch_is_active('enable-wat')
):
ScannerResult.objects.filter(upload_id=upload.id).update(
version=version)
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Schedule this version for git extraction.
transaction.on_commit(
lambda: create_git_extraction_entry(version=version)
)
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME and
channel == amo.RELEASE_CHANNEL_LISTED):
theme_data = parsed_data.get('theme', {})
generate_static_theme_preview(theme_data, version.pk)
# Authors need to be notified about auto-approval delay again since
# they are submitting a new version.
addon.reset_notified_about_auto_approval_delay()
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'
.format(delta=upload_time, version=version,
created=upload.created, now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
def sign_file(file_obj):
"""Sign a File if necessary.
If it's not necessary (file exists but it's a mozilla signed one) then
return the file directly.
If there's no endpoint (signing is not enabled) or isn't reviewed yet,
or there was an error while signing, raise an exception - it
shouldn't happen.
Otherwise proceed with signing and return the signed file.
"""
from olympia.git.utils import create_git_extraction_entry
if not settings.ENABLE_ADDON_SIGNING:
raise SigningError(
f'Not signing file {file_obj.pk}: no active endpoint')
# No file? No signature.
if not os.path.exists(file_obj.current_file_path):
raise SigningError(
f"File {file_obj.current_file_path} doesn't exist on disk")
# Don't sign Mozilla signed extensions (they're already signed).
if file_obj.is_mozilla_signed_extension:
# Don't raise an exception here, just log and return file_obj even
# though we didn't sign, it's not an error - we just don't need to do
# anything in this case.
log.info('Not signing file {}: mozilla signed extension is already '
'signed'.format(file_obj.pk))
return file_obj
# We only sign files that are compatible with Firefox.
if not supports_firefox(file_obj):
raise SigningError(
'Not signing version {}: not for a Firefox version we support'.
format(file_obj.version.pk))
# Get the path before call_signing modifies it... We'll delete it after if
# signing was successful and we ended up changing it.
old_path = file_obj.current_file_path
# Sign the file. If there's any exception, we skip the rest.
cert_serial_num = str(call_signing(file_obj))
size = storage.size(file_obj.current_file_path)
# Save the certificate serial number for revocation if needed, change the
# filename to use a .xpi extension (cachebusting anything that depends on
# the filename with the old .zip extension) and re-hash the file now that
# it's been signed.
file_obj.update(
cert_serial_num=cert_serial_num,
hash=file_obj.generate_hash(),
size=size,
# We-specify filename and is_signed that we already updated on the
# instance without saving, otherwise those wouldn't get updated.
filename=file_obj.filename,
is_signed=file_obj.is_signed,
)
log.info(f'Signing complete for file {file_obj.pk}')
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Schedule this version for git extraction.
transaction.on_commit(
lambda: create_git_extraction_entry(version=file_obj.version))
# Remove old unsigned path if necessary.
if old_path != file_obj.current_file_path:
storage.delete(old_path)
return file_obj
