def test_call_signing(self):
packaged.sign_file(self.file_)
signature_info = self._get_signature_info()
subject_info = signature_info.signer_certificate['subject']
assert subject_info['common_name'] == 'xxxxx'
def process_public(self, auto_validation=False):
"""Set an addon to public."""
# Sign addon.
for file_ in self.files:
sign_file(file_, settings.SIGNING_SERVER)
# Hold onto the status before we change it.
status = self.addon.status
# Save files first, because set_addon checks to make sure there
# is at least one public file or it won't make the addon public.
self.set_files(amo.STATUS_PUBLIC, self.files, copy_to_mirror=True)
self.set_addon(status=amo.STATUS_PUBLIC)
self.log_action(amo.LOG.APPROVE_VERSION)
template = u'%s_to_public' % self.review_type
subject = u'Mozilla Add-ons: %s %s Approved'
if not self.addon.is_listed:
template = u'unlisted_to_reviewed'
if auto_validation:
template = u'unlisted_to_reviewed_auto'
subject = u'Mozilla Add-ons: %s %s signed and ready to download'
self.notify_email(template, subject)
log.info(u'Making %s public' % (self.addon))
log.info(u'Sending email for %s' % (self.addon))
# Assign reviewer incentive scores.
if self.request and not auto_validation:
ReviewerScore.award_points(self.request.user, self.addon, status)
def process_public(self, auto_validation=False):
"""Set an addons files to public."""
if self.review_type == 'preliminary':
raise AssertionError('Preliminary addons cannot be made public.')
# Sign addon.
for file_ in self.files:
sign_file(file_, settings.SIGNING_SERVER)
# Hold onto the status before we change it.
status = self.addon.status
self.set_files(amo.STATUS_PUBLIC, self.files, copy_to_mirror=True)
self.log_action(amo.LOG.APPROVE_VERSION)
template = u'%s_to_public' % self.review_type
subject = u'Mozilla Add-ons: %s %s Fully Reviewed'
if not self.addon.is_listed:
template = u'unlisted_to_reviewed'
if auto_validation:
template = u'unlisted_to_reviewed_auto'
subject = u'Mozilla Add-ons: %s %s signed and ready to download'
self.notify_email(template, subject)
log.info(u'Making %s files %s public' %
(self.addon, ', '.join([f.filename for f in self.files])))
log.info(u'Sending email for %s' % (self.addon))
# Assign reviewer incentive scores.
if self.request and not auto_validation:
ReviewerScore.award_points(self.request.user, self.addon, status)
def process_preliminary(self, auto_validation=False):
"""Set an addons files to preliminary."""
# Sign addon.
for file_ in self.files:
sign_file(file_, settings.PRELIMINARY_SIGNING_SERVER)
# Hold onto the status before we change it.
status = self.addon.status
self.set_files(amo.STATUS_LITE, self.files, copy_to_mirror=True)
self.log_action(amo.LOG.PRELIMINARY_VERSION)
template = u'%s_to_preliminary' % self.review_type
subject = u'Mozilla Add-ons: %s %s Preliminary Reviewed'
if not self.addon.is_listed:
template = u'unlisted_to_reviewed'
if auto_validation:
template = u'unlisted_to_reviewed_auto'
subject = u'Mozilla Add-ons: %s %s signed and ready to download'
self.notify_email(template, subject)
log.info(u'Making %s files %s preliminary' %
(self.addon, ', '.join([f.filename for f in self.files])))
log.info(u'Sending email for %s' % (self.addon))
if self.request and not auto_validation:
# Assign reviewer incentive scores.
ReviewerScore.award_points(self.request.user, self.addon, status)
def test_sign_file_non_ascii_filename(self):
src = self.file_.file_path
self.file_.update(filename=u'jétpack.xpi')
shutil.move(src, self.file_.file_path)
self.assert_not_signed()
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
self.assert_signed()
def migrate_legacy_dictionary_to_webextension(addon):
"""Migrate a single legacy dictionary to webextension format, creating a
new package from the current_version, faking an upload to create a new
Version instance."""
user = UserProfile.objects.get(pk=settings.TASK_USER_ID)
now = datetime.now()
# Wrap zip in FileUpload for Version.from_upload() to consume.
upload = FileUpload.objects.create(
user=user, valid=True)
destination = os.path.join(
user_media_path('addons'), 'temp', uuid.uuid4().hex + '.xpi')
target_language = build_webext_dictionary_from_legacy(addon, destination)
if not addon.target_locale:
addon.update(target_locale=target_language)
upload.update(path=destination)
parsed_data = parse_addon(upload, addon=addon, user=user)
# Create version.
# WebExtension dictionaries are only compatible with Firefox Desktop
# Firefox for Android uses the OS spellchecking.
version = Version.from_upload(
upload, addon, selected_apps=[amo.FIREFOX.id],
channel=amo.RELEASE_CHANNEL_LISTED, parsed_data=parsed_data)
activity.log_create(amo.LOG.ADD_VERSION, version, addon, user=user)
# Sign the file, and set it to public. That should automatically set
# current_version to the version we created.
file_ = version.all_files[0]
sign_file(file_)
file_.update(datestatuschanged=now, reviewed=now, status=amo.STATUS_PUBLIC)
def process_public(self):
"""Set an addons files to public."""
assert not self.version or (
self.version.channel == amo.RELEASE_CHANNEL_LISTED)
# Sign addon.
for file_ in self.files:
sign_file(file_, settings.SIGNING_SERVER)
# Hold onto the status before we change it.
status = self.addon.status
self.set_files(amo.STATUS_PUBLIC, self.files)
self.log_action(amo.LOG.APPROVE_VERSION)
template = u'%s_to_public' % self.review_type
subject = u'Mozilla Add-ons: %s %s Approved'
self.notify_email(template, subject)
log.info(u'Making %s files %s public' %
(self.addon, ', '.join([f.filename for f in self.files])))
log.info(u'Sending email for %s' % (self.addon))
# Assign reviewer incentive scores.
if self.request:
ReviewerScore.award_points(self.request.user, self.addon, status)
def test_supports_firefox_old_not_default_to_compatible(self):
max_appversion = self.version.apps.first().max
# Old, and not default to compatible.
max_appversion.update(version='4', version_int=version_int('4'))
self.file_.update(binary_components=True, strict_compatibility=True)
self.assert_not_signed()
packaged.sign_file(self.file_)
self.assert_signed()
def test_supports_firefox_recent_default_to_compatible(self):
max_appversion = self.version.apps.first().max
# Recent, default to compatible.
max_appversion.update(version='37', version_int=version_int('37'))
self.file_.update(binary_components=False, strict_compatibility=False)
self.assert_not_signed()
packaged.sign_file(self.file_)
self.assert_signed()
def test_sign_file(self):
self.assert_not_signed()
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
self.assert_signed()
# Make sure there's two newlines at the end of the mozilla.sf file (see
# bug 1158938).
with zipfile.ZipFile(self.file_.file_path, mode='r') as zf:
with zf.open('META-INF/mozilla.sf', 'r') as mozillasf:
assert mozillasf.read().endswith('\n\n')
def test_call_signing_too_long_guid_bug_1203365(self):
long_guid = 'x' * 65
hashed = hashlib.sha256(long_guid).hexdigest()
self.addon.update(guid=long_guid)
packaged.sign_file(self.file_)
signature_info = self._get_signature_info()
subject_info = signature_info.signer_certificate['subject']
assert subject_info['common_name'] == hashed
def test_no_sign_missing_file(self):
os.unlink(self.file_.file_path)
assert not self.file_.is_signed
assert not self.file_.cert_serial_num
assert not self.file_.hash
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
assert not self.file_.is_signed
assert not self.file_.cert_serial_num
assert not self.file_.hash
assert not packaged.is_signed(self.file_.file_path)
def test_supports_firefox_android_old_default_to_compatible(self):
max_appversion = self.version.apps.first().max
# Old, and default to compatible.
max_appversion.update(application=amo.ANDROID.id,
version='4', version_int=version_int('4'))
self.file_.update(binary_components=False, strict_compatibility=False)
self.assert_not_signed()
packaged.sign_file(self.file_)
self.assert_signed()
def test_supports_firefox_android_recent_not_default_to_compatible(self):
max_appversion = self.version.apps.first().max
# Recent, not default to compatible.
max_appversion.update(application=amo.ANDROID.id,
version='37', version_int=version_int('37'))
self.file_.update(binary_components=True, strict_compatibility=True)
self.assert_not_signed()
packaged.sign_file(self.file_)
self.assert_signed()
def process_public(self):
"""Set an add-on or a version to public."""
# Safeguard to force implementation for unlisted add-ons to completely
# override this method.
assert self.version.channel == amo.RELEASE_CHANNEL_LISTED
# Safeguard to make sure this action is not used for content review
# (it should use confirm_auto_approved instead).
assert not self.content_review_only
# Sign addon.
for file_ in self.files:
sign_file(file_)
# Hold onto the status before we change it.
status = self.addon.status
# Save files first, because set_addon checks to make sure there
# is at least one public file or it won't make the addon public.
self.set_files(amo.STATUS_PUBLIC, self.files)
if self.set_addon_status:
self.set_addon(status=amo.STATUS_PUBLIC)
# If we've approved a webextension, add a tag identifying them as such.
if any(file_.is_webextension for file_ in self.files):
Tag(tag_text='firefox57').save_tag(self.addon)
# If we've approved a mozilla signed add-on, add the firefox57 tag
if all(file_.is_mozilla_signed_extension for file_ in self.files):
Tag(tag_text='firefox57').save_tag(self.addon)
# Increment approvals counter if we have a request (it means it's a
# human doing the review) otherwise reset it as it's an automatic
# approval.
if self.request:
AddonApprovalsCounter.increment_for_addon(addon=self.addon)
else:
AddonApprovalsCounter.reset_for_addon(addon=self.addon)
self.log_action(amo.LOG.APPROVE_VERSION)
template = u'%s_to_public' % self.review_type
if self.review_type == 'pending':
subject = u'Mozilla Add-ons: %s %s Updated'
else:
subject = u'Mozilla Add-ons: %s %s Approved'
self.notify_email(template, subject)
self.log_public_message()
log.info(u'Sending email for %s' % (self.addon))
# Assign reviewer incentive scores.
if self.request:
ReviewerScore.award_points(
self.request.user, self.addon, status, version=self.version)
def test_supports_firefox_old_not_default_to_compatible(self):
max_appversion = self.version.apps.first().max
# Old, and not default to compatible.
max_appversion.update(version=settings.MIN_D2C_VERSION,
version_int=version_int(
settings.MIN_D2C_VERSION))
self.file_.update(binary_components=True, strict_compatibility=True)
self.assert_not_signed()
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
self.assert_not_signed()
def test_supports_firefox_android_recent_not_default_to_compatible(self):
max_appversion = self.version.apps.first().max
# Recent, not default to compatible.
max_appversion.update(application=amo.ANDROID.id,
version=settings.MIN_NOT_D2C_VERSION,
version_int=version_int(
settings.MIN_NOT_D2C_VERSION))
self.file_.update(binary_components=True, strict_compatibility=True)
self.assert_not_signed()
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
self.assert_signed()
def test_call_signing(self):
packaged.sign_file(self.file_)
signature_info, manifest = self._get_signature_details()
subject_info = signature_info.signer_certificate['subject']
assert subject_info['common_name'] == 'xxxxx'
assert manifest == (
'Manifest-Version: 1.0\n\n'
'Name: install.rdf\n'
'Digest-Algorithms: MD5 SHA1 SHA256\n'
'MD5-Digest: AtjchjiOU/jDRLwMx214hQ==\n'
'SHA1-Digest: W9kwfZrvMkbgjOx6nDdibCNuCjk=\n'
'SHA256-Digest: 3Wjjho1pKD/9VaK+FszzvZFN/2crBmaWbdisLovwo6g=\n\n'
)
def process_public(self):
"""Set an unliste addon version files to public."""
assert self.version.channel == amo.RELEASE_CHANNEL_UNLISTED
# Sign addon.
for file_ in self.files:
sign_file(file_)
self.set_files(amo.STATUS_PUBLIC, self.files)
template = u'unlisted_to_reviewed_auto'
subject = u'Mozilla Add-ons: %s %s signed and ready to download'
self.log_action(amo.LOG.APPROVE_VERSION)
self.notify_email(template, subject)
log.info(u'Making %s files %s public' %
(self.addon, ', '.join([f.filename for f in self.files])))
log.info(u'Sending email for %s' % (self.addon))
def test_call_signing_too_long_guid_bug_1203365(self):
long_guid = 'x' * 65
hashed = hashlib.sha256(long_guid).hexdigest()
self.addon.update(guid=long_guid)
packaged.sign_file(self.file_)
signature_info, manifest = self._get_signature_details()
subject_info = signature_info.signer_certificate['subject']
assert subject_info['common_name'] == hashed
assert manifest == (
'Manifest-Version: 1.0\n\n'
'Name: install.rdf\n'
'Digest-Algorithms: MD5 SHA1 SHA256\n'
'MD5-Digest: AtjchjiOU/jDRLwMx214hQ==\n'
'SHA1-Digest: W9kwfZrvMkbgjOx6nDdibCNuCjk=\n'
'SHA256-Digest: 3Wjjho1pKD/9VaK+FszzvZFN/2crBmaWbdisLovwo6g=\n\n'
)
def test_sign_addon_with_unicode_guid(self):
self.addon.update(guid=u'NavratnePeniaze@NávratnéPeniaze')
packaged.sign_file(self.file_)
signature_info, manifest = self._get_signature_details()
subject_info = signature_info.signer_certificate['subject']
assert (
subject_info['common_name'] ==
u'NavratnePeniaze@NávratnéPeniaze')
assert manifest == (
'Manifest-Version: 1.0\n\n'
'Name: install.rdf\n'
'Digest-Algorithms: MD5 SHA1 SHA256\n'
'MD5-Digest: AtjchjiOU/jDRLwMx214hQ==\n'
'SHA1-Digest: W9kwfZrvMkbgjOx6nDdibCNuCjk=\n'
'SHA256-Digest: 3Wjjho1pKD/9VaK+FszzvZFN/2crBmaWbdisLovwo6g=\n\n')
def test_sign_file_multi_package(self):
fpath = 'src/olympia/files/fixtures/files/multi-package.xpi'
with amo.tests.copy_file(fpath, self.file_.file_path, overwrite=True):
self.file_.update(is_multi_package=True)
self.assert_not_signed()
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
self.assert_not_signed()
# The multi-package itself isn't signed.
assert not packaged.is_signed(self.file_.file_path)
# The internal extensions aren't either.
folder = tempfile.mkdtemp()
try:
extract_xpi(self.file_.file_path, folder)
# The extension isn't.
assert not packaged.is_signed(
os.path.join(folder, 'random_extension.xpi'))
# And the theme isn't either.
assert not packaged.is_signed(
os.path.join(folder, 'random_theme.xpi'))
finally:
amo.utils.rm_local_tmp_dir(folder)
def test_sign_file_multi_package(self):
fpath = 'src/olympia/files/fixtures/files/multi-package.xpi'
with amo.tests.copy_file(fpath, self.file_.file_path, overwrite=True):
self.file_.update(is_multi_package=True)
self.assert_not_signed()
packaged.sign_file(self.file_)
self.assert_not_signed()
# The multi-package itself isn't signed.
assert not packaged.is_signed(self.file_.file_path)
# The internal extensions aren't either.
folder = tempfile.mkdtemp(dir=settings.TMP_PATH)
try:
extract_xpi(self.file_.file_path, folder)
# The extension isn't.
assert not packaged.is_signed(
os.path.join(folder, 'random_extension.xpi'))
# And the theme isn't either.
assert not packaged.is_signed(
os.path.join(folder, 'random_theme.xpi'))
finally:
amo.utils.rm_local_tmp_dir(folder)
def test_call_signing(self):
assert packaged.sign_file(self.file_)
signature_info, manifest = self._get_signature_details()
subject_info = signature_info.signer_certificate['subject']
assert subject_info['common_name'] == 'xxxxx'
assert manifest == (
'Manifest-Version: 1.0\n\n'
'Name: install.rdf\n'
'Digest-Algorithms: MD5 SHA1 SHA256\n'
'MD5-Digest: AtjchjiOU/jDRLwMx214hQ==\n'
'SHA1-Digest: W9kwfZrvMkbgjOx6nDdibCNuCjk=\n'
'SHA256-Digest: 3Wjjho1pKD/9VaK+FszzvZFN/2crBmaWbdisLovwo6g=\n\n')
def process_preliminary(self, auto_validation=False):
"""Set an addon to preliminary."""
# Sign addon.
for file_ in self.files:
sign_file(file_, settings.PRELIMINARY_SIGNING_SERVER)
# Hold onto the status before we change it.
status = self.addon.status
changes = {'status': amo.STATUS_LITE}
if (self.addon.status
in (amo.STATUS_PUBLIC, amo.STATUS_LITE_AND_NOMINATED)):
changes['highest_status'] = amo.STATUS_LITE
template = u'%s_to_preliminary' % self.review_type
subject = u'Mozilla Add-ons: %s %s Preliminary Reviewed'
if (self.review_type == 'preliminary'
and self.addon.status == amo.STATUS_LITE_AND_NOMINATED):
template = u'nominated_to_nominated'
if not self.addon.is_listed:
template = u'unlisted_to_reviewed'
if auto_validation:
template = u'unlisted_to_reviewed_auto'
subject = u'Mozilla Add-ons: %s %s signed and ready to download'
self.set_addon(**changes)
self.set_files(amo.STATUS_LITE, self.files, copy_to_mirror=True)
self.log_action(amo.LOG.PRELIMINARY_VERSION)
self.notify_email(template, subject)
log.info(u'Making %s preliminary' % (self.addon))
log.info(u'Sending email for %s' % (self.addon))
if self.request and not auto_validation:
# Assign reviewer incentive scores.
ReviewerScore.award_points(self.request.user, self.addon, status)
def resign_files(file_ids, **kw):
"""Used to force re-signing the files.
This is used in the 'resign_files' management command. We need to
re-sign the files that were signed with the wrong certificate (see bug
1225036).
It also bumps the version number of the file and the Version, so the
Firefox extension update mecanism picks this new signed version and
installs it.
"""
log.info(u'[{0}] Signing files.'.format(len(file_ids)))
addons_emailed = set()
for file_ in File.objects.filter(id__in=file_ids):
# We only resign files that have been reviewed and signed.
if file_.status not in amo.REVIEWED_STATUSES:
log.info(u'Not re-signing file {0}, not reviewed'.format(file_.pk))
continue
if not file_.is_signed:
log.info(u'Not re-signing file {0}, not signed'.format(file_.pk))
continue
log.info(u'Re-signing file {0}'.format(file_.pk))
bumped_version_number = u'{0}.1-signed'.format(file_.version.version)
bump_version = False # Did we resign the file?
if not os.path.isfile(file_.file_path):
log.info(u'File {0} does not exist, skip'.format(file_.pk))
continue
# Save the original file, before bumping the version.
backup_path = u'{0}.backup_signature'.format(file_.file_path)
shutil.copy(file_.file_path, backup_path)
try:
# Need to bump the version (modify manifest file)
# before the file is signed.
update_version_number(file_, bumped_version_number)
if file_.status == amo.STATUS_PUBLIC:
server = settings.SIGNING_SERVER
else:
server = settings.PRELIMINARY_SIGNING_SERVER
signed = bool(sign_file(file_, server))
if signed: # Bump the version number if at least one signed.
bump_version = True
else: # We didn't sign, so revert the version bump.
shutil.move(backup_path, file_.file_path)
except:
log.error(u'Failed signing file {0}'.format(file_.pk),
exc_info=True)
# Revert the version bump, restore the backup.
shutil.move(backup_path, file_.file_path)
# Now update the Version model, if we resigned the file.
if bump_version:
version = file_.version
version.update(version=bumped_version_number,
version_int=version_int(bumped_version_number))
addon = version.addon
if addon.pk not in addons_emailed:
# Send a mail to the owners/devs warning them we've
# automatically signed their addon.
qs = (AddonUser.objects.filter(
role=amo.AUTHOR_ROLE_OWNER,
addon=addon).exclude(user__email=None))
emails = qs.values_list('user__email', flat=True)
subject = MAIL_RESIGN_SUBJECT.format(addon=addon.name)
message = MAIL_RESIGN_MESSAGE.format(
addon=addon.name,
addon_url=amo.helpers.absolutify(
addon.get_dev_url(action='versions')))
amo.utils.send_mail(
subject,
message,
recipient_list=emails,
fail_silently=True,
headers={'Reply-To': '*****@*****.**'})
addons_emailed.add(addon.pk)
if static_category:
category = Category.from_static_category(static_category, True)
AddonCategory.objects.get_or_create(addon=addon,
category=category)
# Add a license if there isn't one already
if not version.license:
license = License.objects.builtins().get(builtin=1)
version.update(license=license)
file_ = version.files.get()
if not is_beta:
# Not `version.files.update`, because we need to trigger save
# hooks.
file_.update(status=amo.STATUS_PUBLIC)
sign_file(file_)
# Finally, set the addon summary if one wasn't provided in the xpi.
addon.status = amo.STATUS_PUBLIC
addon.summary = addon.summary if addon.summary else unicode(addon.name)
addon.save(update_fields=('status', 'summary'))
addon.update_status()
@task
def celery_error(**kw):
"""
This task raises an exception from celery to test error logging and
Sentry hookup.
"""
log.info('about to raise an exception from celery')
def sign_addons(addon_ids, force=False, **kw):
"""Used to sign all the versions of an addon.
This is used in the 'sign_addons' and 'process_addons --task sign_addons'
management commands.
It also bumps the version number of the file and the Version, so the
Firefox extension update mechanism picks this new signed version and
installs it.
"""
log.info(u'[{0}] Signing addons.'.format(len(addon_ids)))
reasons = {
'default': [MAIL_SUBJECT, MAIL_MESSAGE],
'expiry': [MAIL_EXPIRY_SUBJECT, MAIL_EXPIRY_MESSAGE]
}
mail_subject, mail_message = reasons[kw.get('reason', 'default')]
def file_supports_firefox(version):
"""Return a Q object: files supporting at least a firefox version."""
return Q(version__apps__max__application__in=SIGN_FOR_APPS,
version__apps__max__version_int__gte=version_int(version))
is_default_compatible = Q(binary_components=False,
strict_compatibility=False)
# We only want to sign files that are at least compatible with Firefox
# MIN_D2C_VERSION, or Firefox MIN_NOT_D2C_VERSION if they are not default
# to compatible.
# The signing feature should be supported from Firefox 40 and above, but
# we're still signing some files that are a bit older just in case.
ff_version_filter = (
(is_default_compatible &
file_supports_firefox(settings.MIN_D2C_VERSION)) |
(~is_default_compatible &
file_supports_firefox(settings.MIN_NOT_D2C_VERSION)))
addons_emailed = set()
# We only care about extensions.
for version in Version.objects.filter(addon_id__in=addon_ids,
addon__type=amo.ADDON_EXTENSION):
# We only sign files that have been reviewed and are compatible with
# versions of Firefox that are recent enough.
to_sign = version.files.filter(ff_version_filter,
status__in=amo.REVIEWED_STATUSES)
if force:
to_sign = to_sign.all()
else:
to_sign = to_sign.filter(is_signed=False)
if not to_sign:
log.info(u'Not signing addon {0}, version {1} (no files or already'
u' signed)'.format(version.addon, version))
log.info(u'Signing addon {0}, version {1}'.format(version.addon,
version))
bumped_version_number = get_new_version_number(version.version)
signed_at_least_a_file = False # Did we sign at least one file?
for file_obj in to_sign:
if not os.path.isfile(file_obj.file_path):
log.info(u'File {0} does not exist, skip'.format(file_obj.pk))
continue
# Save the original file, before bumping the version.
backup_path = u'{0}.backup_signature'.format(file_obj.file_path)
shutil.copy(file_obj.file_path, backup_path)
try:
# Need to bump the version (modify manifest file)
# before the file is signed.
update_version_number(file_obj, bumped_version_number)
server = settings.SIGNING_SERVER
signed = bool(sign_file(file_obj, server))
if signed: # Bump the version number if at least one signed.
signed_at_least_a_file = True
else: # We didn't sign, so revert the version bump.
shutil.move(backup_path, file_obj.file_path)
except:
log.error(u'Failed signing file {0}'.format(file_obj.pk),
exc_info=True)
# Revert the version bump, restore the backup.
shutil.move(backup_path, file_obj.file_path)
# Now update the Version model, if we signed at least one file.
if signed_at_least_a_file:
version.update(version=bumped_version_number,
version_int=version_int(bumped_version_number))
addon = version.addon
if addon.pk not in addons_emailed:
# Send a mail to the owners/devs warning them we've
# automatically signed their addon.
qs = (AddonUser.objects
.filter(role=amo.AUTHOR_ROLE_OWNER, addon=addon)
.exclude(user__email__isnull=True))
emails = qs.values_list('user__email', flat=True)
subject = mail_subject.format(addon=addon.name)
message = mail_message.format(
addon=addon.name,
addon_url=amo.helpers.absolutify(
addon.get_dev_url(action='versions')))
amo.utils.send_mail(
subject, message, recipient_list=emails,
fail_silently=True,
headers={'Reply-To': '*****@*****.**'})
addons_emailed.add(addon.pk)
def add_static_theme_from_lwt(lwt):
from olympia.activity.models import AddonLog
olympia.core.set_user(UserProfile.objects.get(pk=settings.TASK_USER_ID))
# Try to handle LWT with no authors
author = (lwt.listed_authors or [_get_lwt_default_author()])[0]
# Wrap zip in FileUpload for Addon/Version from_upload to consume.
upload = FileUpload.objects.create(user=author, valid=True)
destination = os.path.join(user_media_path('addons'), 'temp',
uuid.uuid4().hex + '.xpi')
build_static_theme_xpi_from_lwt(lwt, destination)
upload.update(path=destination)
# Create addon + version
parsed_data = parse_addon(upload, user=author)
addon = Addon.initialize_addon_from_upload(parsed_data, upload,
amo.RELEASE_CHANNEL_LISTED,
author)
addon_updates = {}
# static themes are only compatible with Firefox at the moment,
# not Android
version = Version.from_upload(upload,
addon,
selected_apps=[amo.FIREFOX.id],
channel=amo.RELEASE_CHANNEL_LISTED,
parsed_data=parsed_data)
# Set category
static_theme_categories = CATEGORIES.get(amo.FIREFOX.id,
[]).get(amo.ADDON_STATICTHEME, [])
lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat.
lwt_category_slug = lwt_category.slug if lwt_category else 'other'
static_category = static_theme_categories.get(
lwt_category_slug, static_theme_categories.get('other'))
AddonCategory.objects.create(addon=addon,
category=Category.from_static_category(
static_category, True))
# Set license
lwt_license = PERSONA_LICENSES_IDS.get(
lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright
static_license = License.objects.get(builtin=lwt_license.builtin)
version.update(license=static_license)
# Set tags
for addon_tag in AddonTag.objects.filter(addon=lwt):
AddonTag.objects.create(addon=addon, tag=addon_tag.tag)
# Steal the ratings (even with soft delete they'll be deleted anyway)
addon_updates.update(average_rating=lwt.average_rating,
bayesian_rating=lwt.bayesian_rating,
total_ratings=lwt.total_ratings,
text_ratings_count=lwt.text_ratings_count)
Rating.unfiltered.filter(addon=lwt).update(addon=addon, version=version)
# Modify the activity log entry too.
rating_activity_log_ids = [
l.id for l in amo.LOG if getattr(l, 'action_class', '') == 'review'
]
addonlog_qs = AddonLog.objects.filter(
addon=lwt, activity_log__action__in=rating_activity_log_ids)
[alog.transfer(addon) for alog in addonlog_qs.iterator()]
# Copy the ADU statistics - the raw(ish) daily UpdateCounts for stats
# dashboard and future update counts, and copy the summary numbers for now.
migrate_theme_update_count(lwt, addon)
addon_updates.update(average_daily_users=lwt.persona.popularity or 0,
hotness=lwt.persona.movers or 0)
# Logging
activity.log_create(amo.LOG.CREATE_STATICTHEME_FROM_PERSONA,
addon,
user=author)
# And finally sign the files (actually just one)
for file_ in version.all_files:
sign_file(file_)
file_.update(datestatuschanged=lwt.last_updated,
reviewed=datetime.now(),
status=amo.STATUS_PUBLIC)
addon_updates['status'] = amo.STATUS_PUBLIC
# set the modified and creation dates to match the original.
addon_updates['created'] = lwt.created
addon_updates['modified'] = lwt.modified
addon_updates['last_updated'] = lwt.last_updated
addon.update(**addon_updates)
return addon
def test_is_signed(self):
assert not packaged.is_signed(self.file_.file_path)
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
assert packaged.is_signed(self.file_.file_path)
if static_category:
category = Category.from_static_category(static_category, True)
AddonCategory.objects.get_or_create(addon=addon,
category=category)
# Add a license if there isn't one already
if not version.license:
license = License.objects.builtins().get(builtin=1)
version.update(license=license)
file_ = version.files.get()
if not is_beta:
# Not `version.files.update`, because we need to trigger save
# hooks.
file_.update(status=amo.STATUS_PUBLIC)
sign_file(file_, settings.SIGNING_SERVER)
# Finally, set the addon summary if one wasn't provided in the xpi.
addon.status = amo.STATUS_PUBLIC
addon.summary = addon.summary if addon.summary else unicode(addon.name)
addon.save(update_fields=('status', 'summary'))
addon.update_status()
@task
def celery_error(**kw):
"""
This task raises an exception from celery to test error logging and
Sentry hookup.
"""
log.info('about to raise an exception from celery')
def test_no_server_full(self):
with self.settings(SIGNING_SERVER=''):
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
self.assert_not_signed()
def test_is_signed(self):
assert not packaged.is_signed(self.file_.file_path)
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
assert packaged.is_signed(self.file_.file_path)
AddonUser(addon=addon, user=owner).save()
version = addon.versions.get()
addon.status = amo.STATUS_PUBLIC
if addon.default_locale.lower() == lang.lower():
addon.target_locale = addon.default_locale
addon.save()
log.info(
'[@None] Created new "{0}" language pack, version {1}'.format(
xpi, data['version']))
file_ = version.files.get()
if is_beta:
sign_file(file_, settings.PRELIMINARY_SIGNING_SERVER)
else:
# Not `version.files.update`, because we need to trigger save
# hooks.
file_.update(status=amo.STATUS_PUBLIC)
sign_file(file_, settings.SIGNING_SERVER)
addon.update_version()
@task
def celery_error(**kw):
"""
This task raises an exception from celery to test error logging and
Sentry hookup.
"""
def test_no_server_full(self):
with self.settings(SIGNING_SERVER=''):
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
self.assert_not_signed()
def test_size_updated(self):
unsigned_size = storage.size(self.file_.file_path)
packaged.sign_file(self.file_)
signed_size = storage.size(self.file_.file_path)
assert self.file_.size == signed_size
assert unsigned_size < signed_size
def test_no_server_prelim(self):
with self.settings(PRELIMINARY_SIGNING_SERVER=''):
packaged.sign_file(self.file_, settings.PRELIMINARY_SIGNING_SERVER)
self.assert_not_signed()
def test_size_updated(self):
unsigned_size = storage.size(self.file_.file_path)
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
signed_size = storage.size(self.file_.file_path)
assert self.file_.size == signed_size
assert unsigned_size < signed_size
def _sign_file(self, file_):
packaged.sign_file(file_, use_autograph=True)
def _sign_file(self, file_):
packaged.sign_file(file_)
def test_no_server_prelim(self):
with self.settings(PRELIMINARY_SIGNING_SERVER=''):
packaged.sign_file(self.file_, settings.PRELIMINARY_SIGNING_SERVER)
self.assert_not_signed()
def add_static_theme_from_lwt(lwt):
from olympia.activity.models import AddonLog
# Try to handle LWT with no authors
author = (lwt.listed_authors or [_get_lwt_default_author()])[0]
# Wrap zip in FileUpload for Addon/Version from_upload to consume.
upload = FileUpload.objects.create(user=author, valid=True)
destination = os.path.join(user_media_path('addons'), 'temp',
uuid.uuid4().hex + '.xpi')
build_static_theme_xpi_from_lwt(lwt, destination)
upload.update(path=destination)
# Create addon + version
parsed_data = parse_addon(upload, user=author)
addon = Addon.initialize_addon_from_upload(parsed_data, upload,
amo.RELEASE_CHANNEL_LISTED,
author)
addon_updates = {}
# Version.from_upload sorts out platforms for us.
version = Version.from_upload(upload,
addon,
platforms=None,
channel=amo.RELEASE_CHANNEL_LISTED,
parsed_data=parsed_data)
# Set category
static_theme_categories = CATEGORIES.get(amo.FIREFOX.id,
[]).get(amo.ADDON_STATICTHEME, [])
lwt_category = (lwt.categories.all() or [None])[0] # lwt only have 1 cat.
lwt_category_slug = lwt_category.slug if lwt_category else 'other'
static_category = static_theme_categories.get(
lwt_category_slug, static_theme_categories.get('other'))
AddonCategory.objects.create(addon=addon,
category=Category.from_static_category(
static_category, True))
# Set license
lwt_license = PERSONA_LICENSES_IDS.get(
lwt.persona.license, LICENSE_COPYRIGHT_AR) # default to full copyright
static_license = License.objects.get(builtin=lwt_license.builtin)
version.update(license=static_license)
# Set tags
for addon_tag in AddonTag.objects.filter(addon=lwt):
AddonTag.objects.create(addon=addon, tag=addon_tag.tag)
# Steal the ratings (even with soft delete they'll be deleted anyway)
addon_updates.update(average_rating=lwt.average_rating,
bayesian_rating=lwt.bayesian_rating,
total_ratings=lwt.total_ratings,
text_ratings_count=lwt.text_ratings_count)
Rating.unfiltered.filter(addon=lwt).update(addon=addon, version=version)
# Modify the activity log entry too.
rating_activity_log_ids = [
l.id for l in amo.LOG if getattr(l, 'action_class', '') == 'review'
]
addonlog_qs = AddonLog.objects.filter(
addon=lwt, activity_log__action__in=rating_activity_log_ids)
[alog.transfer(addon) for alog in addonlog_qs.iterator()]
# Logging
activity.log_create(amo.LOG.CREATE_STATICTHEME_FROM_PERSONA,
addon,
user=author)
log.debug('New static theme %r created from %r' % (addon, lwt))
# And finally sign the files (actually just one)
for file_ in version.all_files:
sign_file(file_)
file_.update(datestatuschanged=datetime.now(),
reviewed=datetime.now(),
status=amo.STATUS_PUBLIC)
addon_updates['status'] = amo.STATUS_PUBLIC
addon.update(**addon_updates)
return addon
def test_is_signed(self):
assert not packaged.is_signed(self.file_.file_path)
packaged.sign_file(self.file_)
assert packaged.is_signed(self.file_.file_path)
def test_no_sign_hotfix_addons(self):
"""Don't sign hotfix addons."""
for hotfix_guid in settings.HOTFIX_ADDON_GUIDS:
self.addon.update(guid=hotfix_guid)
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
self.assert_not_signed()
def test_no_sign_hotfix_addons(self):
"""Don't sign hotfix addons."""
for hotfix_guid in settings.HOTFIX_ADDON_GUIDS:
self.addon.update(guid=hotfix_guid)
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
self.assert_not_signed()
def test_is_signed(self):
assert not packaged.is_signed(self.file_.file_path)
packaged.sign_file(self.file_)
assert packaged.is_signed(self.file_.file_path)
def test_size_updated(self):
unsigned_size = storage.size(self.file_.file_path)
packaged.sign_file(self.file_)
signed_size = storage.size(self.file_.file_path)
assert self.file_.size == signed_size
assert unsigned_size < signed_size
def test_no_sign_again_mozilla_signed_extensions(self):
"""Don't try to resign mozilla signed extensions."""
self.file_.update(is_mozilla_signed_extension=True)
packaged.sign_file(self.file_, settings.SIGNING_SERVER)
self.assert_not_signed()
def test_no_sign_again_mozilla_signed_extensions(self):
"""Don't try to resign mozilla signed extensions."""
self.file_.update(is_mozilla_signed_extension=True)
packaged.sign_file(self.file_)
self.assert_not_signed()
def fix_let_scope_bustage_in_addons(addon_ids):
"""Used to fix the "let scope bustage" (bug 1224686) in the last version of
the provided add-ons.
This is used in the 'fix_let_scope_bustage' management commands.
It also bumps the version number of the file and the Version, so the
Firefox extension update mechanism picks this new fixed version and
installs it.
"""
log.info(u'[{0}] Fixing addons.'.format(len(addon_ids)))
addons_emailed = []
for addon in Addon.objects.filter(id__in=addon_ids):
# We only care about the latest added version for each add-on.
version = addon.versions.first()
log.info(u'Fixing addon {0}, version {1}'.format(addon, version))
bumped_version_number = u'{0}.1-let-fixed'.format(version.version)
for file_obj in version.files.all():
if not os.path.isfile(file_obj.file_path):
log.info(u'File {0} does not exist, skip'.format(file_obj.pk))
continue
# Save the original file, before bumping the version.
backup_path = u'{0}.backup_let_fix'.format(file_obj.file_path)
shutil.copy(file_obj.file_path, backup_path)
try:
# Apply the fix itself.
fix_let_scope_bustage_in_xpi(file_obj.file_path)
except:
log.error(u'Failed fixing file {0}'.format(file_obj.pk),
exc_info=True)
# Revert the fix by restoring the backup.
shutil.move(backup_path, file_obj.file_path)
continue # We move to the next file.
# Need to bump the version (modify the manifest file)
# before the file is signed.
update_version_number(file_obj, bumped_version_number)
if file_obj.is_signed: # Only sign if it was already signed.
if file_obj.status == amo.STATUS_PUBLIC:
server = settings.SIGNING_SERVER
else:
server = settings.PRELIMINARY_SIGNING_SERVER
sign_file(file_obj, server)
# Now update the Version model.
version.update(version=bumped_version_number,
version_int=version_int(bumped_version_number))
addon = version.addon
if addon.pk not in addons_emailed:
# Send a mail to the owners/devs warning them we've
# automatically fixed their addon.
qs = (AddonUser.objects
.filter(role=amo.AUTHOR_ROLE_OWNER, addon=addon)
.exclude(user__email__isnull=True))
emails = qs.values_list('user__email', flat=True)
subject = MAIL_SUBJECT.format(addon=addon.name)
message = MAIL_MESSAGE.format(
addon=addon.name,
addon_url=amo.helpers.absolutify(
addon.get_dev_url(action='versions')))
amo.utils.send_mail(
subject, message, recipient_list=emails,
fail_silently=True,
headers={'Reply-To': '*****@*****.**'})
addons_emailed.append(addon.pk)
