def disable_addon_for_block(block):
"""Disable appropriate addon versions that are affected by the Block, and
the addon too if 0 - *."""
from .models import Block
from olympia.reviewers.utils import ReviewBase
review = ReviewBase(
request=None,
addon=block.addon,
version=None,
review_type='pending',
user=get_task_user(),
)
review.set_data({
'versions': [
ver for ver in block.addon_versions
# We don't need to reject versions from older deleted instances
if ver.addon == block.addon
and block.is_version_blocked(ver.version)
]
})
review.reject_multiple_versions()
for version in review.data['versions']:
# Clear needs_human_review on rejected versions, we consider that
# the admin looked at them before blocking.
review.clear_specific_needs_human_review_flags(version)
if block.min_version == Block.MIN and block.max_version == Block.MAX:
if block.addon.status == amo.STATUS_DELETED:
block.addon.deny_resubmission()
else:
block.addon.update(status=amo.STATUS_DISABLED)
def log_and_notify(action, comments, note_creator, version):
log_kwargs = {
'user': note_creator,
'created': datetime.datetime.now(),
}
if comments:
log_kwargs['details'] = {
'comments': comments,
'version': version.version}
else:
# Just use the name of the action if no comments provided. Alas we
# can't know the locale of recipient, and our templates are English
# only so prevent language jumble by forcing into en-US.
with no_translation():
comments = '%s' % action.short
note = amo.log(action, version.addon, version, **log_kwargs)
# Collect reviewers involved with this version.
review_perm = ('Review' if version.channel == amo.RELEASE_CHANNEL_LISTED
else 'ReviewUnlisted')
log_users = {
alog.user for alog in ActivityLog.objects.for_version(version) if
acl.action_allowed_user(alog.user, 'Addons', review_perm)}
# Collect add-on authors (excl. the person who sent the email.)
addon_authors = set(version.addon.authors.all()) - {note_creator}
# Collect staff that want a copy of the email
staff_cc = set(
UserProfile.objects.filter(groups__name=ACTIVITY_MAIL_GROUP))
# If task_user doesn't exist that's no big issue (i.e. in tests)
try:
task_user = {get_task_user()}
except UserProfile.DoesNotExist:
task_user = set()
# Collect reviewers on the thread (excl. the email sender and task user for
# automated messages).
reviewers = ((log_users | staff_cc) - addon_authors - task_user -
{note_creator})
author_context_dict = {
'name': version.addon.name,
'number': version.version,
'author': note_creator.name,
'comments': comments,
'url': absolutify(version.addon.get_dev_url('versions')),
'SITE_URL': settings.SITE_URL,
}
reviewer_context_dict = author_context_dict.copy()
reviewer_context_dict['url'] = absolutify(
reverse('editors.review', args=[version.addon.pk], add_prefix=False))
# Not being localised because we don't know the recipients locale.
subject = 'Mozilla Add-ons: %s Updated' % version.addon.name
template = loader.get_template('activity/emails/developer.txt')
send_activity_mail(
subject, template.render(Context(author_context_dict)), version,
addon_authors, settings.EDITORS_EMAIL)
send_activity_mail(
subject, template.render(Context(reviewer_context_dict)), version,
reviewers, settings.EDITORS_EMAIL)
return note
def wrapper(*args, **kw):
old_user = get_user()
set_user(get_task_user())
try:
result = f(*args, **kw)
finally:
set_user(old_user)
return result
def delete_imported_block_from_blocklist(kinto_id):
existing_blocks = (Block.objects.filter(kinto_id__in=(kinto_id,
f'*{kinto_id}')))
task_user = get_task_user()
for block in existing_blocks:
block_activity_log_delete(block, delete_user=task_user)
block.delete()
KintoImport.objects.get(kinto_id=kinto_id).delete()
def wrapper(*args, **kw):
old_user = get_user()
set_user(get_task_user())
try:
result = f(*args, **kw)
finally:
set_user(old_user)
return result
def delete_imported_block_from_blocklist(legacy_id):
existing_blocks = Block.objects.filter(legacy_id__in=(legacy_id,
f'*{legacy_id}'))
task_user = get_task_user()
for block in existing_blocks:
block_activity_log_delete(block, delete_user=task_user)
block.delete()
statsd.incr('blocklist.tasks.import_blocklist.block_deleted')
LegacyImport.objects.get(legacy_id=legacy_id).delete()
statsd.incr('blocklist.tasks.import_blocklist.deleted_record_processed')
def log_and_notify(action, comments, note_creator, version):
log_kwargs = {
'user': note_creator,
'created': datetime.datetime.now(),
'details': {
'comments': comments,
'version': version.version
}
}
note = amo.log(action, version.addon, version, **log_kwargs)
# Collect reviewers involved with this version.
review_perm = ('Review' if version.channel == amo.RELEASE_CHANNEL_LISTED
else 'ReviewUnlisted')
log_users = {
alog.user
for alog in ActivityLog.objects.for_version(version)
if acl.action_allowed_user(alog.user, 'Addons', review_perm)
}
# Collect add-on authors (excl. the person who sent the email.)
addon_authors = set(version.addon.authors.all()) - {note_creator}
# Collect staff that want a copy of the email
staff_cc = set(
UserProfile.objects.filter(groups__name=ACTIVITY_MAIL_GROUP))
# If task_user doesn't exist that's no big issue (i.e. in tests)
try:
task_user = {get_task_user()}
except UserProfile.DoesNotExist:
task_user = set()
# Collect reviewers on the thread (excl. the email sender and task user for
# automated messages).
reviewers = ((log_users | staff_cc) - addon_authors - task_user -
{note_creator})
author_context_dict = {
'name': version.addon.name,
'number': version.version,
'author': note_creator.name,
'comments': comments,
'url': version.addon.get_dev_url('versions'),
'SITE_URL': settings.SITE_URL,
}
reviewer_context_dict = author_context_dict.copy()
reviewer_context_dict['url'] = absolutify(
reverse('editors.review', args=[version.addon.pk], add_prefix=False))
# Not being localised because we don't know the recipients locale.
subject = 'Mozilla Add-ons: %s Updated' % version.addon.name
template = loader.get_template('activity/emails/developer.txt')
send_activity_mail(subject, template.render(Context(author_context_dict)),
version, addon_authors, settings.EDITORS_EMAIL)
send_activity_mail(subject,
template.render(Context(reviewer_context_dict)),
version, reviewers, settings.EDITORS_EMAIL)
return note
def import_block_from_blocklist(record):
log.debug('Processing block id: [%s]', record.get('id'))
guid = record.get('guid')
if not guid:
log.error('GUID is falsey, skipping.')
return
version_range = record.get('versionRange', [{}])[0]
target_application = version_range.get('targetApplication') or [{}]
target_GUID = target_application[0].get('guid')
if target_GUID and target_GUID != amo.FIREFOX.guid:
log.error('targetApplication (%s) is not Firefox, skipping.',
target_GUID)
return
block_kw = {
'min_version': version_range.get('minVersion', '0'),
'max_version': version_range.get('maxVersion', '*'),
'url': record.get('details', {}).get('bug'),
'reason': record.get('details', {}).get('why', ''),
'kinto_id': record.get('id'),
'include_in_legacy': True,
'updated_by': get_task_user(),
}
modified_date = datetime.fromtimestamp(
record.get('last_modified',
time.time() * 1000) / 1000)
if guid.startswith('/'):
# need to escape the {} brackets or mysql chokes.
guid_regexp = bracket_open_regex.sub(r'\{', guid[1:-1])
guid_regexp = bracket_close_regex.sub(r'\}', guid_regexp)
log.debug('Attempting to create Blocks for addons matching [%s]',
guid_regexp)
addons_qs = Addon.unfiltered.filter(guid__regex=guid_regexp)
# We need to mark this id in a way so we know its from a
# regex guid - otherwise we might accidentally overwrite it.
block_kw['kinto_id'] = '*' + block_kw['kinto_id']
else:
log.debug('Attempting to create a Block for guid [%s]', guid)
addons_qs = Addon.unfiltered.filter(guid=guid)
for addon in addons_qs:
(block, created) = Block.objects.update_or_create(guid=addon.guid,
defaults=dict(
guid=addon.guid,
**block_kw))
block_activity_log_save(block, change=not created)
if created:
log.debug('Added Block for [%s]', block.guid)
block.update(modified=modified_date)
else:
log.debug('Updated Block for [%s]', block.guid)
else:
log.debug('No addon found for block id: [%s]', record.get('id'))
def update_maxversions(version_pks, job_pk, data, **kw):
log.info('[%s@%s] Updating max version for job %s.' %
(len(version_pks), update_maxversions.rate_limit, job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
dry_run = data['preview_only']
app_id = job.target_version.application
stats = collections.defaultdict(int)
stats['processed'] = 0
stats['is_dry_run'] = int(dry_run)
for version in Version.objects.filter(pk__in=version_pks):
stats['processed'] += 1
file_pks = version.files.values_list('pk', flat=True)
errors = (ValidationResult.objects.filter(
validation_job=job, file__pk__in=file_pks).values_list('errors',
flat=True))
if any(errors):
stats['invalid'] += 1
log.info('Version %s for addon %s not updated, '
'one of the files did not pass validation' %
(version.pk, version.addon.pk))
continue
for app in version.apps.filter(
application=job.curr_max_version.application,
max__version_int__gte=job.curr_max_version.version_int,
max__version_int__lt=job.target_version.version_int):
stats['bumped'] += 1
log.info(
'Updating version %s%s for addon %s from version %s '
'to version %s' %
(version.pk, ' [DRY RUN]' if dry_run else '', version.addon.pk,
job.curr_max_version.version, job.target_version.version))
app.max = job.target_version
if not dry_run:
app.save()
amo.log(amo.LOG.MAX_APPVERSION_UPDATED,
version.addon,
version,
details={
'version': version.version,
'target': job.target_version.version,
'application': app_id
})
log.info('[%s@%s] bulk update stats for job %s: {%s}' %
(len(version_pks), update_maxversions.rate_limit, job_pk,
', '.join('%s: %s' % (k, stats[k])
for k in sorted(stats.keys()))))
def update_maxversions(version_pks, job_pk, data, **kw):
log.info('[%s@%s] Updating max version for job %s.'
% (len(version_pks), update_maxversions.rate_limit, job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
dry_run = data['preview_only']
app_id = job.target_version.application
stats = collections.defaultdict(int)
stats['processed'] = 0
stats['is_dry_run'] = int(dry_run)
for version in Version.objects.filter(pk__in=version_pks):
stats['processed'] += 1
file_pks = version.files.values_list('pk', flat=True)
errors = (ValidationResult.objects.filter(validation_job=job,
file__pk__in=file_pks)
.values_list('errors', flat=True))
if any(errors):
stats['invalid'] += 1
log.info('Version %s for addon %s not updated, '
'one of the files did not pass validation'
% (version.pk, version.addon.pk))
continue
for app in version.apps.filter(
application=job.curr_max_version.application,
max__version_int__gte=job.curr_max_version.version_int,
max__version_int__lt=job.target_version.version_int):
stats['bumped'] += 1
log.info('Updating version %s%s for addon %s from version %s '
'to version %s'
% (version.pk,
' [DRY RUN]' if dry_run else '',
version.addon.pk,
job.curr_max_version.version,
job.target_version.version))
app.max = job.target_version
if not dry_run:
app.save()
amo.log(amo.LOG.MAX_APPVERSION_UPDATED,
version.addon, version,
details={'version': version.version,
'target': job.target_version.version,
'application': app_id})
log.info('[%s@%s] bulk update stats for job %s: {%s}'
% (len(version_pks), update_maxversions.rate_limit, job_pk,
', '.join('%s: %s' % (k, stats[k])
for k in sorted(stats.keys()))))
def __init__(self, request, addon, version, review_type,
content_review=False, user=None):
self.request = request
if request:
self.user = user or self.request.user
self.human_review = True
else:
# Use the addons team go-to user "Mozilla" for the automatic
# validations.
self.user = user or get_task_user()
self.human_review = False
self.addon = addon
self.version = version
self.review_type = (
('theme_%s' if addon.type == amo.ADDON_STATICTHEME
else 'extension_%s') % review_type)
self.files = self.version.unreviewed_files if self.version else []
self.content_review = content_review
self.redirect_url = None
def handle(self, *args, **options):
with open(options.get('guids_input'), 'r') as guid_file:
input_guids = guid_file.read()
guids = splitlines(input_guids)
block_args = {
prop: options.get(prop)
for prop in ('min_version', 'max_version', 'reason', 'url')
if options.get(prop)
}
block_args['updated_by'] = get_task_user()
block_args['include_in_legacy'] = False
submission = BlocklistSubmission(**block_args)
for guids_chunk in chunked(guids, 100):
blocks = save_guids_to_blocks(guids_chunk, submission)
print(
f'Added/Updated {len(blocks)} blocks from {len(guids_chunk)} '
'guids')
def notify_about_activity_log(addon,
version,
note,
perm_setting=None,
send_to_reviewers=True,
send_to_staff=True):
"""Notify relevant users about an ActivityLog note."""
comments = (note.details or {}).get('comments')
if not comments:
# Just use the name of the action if no comments provided. Alas we
# can't know the locale of recipient, and our templates are English
# only so prevent language jumble by forcing into en-US.
with translation.override(settings.LANGUAGE_CODE):
comments = '%s' % amo.LOG_BY_ID[note.action].short
else:
htmlparser = HTMLParser()
comments = htmlparser.unescape(comments)
# Collect add-on authors (excl. the person who sent the email.) and build
# the context for them.
addon_authors = set(addon.authors.all()) - {note.user}
author_context_dict = {
'name': addon.name,
'number': version.version,
'author': note.user.name,
'comments': comments,
'url': absolutify(addon.get_dev_url('versions')),
'SITE_URL': settings.SITE_URL,
'email_reason': 'you are listed as an author of this add-on',
'is_info_request': note.action == amo.LOG.REQUEST_INFORMATION.id,
}
# Not being localised because we don't know the recipients locale.
with translation.override('en-US'):
if note.action == amo.LOG.REQUEST_INFORMATION.id:
if addon.pending_info_request:
days_left = (
# We pad the time left with an extra hour so that the email
# does not end up saying "6 days left" because a few
# seconds or minutes passed between the datetime was saved
# and the email was sent.
addon.pending_info_request + timedelta(hours=1) -
datetime.now()).days
if days_left > 9:
author_context_dict['number_of_days_left'] = ('%d days' %
days_left)
elif days_left > 1:
author_context_dict['number_of_days_left'] = (
'%s (%d) days' % (apnumber(days_left), days_left))
else:
author_context_dict['number_of_days_left'] = 'one (1) day'
subject = u'Mozilla Add-ons: Action Required for %s %s' % (
addon.name, version.version)
reviewer_subject = u'Mozilla Add-ons: %s %s' % (addon.name,
version.version)
else:
subject = reviewer_subject = u'Mozilla Add-ons: %s %s' % (
addon.name, version.version)
# Build and send the mail for authors.
template = template_from_user(note.user, version)
from_email = formataddr((note.user.name, NOTIFICATIONS_FROM_EMAIL))
send_activity_mail(subject, template.render(author_context_dict), version,
addon_authors, from_email, note.id, perm_setting)
if send_to_reviewers or send_to_staff:
# If task_user doesn't exist that's no big issue (i.e. in tests)
try:
task_user = {get_task_user()}
except UserProfile.DoesNotExist:
task_user = set()
if send_to_reviewers:
# Collect reviewers on the thread (excl. the email sender and task user
# for automated messages), build the context for them and send them
# their copy.
log_users = {
alog.user
for alog in ActivityLog.objects.for_version(version)
if acl.is_user_any_kind_of_reviewer(alog.user)
}
reviewers = log_users - addon_authors - task_user - {note.user}
reviewer_context_dict = author_context_dict.copy()
reviewer_context_dict['url'] = absolutify(
reverse('reviewers.review',
kwargs={
'addon_id': version.addon.pk,
'channel': amo.CHANNEL_CHOICES_API[version.channel]
},
add_prefix=False))
reviewer_context_dict['email_reason'] = 'you reviewed this add-on'
send_activity_mail(reviewer_subject,
template.render(reviewer_context_dict), version,
reviewers, from_email, note.id, perm_setting)
if send_to_staff:
# Collect staff that want a copy of the email, build the context for
# them and send them their copy.
staff = set(
UserProfile.objects.filter(groups__name=ACTIVITY_MAIL_GROUP))
staff_cc = (staff - reviewers - addon_authors - task_user -
{note.user})
staff_cc_context_dict = reviewer_context_dict.copy()
staff_cc_context_dict['email_reason'] = (
'you are member of the activity email cc group')
send_activity_mail(reviewer_subject,
template.render(staff_cc_context_dict), version,
staff_cc, from_email, note.id, perm_setting)
def from_upload(cls, upload, addon, selected_apps, channel,
parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of compatible app ids, a channel id and
the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
from olympia.git.utils import create_git_extraction_entry
assert parsed_data is not None
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(
channel=channel, exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
approval_notes = None
if parsed_data.get('is_mozilla_signed_extension'):
approval_notes = (u'This version has been signed with '
u'Mozilla internal certificate.')
version = cls.objects.create(
addon=addon,
approval_notes=approval_notes,
version=parsed_data['version'],
license_id=license_id,
channel=channel,
)
email = upload.user.email if upload.user and upload.user.email else ''
with core.override_remote_addr(upload.ip_address):
log.info(
'New version: %r (%s) from %r' % (version, version.id, upload),
extra={
'email': email,
'guid': addon.guid,
'upload': upload.uuid.hex,
'user_id': upload.user_id,
'from_api': upload.source == amo.UPLOAD_SOURCE_API,
}
)
activity.log_create(
amo.LOG.ADD_VERSION, version, addon,
user=upload.user or get_task_user())
if addon.type == amo.ADDON_STATICTHEME:
# We don't let developers select apps for static themes
selected_apps = [app.id for app in amo.APP_USAGE]
compatible_apps = {}
for app in parsed_data.get('apps', []):
if app.id not in selected_apps:
# If the user chose to explicitly deselect Firefox for Android
# we're not creating the respective `ApplicationsVersions`
# which will have this add-on then be listed only for
# Firefox specifically.
continue
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# See #2828: sometimes when we generate the filename(s) below, in
# File.from_upload(), cache-machine is confused and has trouble
# fetching the ApplicationsVersions that were just created. To work
# around this we pre-generate version.compatible_apps and avoid the
# queries completely.
version._compatible_apps = compatible_apps
# For backwards compatibility. We removed specific platform
# support during submission but we don't handle it any different
# beyond that yet. That means, we're going to simply set it
# to `PLATFORM_ALL` and also have the backend create separate
# files for each platform. Cleaning that up is another step.
# Given the timing on this, we don't care about updates to legacy
# add-ons as well.
# Create relevant file and update the all_files cached property on the
# Version, because we might need it afterwards.
version.all_files = [File.from_upload(
upload=upload, version=version, platform=amo.PLATFORM_ALL.id,
parsed_data=parsed_data
)]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to all platforms, remove the upload.
storage.delete(upload.path)
upload.path = ''
upload.save()
version_uploaded.send(instance=version, sender=Version)
if version.is_webextension:
if (
waffle.switch_is_active('enable-yara') or
waffle.switch_is_active('enable-customs') or
waffle.switch_is_active('enable-wat')
):
ScannerResult.objects.filter(upload_id=upload.id).update(
version=version)
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Schedule this version for git extraction.
transaction.on_commit(
lambda: create_git_extraction_entry(version=version)
)
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME and
channel == amo.RELEASE_CHANNEL_LISTED):
theme_data = parsed_data.get('theme', {})
generate_static_theme_preview(theme_data, version.pk)
# Authors need to be notified about auto-approval delay again since
# they are submitting a new version.
addon.reset_notified_about_auto_approval_delay()
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'
.format(delta=upload_time, version=version,
created=upload.created, now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
def log_and_notify(action, comments, note_creator, version, perm_setting=None,
detail_kwargs=None):
log_kwargs = {
'user': note_creator,
'created': datetime.datetime.now(),
}
if detail_kwargs is None:
detail_kwargs = {}
if comments:
detail_kwargs['version'] = version.version
detail_kwargs['comments'] = comments
else:
# Just use the name of the action if no comments provided. Alas we
# can't know the locale of recipient, and our templates are English
# only so prevent language jumble by forcing into en-US.
with no_translation():
comments = '%s' % action.short
if detail_kwargs:
log_kwargs['details'] = detail_kwargs
note = ActivityLog.create(action, version.addon, version, **log_kwargs)
if not note:
return
# Collect reviewers involved with this version.
review_perm = (amo.permissions.ADDONS_REVIEW
if version.channel == amo.RELEASE_CHANNEL_LISTED
else amo.permissions.ADDONS_REVIEW_UNLISTED)
log_users = {
alog.user for alog in ActivityLog.objects.for_version(version) if
acl.action_allowed_user(alog.user, review_perm)}
# Collect add-on authors (excl. the person who sent the email.)
addon_authors = set(version.addon.authors.all()) - {note_creator}
# Collect staff that want a copy of the email
staff = set(
UserProfile.objects.filter(groups__name=ACTIVITY_MAIL_GROUP))
# If task_user doesn't exist that's no big issue (i.e. in tests)
try:
task_user = {get_task_user()}
except UserProfile.DoesNotExist:
task_user = set()
# Collect reviewers on the thread (excl. the email sender and task user for
# automated messages).
reviewers = log_users - addon_authors - task_user - {note_creator}
staff_cc = staff - reviewers - addon_authors - task_user - {note_creator}
author_context_dict = {
'name': version.addon.name,
'number': version.version,
'author': note_creator.name,
'comments': comments,
'url': absolutify(version.addon.get_dev_url('versions')),
'SITE_URL': settings.SITE_URL,
'email_reason': 'you are an author of this add-on'
}
reviewer_context_dict = author_context_dict.copy()
reviewer_context_dict['url'] = absolutify(
reverse('reviewers.review',
kwargs={'addon_id': version.addon.pk,
'channel': amo.CHANNEL_CHOICES_API[version.channel]},
add_prefix=False))
reviewer_context_dict['email_reason'] = 'you reviewed this add-on'
staff_cc_context_dict = reviewer_context_dict.copy()
staff_cc_context_dict['email_reason'] = (
'you are member of the activity email cc group')
# Not being localised because we don't know the recipients locale.
with translation.override('en-US'):
subject = u'Mozilla Add-ons: %s %s' % (
version.addon.name, version.version)
template = template_from_user(note_creator, version)
from_email = formataddr((note_creator.name, NOTIFICATIONS_FROM_EMAIL))
send_activity_mail(
subject, template.render(author_context_dict),
version, addon_authors, from_email, note.id, perm_setting)
send_activity_mail(
subject, template.render(reviewer_context_dict),
version, reviewers, from_email, note.id, perm_setting)
send_activity_mail(
subject, template.render(staff_cc_context_dict),
version, staff_cc, from_email, note.id, perm_setting)
if action == amo.LOG.DEVELOPER_REPLY_VERSION:
version.update(has_info_request=False)
return note
def log_and_notify(action,
comments,
note_creator,
version,
perm_setting=None,
detail_kwargs=None):
log_kwargs = {
'user': note_creator,
'created': datetime.datetime.now(),
}
if detail_kwargs is None:
detail_kwargs = {}
if comments:
detail_kwargs['version'] = version.version
detail_kwargs['comments'] = comments
else:
# Just use the name of the action if no comments provided. Alas we
# can't know the locale of recipient, and our templates are English
# only so prevent language jumble by forcing into en-US.
with no_translation():
comments = '%s' % action.short
if detail_kwargs:
log_kwargs['details'] = detail_kwargs
note = ActivityLog.create(action, version.addon, version, **log_kwargs)
if not note:
return
# Collect reviewers involved with this version.
review_perm = (amo.permissions.ADDONS_REVIEW
if version.channel == amo.RELEASE_CHANNEL_LISTED else
amo.permissions.ADDONS_REVIEW_UNLISTED)
log_users = {
alog.user
for alog in ActivityLog.objects.for_version(version)
if acl.action_allowed_user(alog.user, review_perm)
}
# Collect add-on authors (excl. the person who sent the email.)
addon_authors = set(version.addon.authors.all()) - {note_creator}
# Collect staff that want a copy of the email
staff = set(UserProfile.objects.filter(groups__name=ACTIVITY_MAIL_GROUP))
# If task_user doesn't exist that's no big issue (i.e. in tests)
try:
task_user = {get_task_user()}
except UserProfile.DoesNotExist:
task_user = set()
# Collect reviewers on the thread (excl. the email sender and task user for
# automated messages).
reviewers = log_users - addon_authors - task_user - {note_creator}
staff_cc = staff - reviewers - addon_authors - task_user - {note_creator}
author_context_dict = {
'name': version.addon.name,
'number': version.version,
'author': note_creator.name,
'comments': comments,
'url': absolutify(version.addon.get_dev_url('versions')),
'SITE_URL': settings.SITE_URL,
'email_reason': 'you are an author of this add-on'
}
reviewer_context_dict = author_context_dict.copy()
reviewer_context_dict['url'] = absolutify(
reverse('editors.review',
kwargs={
'addon_id': version.addon.pk,
'channel': amo.CHANNEL_CHOICES_API[version.channel]
},
add_prefix=False))
reviewer_context_dict['email_reason'] = 'you reviewed this add-on'
staff_cc_context_dict = reviewer_context_dict.copy()
staff_cc_context_dict['email_reason'] = (
'you are member of the activity email cc group')
# Not being localised because we don't know the recipients locale.
with translation.override('en-US'):
subject = u'Mozilla Add-ons: %s %s' % (version.addon.name,
version.version)
template = template_from_user(note_creator, version)
from_email = formataddr((note_creator.name, NOTIFICATIONS_FROM_EMAIL))
send_activity_mail(subject, template.render(author_context_dict), version,
addon_authors, from_email, note.id, perm_setting)
send_activity_mail(subject, template.render(reviewer_context_dict),
version, reviewers, from_email, note.id, perm_setting)
send_activity_mail(subject, template.render(staff_cc_context_dict),
version, staff_cc, from_email, note.id, perm_setting)
if action == amo.LOG.DEVELOPER_REPLY_VERSION:
version.update(has_info_request=False)
return note
def notify_compatibility_chunk(users, job, data, **kw):
log.info('[%s@%s] Sending notification mail for job %s.' %
(len(users), notify_compatibility.rate_limit, job.pk))
set_user(get_task_user())
dry_run = data['preview_only']
app_id = job.target_version.application
stats = collections.defaultdict(int)
stats['processed'] = 0
stats['is_dry_run'] = int(dry_run)
for user in users:
stats['processed'] += 1
try:
for a in chain(user.passing_addons, user.failing_addons):
try:
results = job.result_set.filter(file__version__addon=a)
a.links = [
absolutify(
reverse('devhub.bulk_compat_result',
args=[a.slug, r.pk])) for r in results
]
v = a.current_version or a.latest_version
a.compat_link = absolutify(
reverse('devhub.versions.edit', args=[a.pk, v.pk]))
except:
task_error = sys.exc_info()
log.error(
u'Bulk validation email error for user %s, '
u'addon %s: %s: %s' %
(user.email, a.slug, task_error[0], task_error[1]),
exc_info=False)
context = Context({
'APPLICATION':
unicode(amo.APP_IDS[job.application].pretty),
'VERSION':
job.target_version.version,
'PASSING_ADDONS':
user.passing_addons,
'FAILING_ADDONS':
user.failing_addons,
})
log.info(
u'Emailing %s%s for %d addons about '
'bulk validation job %s' %
(user.email, ' [PREVIEW]' if dry_run else '',
len(user.passing_addons) + len(user.failing_addons), job.pk))
args = (Template(data['subject']).render(context),
Template(data['text']).render(context))
kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=[user.email])
if dry_run:
job.preview_notify_mail(*args, **kwargs)
else:
stats['author_emailed'] += 1
send_mail(*args, **kwargs)
amo.log(amo.LOG.BULK_VALIDATION_USER_EMAILED,
user,
details={
'passing': [a.id for a in user.passing_addons],
'failing': [a.id for a in user.failing_addons],
'target': job.target_version.version,
'application': app_id
})
except:
task_error = sys.exc_info()
log.error(u'Bulk validation email error for user %s: %s: %s' %
(user.email, task_error[0], task_error[1]),
exc_info=False)
log.info('[%s@%s] bulk email stats for job %s: {%s}' %
(len(users), notify_compatibility.rate_limit, job.pk, ', '.join(
'%s: %s' % (k, stats[k]) for k in sorted(stats.keys()))))
def notify_about_activity_log(addon,
version,
note,
perm_setting=None,
send_to_reviewers=True,
send_to_staff=True):
"""Notify relevant users about an ActivityLog note."""
comments = (note.details or {}).get('comments')
if not comments:
# Just use the name of the action if no comments provided. Alas we
# can't know the locale of recipient, and our templates are English
# only so prevent language jumble by forcing into en-US.
with translation.override(settings.LANGUAGE_CODE):
comments = '%s' % amo.LOG_BY_ID[note.action].short
else:
comments = unescape(comments)
type_of_sender = type_of_user(note.user, version)
sender_name = (ADDON_REVIEWER_NAME if type_of_sender
== USER_TYPE_ADDON_REVIEWER else note.user.name)
# Collect add-on authors (excl. the person who sent the email.) and build
# the context for them.
addon_authors = set(addon.authors.all()) - {note.user}
author_context_dict = {
'name': addon.name,
'number': version.version,
'author': sender_name,
'comments': comments,
'url': absolutify(addon.get_dev_url('versions')),
'SITE_URL': settings.SITE_URL,
'email_reason': 'you are listed as an author of this add-on',
}
# Not being localised because we don't know the recipients locale.
with translation.override('en-US'):
subject = reviewer_subject = 'Mozilla Add-ons: {} {}'.format(
addon.name,
version.version,
)
# Build and send the mail for authors.
template = template_from_user(note.user, version)
from_email = formataddr((sender_name, NOTIFICATIONS_FROM_EMAIL))
send_activity_mail(
subject,
template.render(author_context_dict),
version,
addon_authors,
from_email,
note.id,
perm_setting,
)
if send_to_reviewers or send_to_staff:
# If task_user doesn't exist that's no big issue (i.e. in tests)
try:
task_user = {get_task_user()}
except UserProfile.DoesNotExist:
task_user = set()
# Update the author and from_email to use the real name because it will
# be used in emails to reviewers and staff, and not add-on developers.
from_email = formataddr((note.user.name, NOTIFICATIONS_FROM_EMAIL))
reviewer_context_dict = author_context_dict.copy()
reviewer_context_dict['author'] = note.user.name
if send_to_reviewers:
# Collect reviewers on the thread (excl. the email sender and task user
# for automated messages) and send them their copy.
log_users = {
alog.user
for alog in ActivityLog.objects.for_versions(version)
if acl.is_user_any_kind_of_reviewer(alog.user)
}
reviewers = log_users - addon_authors - task_user - {note.user}
reviewer_context_dict['url'] = absolutify(
reverse(
'reviewers.review',
kwargs={
'addon_id': version.addon.pk,
'channel': amo.CHANNEL_CHOICES_API[version.channel],
},
add_prefix=False,
))
reviewer_context_dict['email_reason'] = 'you reviewed this add-on'
send_activity_mail(
reviewer_subject,
template.render(reviewer_context_dict),
version,
reviewers,
from_email,
note.id,
perm_setting,
)
if send_to_staff:
# Collect staff that want a copy of the email, build the context for
# them and send them their copy.
staff = set(
UserProfile.objects.filter(groups__name=ACTIVITY_MAIL_GROUP))
staff_cc = staff - reviewers - addon_authors - task_user - {note.user}
staff_cc_context_dict = reviewer_context_dict.copy()
staff_cc_context_dict[
'email_reason'] = 'you are member of the activity email cc group'
send_activity_mail(
reviewer_subject,
template.render(staff_cc_context_dict),
version,
staff_cc,
from_email,
note.id,
perm_setting,
)
def import_block_from_blocklist(record):
kinto_id = record.get('id')
using_db = 'replica' if 'replica' in settings.DATABASES else 'default'
log.debug('Processing block id: [%s]', kinto_id)
kinto_import = KintoImport(kinto_id=kinto_id, record=record)
guid = record.get('guid')
if not guid:
kinto_import.outcome = KintoImport.OUTCOME_MISSINGGUID
kinto_import.save()
log.error('Kinto %s: GUID is falsey, skipping.', kinto_id)
return
version_range = record.get('versionRange', [{}])[0]
target_application = version_range.get('targetApplication') or [{}]
target_GUID = target_application[0].get('guid')
if target_GUID and target_GUID != amo.FIREFOX.guid:
kinto_import.outcome = KintoImport.OUTCOME_NOTFIREFOX
kinto_import.save()
log.error('Kinto %s: targetApplication (%s) is not Firefox, skipping.',
kinto_id, target_GUID)
return
block_kw = {
'min_version': version_range.get('minVersion', '0'),
'max_version': version_range.get('maxVersion', '*'),
'url': record.get('details', {}).get('bug') or '',
'reason': record.get('details', {}).get('why') or '',
'kinto_id': kinto_id,
'include_in_legacy': True,
'updated_by': get_task_user(),
}
modified_date = datetime.fromtimestamp(
record.get('last_modified',
time.time() * 1000) / 1000)
if guid.startswith('/'):
# need to escape the {} brackets or mysql chokes.
guid_regexp = bracket_open_regex.sub(r'\{', guid[1:-1])
guid_regexp = bracket_close_regex.sub(r'\}', guid_regexp)
# we're going to try to split the regex into a list for efficiency.
guids_list = split_regex_to_list(guid_regexp)
if guids_list:
log.debug(
'Kinto %s: Broke down regex into list; '
'attempting to create Blocks for guids in %s', kinto_id,
guids_list)
addons_guids_qs = Addon.unfiltered.using(using_db).filter(
guid__in=guids_list).values_list('guid', flat=True)
else:
log.debug(
'Kinto %s: Unable to break down regex into list; '
'attempting to create Blocks for guids matching [%s]',
kinto_id, guid_regexp)
# mysql doesn't support \d - only [:digit:]
guid_regexp = guid_regexp.replace(r'\d', '[[:digit:]]')
addons_guids_qs = Addon.unfiltered.using(using_db).filter(
guid__regex=guid_regexp).values_list('guid', flat=True)
# We need to mark this id in a way so we know its from a
# regex guid - otherwise we might accidentally overwrite it.
block_kw['kinto_id'] = '*' + block_kw['kinto_id']
regex = True
else:
log.debug('Kinto %s: Attempting to create a Block for guid [%s]',
kinto_id, guid)
addons_guids_qs = Addon.unfiltered.using(using_db).filter(
guid=guid).values_list('guid', flat=True)
regex = False
new_blocks = []
for guid in addons_guids_qs:
valid_files_qs = File.objects.filter(version__addon__guid=guid,
is_webextension=True)
if not valid_files_qs.exists():
log.debug(
'Kinto %s: Skipped Block for [%s] because it has no '
'webextension files', kinto_id, guid)
continue
(block,
created) = Block.objects.update_or_create(guid=guid,
defaults=dict(guid=guid,
**block_kw))
block_activity_log_save(block, change=not created)
if created:
log.debug('Kinto %s: Added Block for [%s]', kinto_id, guid)
block.update(modified=modified_date)
else:
log.debug('Kinto %s: Updated Block for [%s]', kinto_id, guid)
new_blocks.append(block)
if new_blocks:
kinto_import.outcome = (KintoImport.OUTCOME_REGEXBLOCKS
if regex else KintoImport.OUTCOME_BLOCK)
else:
kinto_import.outcome = KintoImport.OUTCOME_NOMATCH
log.debug('Kinto %s: No addon found', kinto_id)
kinto_import.save()
def from_upload(cls,
upload,
addon,
selected_apps,
channel,
parsed_data=None):
"""
Create a Version instance and corresponding File(s) from a
FileUpload, an Addon, a list of compatible app ids, a channel id and
the parsed_data generated by parse_addon().
Note that it's the caller's responsability to ensure the file is valid.
We can't check for that here because an admin may have overridden the
validation results.
"""
from olympia.addons.models import AddonReviewerFlags
from olympia.addons.utils import RestrictionChecker
from olympia.git.utils import create_git_extraction_entry
assert parsed_data is not None
if addon.status == amo.STATUS_DISABLED:
raise VersionCreateError(
'Addon is Mozilla Disabled; no new versions are allowed.')
if upload.addon and upload.addon != addon:
raise VersionCreateError(
'FileUpload was made for a different Addon')
if not upload.user or not upload.ip_address or not upload.source:
raise VersionCreateError(
'FileUpload does not have some required fields')
if not upload.user.last_login_ip or not upload.user.email:
raise VersionCreateError(
'FileUpload user does not have some required fields')
license_id = None
if channel == amo.RELEASE_CHANNEL_LISTED:
previous_version = addon.find_latest_version(channel=channel,
exclude=())
if previous_version and previous_version.license_id:
license_id = previous_version.license_id
approval_notes = None
if parsed_data.get('is_mozilla_signed_extension'):
approval_notes = (
'This version has been signed with Mozilla internal certificate.'
)
version = cls.objects.create(
addon=addon,
approval_notes=approval_notes,
version=parsed_data['version'],
license_id=license_id,
channel=channel,
)
email = upload.user.email if upload.user and upload.user.email else ''
with core.override_remote_addr(upload.ip_address):
# The following log statement is used by foxsec-pipeline.
# We override the IP because it might be called from a task and we
# want the original IP from the submitter.
log.info(
f'New version: {version!r} ({version.id}) from {upload!r}',
extra={
'email': email,
'guid': addon.guid,
'upload': upload.uuid.hex,
'user_id': upload.user_id,
'from_api': upload.source == amo.UPLOAD_SOURCE_API,
},
)
activity.log_create(amo.LOG.ADD_VERSION,
version,
addon,
user=upload.user or get_task_user())
if addon.type == amo.ADDON_STATICTHEME:
# We don't let developers select apps for static themes
selected_apps = [app.id for app in amo.APP_USAGE]
compatible_apps = {}
for app in parsed_data.get('apps', []):
if app.id not in selected_apps:
# If the user chose to explicitly deselect Firefox for Android
# we're not creating the respective `ApplicationsVersions`
# which will have this add-on then be listed only for
# Firefox specifically.
continue
compatible_apps[app.appdata] = ApplicationsVersions(
version=version, min=app.min, max=app.max, application=app.id)
compatible_apps[app.appdata].save()
# Pre-generate _compatible_apps property to avoid accidentally
# triggering queries with that instance later.
version._compatible_apps = compatible_apps
# Create relevant file and update the all_files cached property on the
# Version, because we might need it afterwards.
version.all_files = [
File.from_upload(
upload=upload,
version=version,
parsed_data=parsed_data,
)
]
version.inherit_nomination(from_statuses=[amo.STATUS_AWAITING_REVIEW])
version.disable_old_files()
# After the upload has been copied to its permanent location, delete it
# from storage. Keep the FileUpload instance (it gets cleaned up by a
# cron eventually some time after its creation, in amo.cron.gc()),
# making sure it's associated with the add-on instance.
storage.delete(upload.path)
upload.path = ''
if upload.addon is None:
upload.addon = addon
upload.save()
version_uploaded.send(instance=version, sender=Version)
if version.is_webextension:
if (waffle.switch_is_active('enable-yara')
or waffle.switch_is_active('enable-customs')
or waffle.switch_is_active('enable-wat')):
ScannerResult.objects.filter(upload_id=upload.id).update(
version=version)
if waffle.switch_is_active('enable-uploads-commit-to-git-storage'):
# Schedule this version for git extraction.
transaction.on_commit(
lambda: create_git_extraction_entry(version=version))
# Generate a preview and icon for listed static themes
if (addon.type == amo.ADDON_STATICTHEME
and channel == amo.RELEASE_CHANNEL_LISTED):
theme_data = parsed_data.get('theme', {})
generate_static_theme_preview(theme_data, version.pk)
# Reset add-on reviewer flags to disable auto-approval and require
# admin code review if the package has already been signed by mozilla.
reviewer_flags_defaults = {}
is_mozilla_signed = parsed_data.get('is_mozilla_signed_extension')
if upload.validation_timeout:
reviewer_flags_defaults['needs_admin_code_review'] = True
if is_mozilla_signed and addon.type != amo.ADDON_LPAPP:
reviewer_flags_defaults['needs_admin_code_review'] = True
reviewer_flags_defaults['auto_approval_disabled'] = True
# Check if the approval should be restricted
if not RestrictionChecker(upload=upload).is_auto_approval_allowed():
flag = ('auto_approval_disabled'
if channel == amo.RELEASE_CHANNEL_LISTED else
'auto_approval_disabled_unlisted')
reviewer_flags_defaults[flag] = True
if reviewer_flags_defaults:
AddonReviewerFlags.objects.update_or_create(
addon=addon, defaults=reviewer_flags_defaults)
# Authors need to be notified about auto-approval delay again since
# they are submitting a new version.
addon.reset_notified_about_auto_approval_delay()
# Track the time it took from first upload through validation
# (and whatever else) until a version was created.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
upload_time = now_ts - upload_start
log.info('Time for version {version} creation from upload: {delta}; '
'created={created}; now={now}'.format(delta=upload_time,
version=version,
created=upload.created,
now=now))
statsd.timing('devhub.version_created_from_upload', upload_time)
return version
def import_block_from_blocklist(record):
kinto_id = record.get('id')
using_db = 'replica' if 'replica' in settings.DATABASES else 'default'
log.debug('Processing block id: [%s]', kinto_id)
kinto_import = KintoImport(kinto_id=kinto_id, record=record)
guid = record.get('guid')
if not guid:
kinto_import.outcome = KintoImport.OUTCOME_MISSINGGUID
kinto_import.save()
log.error('Kinto %s: GUID is falsey, skipping.', kinto_id)
return
version_range = record.get('versionRange', [{}])[0]
target_application = version_range.get('targetApplication') or [{}]
target_GUID = target_application[0].get('guid')
if target_GUID and target_GUID != amo.FIREFOX.guid:
kinto_import.outcome = KintoImport.OUTCOME_NOTFIREFOX
kinto_import.save()
log.error(
'Kinto %s: targetApplication (%s) is not Firefox, skipping.',
kinto_id, target_GUID)
return
block_kw = {
'min_version': version_range.get('minVersion', '0'),
'max_version': version_range.get('maxVersion', '*'),
'url': record.get('details', {}).get('bug'),
'reason': record.get('details', {}).get('why', ''),
'kinto_id': kinto_id,
'include_in_legacy': True,
'updated_by': get_task_user(),
}
modified_date = datetime.fromtimestamp(
record.get('last_modified', time.time() * 1000) / 1000)
if guid.startswith('/'):
# need to escape the {} brackets or mysql chokes.
guid_regexp = bracket_open_regex.sub(r'\{', guid[1:-1])
guid_regexp = bracket_close_regex.sub(r'\}', guid_regexp)
log.debug(
'Kinto %s: Attempting to create Blocks for addons matching [%s]',
kinto_id, guid_regexp)
addons_guids_qs = Addon.unfiltered.using(using_db).filter(
guid__regex=guid_regexp).values_list('guid', flat=True)
# We need to mark this id in a way so we know its from a
# regex guid - otherwise we might accidentally overwrite it.
block_kw['kinto_id'] = '*' + block_kw['kinto_id']
regex = True
else:
log.debug(
'Kinto %s: Attempting to create a Block for guid [%s]',
kinto_id, guid)
addons_guids_qs = Addon.unfiltered.using(using_db).filter(
guid=guid).values_list('guid', flat=True)
regex = False
for guid in addons_guids_qs:
(block, created) = Block.objects.update_or_create(
guid=guid, defaults=dict(guid=guid, **block_kw))
block_activity_log_save(block, change=not created)
if created:
log.debug('Kinto %s: Added Block for [%s]', kinto_id, block.guid)
block.update(modified=modified_date)
else:
log.debug('Kinto %s: Updated Block for [%s]', kinto_id, block.guid)
if addons_guids_qs:
kinto_import.outcome = (
KintoImport.OUTCOME_REGEXBLOCKS if regex else
KintoImport.OUTCOME_BLOCK
)
else:
kinto_import.outcome = KintoImport.OUTCOME_NOMATCH
log.debug('Kinto %s: No addon found', kinto_id)
kinto_import.save()
def notify_compatibility_chunk(users, job, data, **kw):
log.info('[%s@%s] Sending notification mail for job %s.'
% (len(users), notify_compatibility.rate_limit, job.pk))
set_user(get_task_user())
dry_run = data['preview_only']
app_id = job.target_version.application
stats = collections.defaultdict(int)
stats['processed'] = 0
stats['is_dry_run'] = int(dry_run)
for user in users:
stats['processed'] += 1
try:
for a in chain(user.passing_addons, user.failing_addons):
try:
results = job.result_set.filter(file__version__addon=a)
a.links = [absolutify(reverse('devhub.bulk_compat_result',
args=[a.slug, r.pk]))
for r in results]
v = a.current_version or a.latest_version
a.compat_link = absolutify(reverse('devhub.versions.edit',
args=[a.pk, v.pk]))
except:
task_error = sys.exc_info()
log.error(u'Bulk validation email error for user %s, '
u'addon %s: %s: %s'
% (user.email, a.slug,
task_error[0], task_error[1]), exc_info=False)
context = Context({
'APPLICATION': unicode(amo.APP_IDS[job.application].pretty),
'VERSION': job.target_version.version,
'PASSING_ADDONS': user.passing_addons,
'FAILING_ADDONS': user.failing_addons,
})
log.info(u'Emailing %s%s for %d addons about '
'bulk validation job %s'
% (user.email,
' [PREVIEW]' if dry_run else '',
len(user.passing_addons) + len(user.failing_addons),
job.pk))
args = (Template(data['subject']).render(context),
Template(data['text']).render(context))
kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=[user.email])
if dry_run:
job.preview_notify_mail(*args, **kwargs)
else:
stats['author_emailed'] += 1
send_mail(*args, **kwargs)
amo.log(
amo.LOG.BULK_VALIDATION_USER_EMAILED,
user,
details={'passing': [a.id for a in user.passing_addons],
'failing': [a.id for a in user.failing_addons],
'target': job.target_version.version,
'application': app_id})
except:
task_error = sys.exc_info()
log.error(u'Bulk validation email error for user %s: %s: %s'
% (user.email,
task_error[0], task_error[1]), exc_info=False)
log.info('[%s@%s] bulk email stats for job %s: {%s}'
% (len(users), notify_compatibility.rate_limit, job.pk,
', '.join('%s: %s' % (k, stats[k])
for k in sorted(stats.keys()))))
def create_version(self, addon=None):
from olympia.addons.models import Addon
from olympia.files.models import FileUpload
from olympia.files.utils import parse_addon
from olympia.versions.models import Version
from olympia.versions.utils import get_next_version_number
version_number = '1.0'
# If passing an existing add-on, we need to bump the version number
# to avoid clashes, and also perform a few checks.
if addon is not None:
# Obviously we want an add-on with the right type.
if addon.type != amo.ADDON_SITE_PERMISSION:
raise ImproperlyConfigured(
'SitePermissionVersionCreator was instantiated with non '
'site-permission add-on'
)
# If the user isn't an author, something is wrong.
if not addon.authors.filter(pk=self.user.pk).exists():
raise ImproperlyConfigured(
'SitePermissionVersionCreator was instantiated with a '
'bogus addon/user'
)
# Changing the origins isn't supported at the moment.
latest_version = addon.find_latest_version(
exclude=(), channel=amo.RELEASE_CHANNEL_UNLISTED
)
previous_origins = sorted(
latest_version.installorigin_set.all().values_list('origin', flat=True)
)
if previous_origins != self.install_origins:
raise ImproperlyConfigured(
'SitePermissionVersionCreator was instantiated with an '
'addon that has different origins'
)
version_number = get_next_version_number(addon)
# Create the manifest, with more user-friendly name & description built
# from install_origins/site_permissions, and then the zipfile with that
# manifest inside.
manifest_data = self._create_manifest(version_number)
file_obj = self._create_zipfile(manifest_data)
# Parse the zip we just created. The user needs to be the Mozilla User
# because regular submissions of this type of add-on is forbidden to
# normal users.
parsed_data = parse_addon(
file_obj,
addon=addon,
user=get_task_user(),
)
with core.override_remote_addr(self.remote_addr):
if addon is None:
# Create the Addon instance (without a Version/File at this point).
addon = Addon.initialize_addon_from_upload(
data=parsed_data,
upload=file_obj,
channel=amo.RELEASE_CHANNEL_UNLISTED,
user=self.user,
)
# Create the FileUpload that will become the File+Version.
upload = FileUpload.from_post(
file_obj,
filename=file_obj.name,
size=file_obj.size,
addon=addon,
version=version_number,
channel=amo.RELEASE_CHANNEL_UNLISTED,
user=self.user,
source=amo.UPLOAD_SOURCE_GENERATED,
)
# And finally create the Version instance from the FileUpload.
return Version.from_upload(
upload,
addon,
amo.RELEASE_CHANNEL_UNLISTED,
selected_apps=[x[0] for x in amo.APPS_CHOICES],
parsed_data=parsed_data,
)
def import_block_from_blocklist(record):
legacy_id = record.get('id')
using_db = get_replica()
log.info('Processing block id: [%s]', legacy_id)
legacy_import, import_created = LegacyImport.objects.update_or_create(
legacy_id=legacy_id,
defaults={
'record': record,
'timestamp': record.get('last_modified')
},
)
if not import_created:
log.info('LegacyRS %s: updating existing LegacyImport object',
legacy_id)
existing_block_ids = list(
Block.objects.filter(legacy_id__in=(legacy_id,
f'*{legacy_id}')).values_list(
'id', flat=True))
guid = record.get('guid')
if not guid:
legacy_import.outcome = LegacyImport.OUTCOME_MISSINGGUID
legacy_import.save()
log.error('LegacyRS %s: GUID is falsey, skipping.', legacy_id)
return
version_range = (record.get('versionRange') or [{}])[0]
target_application = version_range.get('targetApplication') or [{}]
target_GUID = target_application[0].get('guid')
if target_GUID and target_GUID != amo.FIREFOX.guid:
legacy_import.outcome = LegacyImport.OUTCOME_NOTFIREFOX
legacy_import.save()
log.error(
'LegacyRS %s: targetApplication (%s) is not Firefox, skipping.',
legacy_id,
target_GUID,
)
return
block_kw = {
'min_version': version_range.get('minVersion', '0'),
'max_version': version_range.get('maxVersion', '*'),
'url': record.get('details', {}).get('bug') or '',
'reason': record.get('details', {}).get('why') or '',
'legacy_id': legacy_id,
'updated_by': get_task_user(),
}
modified_date = datetime.fromtimestamp(
record.get('last_modified', datetime_to_ts()) / 1000)
if guid.startswith('/'):
# need to escape the {} brackets or mysql chokes.
guid_regexp = bracket_open_regex.sub(r'\{', guid[1:-1])
guid_regexp = bracket_close_regex.sub(r'\}', guid_regexp)
# we're going to try to split the regex into a list for efficiency.
guids_list = split_regex_to_list(guid_regexp)
if guids_list:
log.info(
'LegacyRS %s: Broke down regex into list; '
'attempting to create Blocks for guids in %s',
legacy_id,
guids_list,
)
statsd.incr('blocklist.tasks.import_blocklist.record_guid',
count=len(guids_list))
addons_guids_qs = (Addon.unfiltered.using(using_db).filter(
guid__in=guids_list).values_list('guid', flat=True))
else:
log.info(
'LegacyRS %s: Unable to break down regex into list; '
'attempting to create Blocks for guids matching [%s]',
legacy_id,
guid_regexp,
)
# mysql doesn't support \d - only [:digit:]
guid_regexp = guid_regexp.replace(r'\d', '[[:digit:]]')
addons_guids_qs = (Addon.unfiltered.using(using_db).filter(
guid__regex=guid_regexp).values_list('guid', flat=True))
# We need to mark this id in a way so we know its from a
# regex guid - otherwise we might accidentally overwrite it.
block_kw['legacy_id'] = '*' + block_kw['legacy_id']
regex = True
else:
log.info('LegacyRS %s: Attempting to create a Block for guid [%s]',
legacy_id, guid)
statsd.incr('blocklist.tasks.import_blocklist.record_guid')
addons_guids_qs = (Addon.unfiltered.using(using_db).filter(
guid=guid).values_list('guid', flat=True))
regex = False
new_blocks = []
for guid in addons_guids_qs:
valid_files_qs = File.objects.filter(version__addon__guid=guid,
is_webextension=True)
if not valid_files_qs.exists():
log.info(
'LegacyRS %s: Skipped Block for [%s] because it has no '
'webextension files',
legacy_id,
guid,
)
statsd.incr('blocklist.tasks.import_blocklist.block_skipped')
continue
(block,
created) = Block.objects.update_or_create(guid=guid,
defaults=dict(guid=guid,
**block_kw))
block_activity_log_save(block, change=not created)
if created:
log.info('LegacyRS %s: Added Block for [%s]', legacy_id, guid)
statsd.incr('blocklist.tasks.import_blocklist.block_added')
block.update(modified=modified_date)
else:
log.info('LegacyRS %s: Updated Block for [%s]', legacy_id, guid)
statsd.incr('blocklist.tasks.import_blocklist.block_updated')
new_blocks.append(block)
if new_blocks:
legacy_import.outcome = (LegacyImport.OUTCOME_REGEXBLOCKS
if regex else LegacyImport.OUTCOME_BLOCK)
else:
legacy_import.outcome = LegacyImport.OUTCOME_NOMATCH
log.info('LegacyRS %s: No addon found', legacy_id)
if not import_created:
# now reconcile the blocks that were connected to the import last time
# but weren't changed this time - i.e. blocks we need to delete
delete_qs = Block.objects.filter(id__in=existing_block_ids).exclude(
id__in=(block.id for block in new_blocks))
for block in delete_qs:
block_activity_log_delete(block,
delete_user=block_kw['updated_by'])
block.delete()
statsd.incr('blocklist.tasks.import_blocklist.block_deleted')
legacy_import.save()
if import_created:
statsd.incr('blocklist.tasks.import_blocklist.new_record_processed')
else:
statsd.incr(
'blocklist.tasks.import_blocklist.modified_record_processed')
def sign_addons(addon_ids, force=False, send_emails=True, **kw):
"""Used to sign all the versions of an addon.
This is used in the 'process_addons --task resign_addons_for_cose'
management command.
This is also used to resign some promoted addons after they've been added
to a group (or paid).
It also bumps the version number of the file and the Version, so the
Firefox extension update mechanism picks this new signed version and
installs it.
"""
log.info(f'[{len(addon_ids)}] Signing addons.')
mail_subject, mail_message = MAIL_COSE_SUBJECT, MAIL_COSE_MESSAGE
# query everything except for search-plugins as they're generally
# not signed
current_versions = Addon.objects.filter(id__in=addon_ids).values_list(
'_current_version', flat=True
)
qset = Version.objects.filter(id__in=current_versions)
addons_emailed = set()
task_user = get_task_user()
for version in qset:
file_obj = version.file
# We only sign files that have been reviewed
if file_obj.status not in amo.REVIEWED_STATUSES:
log.info(
'Not signing addon {}, version {} (no files)'.format(
version.addon, version
)
)
continue
log.info(f'Signing addon {version.addon}, version {version}')
bumped_version_number = get_new_version_number(version.version)
did_sign = False # Did we sign at the file?
if not os.path.isfile(file_obj.file_path):
log.info(f'File {file_obj.pk} does not exist, skip')
continue
# Save the original file, before bumping the version.
backup_path = f'{file_obj.file_path}.backup_signature'
shutil.copy(file_obj.file_path, backup_path)
try:
# Need to bump the version (modify manifest file)
# before the file is signed.
update_version_number(file_obj, bumped_version_number)
did_sign = bool(sign_file(file_obj))
if not did_sign: # We didn't sign, so revert the version bump.
shutil.move(backup_path, file_obj.file_path)
except Exception:
log.error(f'Failed signing file {file_obj.pk}', exc_info=True)
# Revert the version bump, restore the backup.
shutil.move(backup_path, file_obj.file_path)
# Now update the Version model, if we signed at least one file.
if did_sign:
previous_version_str = str(version.version)
version.update(version=bumped_version_number)
addon = version.addon
ActivityLog.create(
amo.LOG.VERSION_RESIGNED,
addon,
version,
previous_version_str,
user=task_user,
)
if send_emails and addon.pk not in addons_emailed:
# Send a mail to the owners/devs warning them we've
# automatically signed their addon.
qs = AddonUser.objects.filter(
role=amo.AUTHOR_ROLE_OWNER, addon=addon
).exclude(user__email__isnull=True)
emails = qs.values_list('user__email', flat=True)
subject = mail_subject
message = mail_message.format(addon=addon.name)
amo.utils.send_mail(
subject,
message,
recipient_list=emails,
headers={'Reply-To': '*****@*****.**'},
)
addons_emailed.add(addon.pk)
def notify_about_activity_log(addon, version, note, perm_setting=None,
send_to_reviewers=True, send_to_staff=True):
"""Notify relevant users about an ActivityLog note."""
comments = (note.details or {}).get('comments')
if not comments:
# Just use the name of the action if no comments provided. Alas we
# can't know the locale of recipient, and our templates are English
# only so prevent language jumble by forcing into en-US.
with translation.override(settings.LANGUAGE_CODE):
comments = '%s' % amo.LOG_BY_ID[note.action].short
else:
htmlparser = HTMLParser()
comments = htmlparser.unescape(comments)
# Collect add-on authors (excl. the person who sent the email.) and build
# the context for them.
addon_authors = set(addon.authors.all()) - {note.user}
author_context_dict = {
'name': addon.name,
'number': version.version,
'author': note.user.name,
'comments': comments,
'url': absolutify(addon.get_dev_url('versions')),
'SITE_URL': settings.SITE_URL,
'email_reason': 'you are listed as an author of this add-on',
'is_info_request': note.action == amo.LOG.REQUEST_INFORMATION.id,
}
# Not being localised because we don't know the recipients locale.
with translation.override('en-US'):
if note.action == amo.LOG.REQUEST_INFORMATION.id:
if addon.pending_info_request:
days_left = (
# We pad the time left with an extra hour so that the email
# does not end up saying "6 days left" because a few
# seconds or minutes passed between the datetime was saved
# and the email was sent.
addon.pending_info_request + timedelta(hours=1) -
datetime.now()
).days
if days_left > 9:
author_context_dict['number_of_days_left'] = (
'%d days' % days_left)
elif days_left > 1:
author_context_dict['number_of_days_left'] = (
'%s (%d) days' % (apnumber(days_left), days_left))
else:
author_context_dict['number_of_days_left'] = 'one (1) day'
subject = u'Mozilla Add-ons: Action Required for %s %s' % (
addon.name, version.version)
reviewer_subject = u'Mozilla Add-ons: %s %s' % (
addon.name, version.version)
else:
subject = reviewer_subject = u'Mozilla Add-ons: %s %s' % (
addon.name, version.version)
# Build and send the mail for authors.
template = template_from_user(note.user, version)
from_email = formataddr((note.user.name, NOTIFICATIONS_FROM_EMAIL))
send_activity_mail(
subject, template.render(author_context_dict),
version, addon_authors, from_email, note.id, perm_setting)
if send_to_reviewers or send_to_staff:
# If task_user doesn't exist that's no big issue (i.e. in tests)
try:
task_user = {get_task_user()}
except UserProfile.DoesNotExist:
task_user = set()
if send_to_reviewers:
# Collect reviewers on the thread (excl. the email sender and task user
# for automated messages), build the context for them and send them
# their copy.
log_users = {
alog.user for alog in ActivityLog.objects.for_version(version) if
acl.is_user_any_kind_of_reviewer(alog.user)}
reviewers = log_users - addon_authors - task_user - {note.user}
reviewer_context_dict = author_context_dict.copy()
reviewer_context_dict['url'] = absolutify(
reverse('reviewers.review',
kwargs={
'addon_id': version.addon.pk,
'channel': amo.CHANNEL_CHOICES_API[version.channel]
}, add_prefix=False))
reviewer_context_dict['email_reason'] = 'you reviewed this add-on'
send_activity_mail(
reviewer_subject, template.render(reviewer_context_dict),
version, reviewers, from_email, note.id, perm_setting)
if send_to_staff:
# Collect staff that want a copy of the email, build the context for
# them and send them their copy.
staff = set(
UserProfile.objects.filter(groups__name=ACTIVITY_MAIL_GROUP))
staff_cc = (
staff - reviewers - addon_authors - task_user - {note.user})
staff_cc_context_dict = reviewer_context_dict.copy()
staff_cc_context_dict['email_reason'] = (
'you are member of the activity email cc group')
send_activity_mail(
reviewer_subject, template.render(staff_cc_context_dict),
version, staff_cc, from_email, note.id, perm_setting)
def log_and_notify(action, comments, note_creator, version, perm_setting=None,
detail_kwargs=None):
log_kwargs = {
'user': note_creator,
'created': datetime.datetime.now(),
}
if detail_kwargs is None:
detail_kwargs = {}
if comments:
detail_kwargs['version'] = version.version
detail_kwargs['comments'] = comments
else:
# Just use the name of the action if no comments provided. Alas we
# can't know the locale of recipient, and our templates are English
# only so prevent language jumble by forcing into en-US.
with no_translation():
comments = '%s' % action.short
if detail_kwargs:
log_kwargs['details'] = detail_kwargs
note = ActivityLog.create(action, version.addon, version, **log_kwargs)
# Collect reviewers involved with this version.
review_perm = (amo.permissions.ADDONS_REVIEW
if version.channel == amo.RELEASE_CHANNEL_LISTED
else amo.permissions.ADDONS_REVIEW_UNLISTED)
log_users = {
alog.user for alog in ActivityLog.objects.for_version(version) if
acl.action_allowed_user(alog.user, review_perm)}
# Collect add-on authors (excl. the person who sent the email.)
addon_authors = set(version.addon.authors.all()) - {note_creator}
# Collect staff that want a copy of the email
staff_cc = set(
UserProfile.objects.filter(groups__name=ACTIVITY_MAIL_GROUP))
# If task_user doesn't exist that's no big issue (i.e. in tests)
try:
task_user = {get_task_user()}
except UserProfile.DoesNotExist:
task_user = set()
# Collect reviewers on the thread (excl. the email sender and task user for
# automated messages).
reviewers = ((log_users | staff_cc) - addon_authors - task_user -
{note_creator})
author_context_dict = {
'name': version.addon.name,
'number': version.version,
'author': note_creator.name,
'comments': comments,
'url': absolutify(version.addon.get_dev_url('versions')),
'SITE_URL': settings.SITE_URL,
}
reviewer_context_dict = author_context_dict.copy()
reviewer_context_dict['url'] = absolutify(
reverse('editors.review', args=[version.addon.pk], add_prefix=False))
# Not being localised because we don't know the recipients locale.
with translation.override('en-US'):
subject = u'Mozilla Add-ons: %s %s %s' % (
version.addon.name, version.version, action.short)
template = template_from_user(note_creator, version)
send_activity_mail(
subject, template.render(Context(author_context_dict)), version,
addon_authors, settings.EDITORS_EMAIL, perm_setting)
send_activity_mail(
subject, template.render(Context(reviewer_context_dict)), version,
reviewers, settings.EDITORS_EMAIL, perm_setting)
return note
