def notify_failed(file_pks, job_pk, data, **kw):
log.info('[%s@None] Notifying failed for job %s.' %
(len(file_pks), job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
for result in ValidationResult.objects.filter(validation_job=job,
file__pk__in=file_pks):
file = result.file
version = file.version
addon = version.addon
context = get_context(addon, version, job, [result], fileob=file)
for author in addon.authors.all():
log.info(
u'Emailing %s%s for addon %s, file %s about '
'error from bulk validation job %s' %
(author.email, ' [PREVIEW]' if data['preview_only'] else '',
addon.pk, file.pk, job_pk))
args = (Template(data['subject']).render(context),
Template(data['text']).render(context))
kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=[author.email])
if data['preview_only']:
job.preview_failure_mail(*args, **kwargs)
else:
send_mail(*args, **kwargs)
amo.log(amo.LOG.BULK_VALIDATION_EMAILED,
addon,
version,
details={
'version': version.version,
'file': file.filename,
'target': job.target_version.version
})
def notify_failed(file_pks, job_pk, data, **kw):
log.info("[%s@None] Notifying failed for job %s." % (len(file_pks), job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
for result in ValidationResult.objects.filter(validation_job=job, file__pk__in=file_pks):
file = result.file
version = file.version
addon = version.addon
context = get_context(addon, version, job, [result], fileob=file)
for author in addon.authors.all():
log.info(
u"Emailing %s%s for addon %s, file %s about "
"error from bulk validation job %s"
% (author.email, " [PREVIEW]" if data["preview_only"] else "", addon.pk, file.pk, job_pk)
)
args = (Template(data["subject"]).render(context), Template(data["text"]).render(context))
kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[author.email])
if data["preview_only"]:
job.preview_failure_mail(*args, **kwargs)
else:
send_mail(*args, **kwargs)
amo.log(
amo.LOG.BULK_VALIDATION_EMAILED,
addon,
version,
details={"version": version.version, "file": file.filename, "target": job.target_version.version},
)
def find_abuse_escalations(addon_id, **kw):
amo.set_user(get_task_user())
weekago = datetime.date.today() - datetime.timedelta(days=7)
for abuse in AbuseReport.recent_high_abuse_reports(1, weekago, addon_id):
if EscalationQueue.objects.filter(addon=abuse.addon).exists():
# App is already in the queue, no need to re-add it.
# TODO: If not in queue b/c of abuse reports, add an
# amo.LOG.ESCALATED_HIGH_ABUSE for reviewers.
log.info(u'[addon:%s] High abuse reports, but already escalated' %
(abuse.addon,))
continue
# We have an abuse report and this add-on isn't currently in the
# escalated queue... let's see if it has been detected and dealt with
# already.
logs = (AppLog.objects.filter(
activity_log__action=amo.LOG.ESCALATED_HIGH_ABUSE.id,
addon=abuse.addon).order_by('-created'))
if logs:
abuse_since_log = AbuseReport.recent_high_abuse_reports(
1, logs[0].created, addon_id)
# If no abuse reports have happened since the last logged abuse
# report, do not add to queue.
if not abuse_since_log:
log.info(u'[addon:%s] High abuse reports, but none since last '
'escalation' % abuse.addon)
continue
# If we haven't bailed out yet, escalate this app.
msg = u'High number of abuse reports detected'
EscalationQueue.objects.create(addon=abuse.addon)
amo.log(amo.LOG.ESCALATED_HIGH_ABUSE, abuse.addon,
abuse.addon.current_version, details={'comments': msg})
log.info(u'[addon:%s] %s' % (abuse.addon, msg))
def update_maxversions(version_pks, job_pk, data, **kw):
log.info("[%s@%s] Updating max version for job %s." % (len(version_pks), update_maxversions.rate_limit, job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
dry_run = data["preview_only"]
app_id = job.target_version.application
stats = collections.defaultdict(int)
stats["processed"] = 0
stats["is_dry_run"] = int(dry_run)
for version in Version.objects.filter(pk__in=version_pks):
stats["processed"] += 1
addon = version.addon
file_pks = version.files.values_list("pk", flat=True)
errors = ValidationResult.objects.filter(validation_job=job, file__pk__in=file_pks).values_list(
"errors", flat=True
)
if any(errors):
stats["invalid"] += 1
log.info(
"Version %s for addon %s not updated, "
"one of the files did not pass validation" % (version.pk, version.addon.pk)
)
continue
for app in version.apps.filter(
application=job.curr_max_version.application,
max__version_int__gte=job.curr_max_version.version_int,
max__version_int__lt=job.target_version.version_int,
):
stats["bumped"] += 1
log.info(
"Updating version %s%s for addon %s from version %s "
"to version %s"
% (
version.pk,
" [DRY RUN]" if dry_run else "",
version.addon.pk,
job.curr_max_version.version,
job.target_version.version,
)
)
app.max = job.target_version
if not dry_run:
app.save()
amo.log(
amo.LOG.MAX_APPVERSION_UPDATED,
version.addon,
version,
details={"version": version.version, "target": job.target_version.version, "application": app_id},
)
log.info(
"[%s@%s] bulk update stats for job %s: {%s}"
% (
len(version_pks),
update_maxversions.rate_limit,
job_pk,
", ".join("%s: %s" % (k, stats[k]) for k in sorted(stats.keys())),
)
)
def notify_failed(file_pks, job_pk, data, **kw):
log.info('[%s@None] Notifying failed for job %s.'
% (len(file_pks), job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
for result in ValidationResult.objects.filter(validation_job=job,
file__pk__in=file_pks):
file = result.file
version = file.version
addon = version.addon
context = get_context(addon, version, job, [result])
for author in addon.authors.all():
log.info(u'Emailing %s%s for addon %s, file %s about '
'error from bulk validation job %s'
% (author.email,
' [PREVIEW]' if data['preview_only'] else '',
addon.pk, file.pk, job_pk))
args = (Template(data['subject']).render(context),
Template(data['text']).render(context))
kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=[author.email])
if data['preview_only']:
job.preview_failure_mail(*args, **kwargs)
else:
send_mail(*args, **kwargs)
amo.log(amo.LOG.BULK_VALIDATION_EMAILED,
addon, version,
details={'version': version.version,
'file': file.filename,
'target': job.target_version.version})
def update_manifests(ids, **kw):
retry_secs = 3600
task_log.info('[%s@%s] Update manifests.' %
(len(ids), update_manifests.rate_limit))
check_hash = kw.pop('check_hash', True)
retries = kw.pop('retries', {})
# Since we'll be logging the updated manifest change to the users log,
# we'll need to log in as user.
amo.set_user(get_task_user())
for id in ids:
_update_manifest(id, check_hash, retries)
if retries:
try:
update_manifests.retry(args=(retries.keys(), ),
kwargs={
'check_hash': check_hash,
'retries': retries
},
eta=datetime.datetime.now() +
datetime.timedelta(seconds=retry_secs),
max_retries=5)
except RetryTaskError:
_log(id, 'Retrying task in %d seconds.' % retry_secs)
return retries
def test_update_pks_logs(self):
self.create_result(self.job, self.create_file(self.version))
eq_(ActivityLog.objects.for_addons(self.addon).count(), 0)
self.client.post(self.update_url, self.data)
upd = amo.LOG.BULK_VALIDATION_UPDATED.id
logs = ActivityLog.objects.for_addons(self.addon).filter(action=upd)
eq_(logs.count(), 1)
eq_(logs[0].user, get_task_user())
def test_update_pks_logs(self):
self.create_result(self.job, self.create_file(self.version))
eq_(ActivityLog.objects.for_addons(self.addon).count(), 0)
self.client.post(self.update_url, self.data)
upd = amo.LOG.BULK_VALIDATION_UPDATED.id
logs = ActivityLog.objects.for_addons(self.addon).filter(action=upd)
eq_(logs.count(), 1)
eq_(logs[0].user, get_task_user())
def wrapper(*args, **kw):
old_user = get_user()
set_user(get_task_user())
try:
result = f(*args, **kw)
finally:
set_user(old_user)
return result
def notify_compatibility(users, job, data, **kw):
log.info('[%s@%s] Sending notification mail for job %s.'
% (len(users), notify_compatibility.rate_limit, job.pk))
set_user(get_task_user())
dry_run = data['preview_only']
app_id = job.target_version.application.pk
stats = collections.defaultdict(int)
stats['processed'] = 0
stats['is_dry_run'] = int(dry_run)
for user in users:
stats['processed'] += 1
for a in chain(user.passing_addons, user.failing_addons):
results = job.result_set.filter(file__version__addon=a)
a.links = ' '.join(absolutify(reverse('devhub.bulk_compat_result',
args=[a.slug, r.pk]))
for r in results)
a.compat_link = absolutify(reverse('devhub.versions.edit',
args=[a.pk,
a.current_version.pk]))
context = Context({
'APPLICATION': str(job.application),
'VERSION': job.target_version.version,
'PASSING_ADDONS': user.passing_addons,
'FAILING_ADDONS': user.failing_addons,
})
log.info(u'Emailing %s%s for %d addons about '
'bulk validation job %s'
% (user.email,
' [PREVIEW]' if dry_run else '',
len(user.passing_addons) + len(user.failing_addons),
job.pk))
args = (Template(data['subject']).render(context),
Template(data['text']).render(context))
kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=[user.email])
if dry_run:
job.preview_notify_mail(*args, **kwargs)
else:
stats['author_emailed'] += 1
send_mail(*args, **kwargs)
amo.log(amo.LOG.BULK_VALIDATION_USER_EMAILED,
user,
details={'passing': [a.id for a in user.passing_addons],
'failing': [a.id for a in user.failing_addons],
'target': job.target_version.version,
'application': app_id})
log.info('[%s@%s] bulk email stats for job %s: {%s}'
% (len(users), notify_compatibility.rate_limit, job.pk,
', '.join('%s: %s' % (k, stats[k])
for k in sorted(stats.keys()))))
def update_manifests(ids, **kw):
task_log.info('[%s@%s] Update manifests.' %
(len(ids), update_manifests.rate_limit))
# Since we'll be logging the updated manifest change to the users log,
# we'll need to log in as user.
amo.set_user(get_task_user())
for id in ids:
_update_manifest(id)
def update_manifests(ids, **kw):
task_log.info('[%s@%s] Update manifests.' %
(len(ids), update_manifests.rate_limit))
# Since we'll be logging the updated manifest change to the users log,
# we'll need to log in as user.
amo.set_user(get_task_user())
for id in ids:
_update_manifest(id)
def update_maxversions(version_pks, job_pk, data, **kw):
log.info('[%s@%s] Updating max version for job %s.' %
(len(version_pks), update_maxversions.rate_limit, job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
dry_run = data['preview_only']
app_id = job.target_version.application
stats = collections.defaultdict(int)
stats['processed'] = 0
stats['is_dry_run'] = int(dry_run)
for version in Version.objects.filter(pk__in=version_pks):
stats['processed'] += 1
addon = version.addon
file_pks = version.files.values_list('pk', flat=True)
errors = (ValidationResult.objects.filter(
validation_job=job, file__pk__in=file_pks).values_list('errors',
flat=True))
if any(errors):
stats['invalid'] += 1
log.info('Version %s for addon %s not updated, '
'one of the files did not pass validation' %
(version.pk, version.addon.pk))
continue
for app in version.apps.filter(
application=job.curr_max_version.application,
max__version_int__gte=job.curr_max_version.version_int,
max__version_int__lt=job.target_version.version_int):
stats['bumped'] += 1
log.info(
'Updating version %s%s for addon %s from version %s '
'to version %s' %
(version.pk, ' [DRY RUN]' if dry_run else '', version.addon.pk,
job.curr_max_version.version, job.target_version.version))
app.max = job.target_version
if not dry_run:
app.save()
amo.log(amo.LOG.MAX_APPVERSION_UPDATED,
version.addon,
version,
details={
'version': version.version,
'target': job.target_version.version,
'application': app_id
})
log.info('[%s@%s] bulk update stats for job %s: {%s}' %
(len(version_pks), update_maxversions.rate_limit, job_pk,
', '.join('%s: %s' % (k, stats[k])
for k in sorted(stats.keys()))))
def update_maxversions(version_pks, job_pk, data, **kw):
log.info('[%s@%s] Updating max version for job %s.'
% (len(version_pks), update_maxversions.rate_limit, job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
dry_run = data['preview_only']
app_id = job.target_version.application.pk
stats = collections.defaultdict(int)
stats['processed'] = 0
stats['is_dry_run'] = int(dry_run)
for version in Version.objects.filter(pk__in=version_pks):
stats['processed'] += 1
addon = version.addon
file_pks = version.files.values_list('pk', flat=True)
errors = (ValidationResult.objects.filter(validation_job=job,
file__pk__in=file_pks)
.values_list('errors', flat=True))
if any(errors):
stats['invalid'] += 1
log.info('Version %s for addon %s not updated, '
'one of the files did not pass validation'
% (version.pk, version.addon.pk))
continue
for app in version.apps.filter(
application=job.curr_max_version.application,
max__version_int__gte=job.curr_max_version.version_int,
max__version_int__lt=job.target_version.version_int):
stats['bumped'] += 1
log.info('Updating version %s%s for addon %s from version %s '
'to version %s'
% (version.pk,
' [DRY RUN]' if dry_run else '',
version.addon.pk,
job.curr_max_version.version,
job.target_version.version))
app.max = job.target_version
if not dry_run:
app.save()
amo.log(amo.LOG.MAX_APPVERSION_UPDATED,
version.addon, version,
details={'version': version.version,
'target': job.target_version.version,
'application': app_id})
log.info('[%s@%s] bulk update stats for job %s: {%s}'
% (len(version_pks), update_maxversions.rate_limit, job_pk,
', '.join('%s: %s' % (k, stats[k])
for k in sorted(stats.keys()))))
def update_manifests(ids, **kw):
task_log.info("[%s@%s] Update manifests." % (len(ids), update_manifests.rate_limit))
check_hash = kw.pop("check_hash", True)
retries = kw.pop("retries", {})
# Since we'll be logging the updated manifest change to the users log,
# we'll need to log in as user.
amo.set_user(get_task_user())
for id in ids:
_update_manifest(id, check_hash, retries)
if retries:
update_manifests.retry(
args=(retries.keys(),), kwargs={"check_hash": check_hash, "retries": retries}, countdown=3600
)
return retries
def find_refund_escalations(addon_id, **kw):
try:
addon = Addon.objects.get(pk=addon_id)
except Addon.DoesNotExist:
log.info(u'[addon:%s] Task called but no addon found.' % addon_id)
return
refund_threshold = 0.05
amo.set_user(get_task_user())
weekago = datetime.date.today() - datetime.timedelta(days=7)
ratio = Refund.recent_refund_ratio(addon.id, weekago)
if ratio > refund_threshold:
if EscalationQueue.objects.filter(addon=addon).exists():
# App is already in the queue, no need to re-add it.
# TODO: If not in queue b/c of refunds, add an
# amo.LOG.ESCALATED_HIGH_REFUNDS for reviewers.
log.info(u'[addon:%s] High refunds, but already escalated' % addon)
return
# High refunds and app isn't in the escalation queue... let's see if
# it has been detected and dealt with already.
logs = (AppLog.objects.filter(
activity_log__action=amo.LOG.ESCALATED_HIGH_REFUNDS.id,
addon=addon).order_by('-created', '-id'))
if logs:
since_ratio = Refund.recent_refund_ratio(addon.id, logs[0].created)
# If not high enough ratio since the last logged, do not add to
# the queue.
if not since_ratio > refund_threshold:
log.info(u'[addon:%s] High refunds, but not enough since last '
'escalation. Ratio: %.0f%%' % (addon,
since_ratio * 100))
return
# If we haven't bailed out yet, escalate this app.
msg = u'High number of refund requests (%.0f%%) detected.' % (
(ratio * 100),)
EscalationQueue.objects.create(addon=addon)
amo.log(amo.LOG.ESCALATED_HIGH_REFUNDS, addon,
addon.current_version, details={'comments': msg})
log.info(u'[addon:%s] %s' % (addon, msg))
def update_manifests(ids, **kw):
task_log.info('[%s@%s] Update manifests.' %
(len(ids), update_manifests.rate_limit))
# Since we'll be logging the updated manifest change to the users log,
# we'll need to log in as user.
set_user(get_task_user())
for id in ids:
task_log.info('Fetching webapp manifest for: %s' % id)
webapp = Webapp.objects.get(pk=id)
file_ = webapp.get_latest_file()
if not file_:
task_log.info('Ignoring, no existing file for: %s' % id)
continue
# Fetch the data.
temp = mkstemp(suffix='.webapp')[1]
try:
hash_ = _get_content_hash(temp, webapp.manifest_url)
except:
task_log.info('Failed to get manifest for: %s, %s' %
(id, webapp.manifest_url), exc_info=True)
rm_local_tmp_file(temp)
continue
# Try to create a new version, if needed.
try:
if file_.hash != hash_:
task_log.info('Webapp manifest different for: %s, %s' %
(id, webapp.manifest_url))
webapp.manifest_updated(temp)
else:
task_log.info('Webapp manifest the same for: %s, %s' %
(id, webapp.manifest_url))
except:
task_log.info('Failed to create version for: %s' % id,
exc_info=True)
finally:
rm_local_tmp_file(temp)
def update_manifests(ids, **kw):
retry_secs = 3600
task_log.info("[%s@%s] Update manifests." % (len(ids), update_manifests.rate_limit))
check_hash = kw.pop("check_hash", True)
retries = kw.pop("retries", {})
# Since we'll be logging the updated manifest change to the users log,
# we'll need to log in as user.
amo.set_user(get_task_user())
for id in ids:
_update_manifest(id, check_hash, retries)
if retries:
try:
update_manifests.retry(
args=(retries.keys(),),
kwargs={"check_hash": check_hash, "retries": retries},
eta=datetime.datetime.now() + datetime.timedelta(seconds=retry_secs),
max_retries=4,
)
except RetryTaskError:
_log(id, "Retrying task in %d seconds." % retry_secs)
return retries
def notify_success(version_pks, job_pk, data, **kw):
log.info('[%s@None] Updating max version for job %s.' %
(len(version_pks), job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
for version in Version.objects.filter(pk__in=version_pks):
addon = version.addon
file_pks = version.files.values_list('pk', flat=True)
errors = (ValidationResult.objects.filter(
validation_job=job, file__pk__in=file_pks).values_list('errors',
flat=True))
if any(errors):
log.info('Version %s for addon %s not updated, '
'one of the files did not pass validation' %
(version.pk, version.addon.pk))
continue
app_flag = False
dry_run = data['preview_only']
for app in version.apps.filter(
application=job.curr_max_version.application):
if (app.max.version == job.curr_max_version.version
and job.target_version.version != app.max.version):
log.info('Updating version %s%s for addon %s from version %s '
'to version %s' %
(version.pk, ' [DRY RUN]' if dry_run else '',
version.addon.pk, job.curr_max_version.version,
job.target_version.version))
app.max = job.target_version
if not dry_run:
app.save()
app_flag = True
else:
log.info('Version %s for addon %s not updated, '
'current max version is %s not %s' %
(version.pk, version.addon.pk, app.max.version,
job.curr_max_version.version))
if app_flag:
results = job.result_set.filter(file__version=version)
context = get_context(addon, version, job, results)
for author in addon.authors.all():
log.info(u'Emailing %s%s for addon %s, version %s about '
'success from bulk validation job %s' %
(author.email, ' [PREVIEW]' if dry_run else '',
addon.pk, version.pk, job_pk))
args = (Template(data['subject']).render(context),
Template(data['text']).render(context))
kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=[author.email])
if dry_run:
job.preview_success_mail(*args, **kwargs)
else:
send_mail(*args, **kwargs)
amo.log(amo.LOG.BULK_VALIDATION_UPDATED,
version.addon,
version,
details={
'version': version.version,
'target': job.target_version.version
})
def _notify(redis, context):
"""
Notify the admins or the developers what happened. Performing a sanity
check in case something went wrong.
"""
log.info('Completed run of paypal addon checks: %s' % context['checked'])
if context['limit'] and context['rate'] > context['limit']:
# This is too cope with something horrible going wrong like, paypal
# goes into maintenance mode, or netops changes something and all of
# a sudden all the apps start failing their paypal checks.
#
# It would really suck if one errant current job disabled every app
# on the marketplace. So this is an attempt to sanity check this.
log.info('Too many failed: %s%%, aborting.' % context['limit'])
template = 'market/emails/check_error.txt'
send_mail('Cron job error on checking addons',
env.get_template(template).render(context),
recipient_list=[settings.FLIGTAR],
from_email=settings.NOBODY_EMAIL)
else:
if not context['failed']:
return
failure_list = []
for k in xrange(context['failed']):
data = json.loads(redis.lindex(failures, k))
addon = Addon.objects.get(pk=data[0])
url = absolutify(shared_url('detail', addon))
# Add this to a list so we can tell the admins who got disabled.
failure_list.append(url)
if not context['do_disable']:
continue
# Write to the developers log that it failed to pass and update
# the status of the addon.
amo.log(amo.LOG.PAYPAL_FAILED, addon, user=get_task_user())
addon.update(status=amo.STATUS_DISABLED)
authors = [u.email for u in addon.authors.all()]
if addon.is_webapp():
template = 'market/emails/check_developer_app.txt'
subject = loc('App disabled on the Mozilla Market.')
else:
template = 'market/emails/check_developer_addon.txt'
subject = loc('Add-on disabled on the Mozilla Market.')
# Now email the developer and tell them the bad news.
url = absolutify(shared_url('detail', addon))
send_mail(subject,
env.get_template(template).render({
'addon': addon,
'errors': data[1:]
}),
recipient_list=authors,
from_email=settings.NOBODY_EMAIL)
context['failure_list'] = failure_list
# Now email the admins and tell them what happened.
template = 'market/emails/check_summary.txt'
send_mail('Cron job disabled %s add-ons' % context['failed'],
env.get_template(template).render(context),
recipient_list=[settings.FLIGTAR],
from_email=settings.NOBODY_EMAIL)
def notify_compatibility_chunk(users, job, data, **kw):
log.info("[%s@%s] Sending notification mail for job %s." % (len(users), notify_compatibility.rate_limit, job.pk))
set_user(get_task_user())
dry_run = data["preview_only"]
app_id = job.target_version.application
stats = collections.defaultdict(int)
stats["processed"] = 0
stats["is_dry_run"] = int(dry_run)
for user in users:
stats["processed"] += 1
try:
for a in chain(user.passing_addons, user.failing_addons):
try:
results = job.result_set.filter(file__version__addon=a)
a.links = [absolutify(reverse("devhub.bulk_compat_result", args=[a.slug, r.pk])) for r in results]
v = a.current_version or a.latest_version
a.compat_link = absolutify(reverse("devhub.versions.edit", args=[a.pk, v.pk]))
except:
task_error = sys.exc_info()
log.error(
u"Bulk validation email error for user %s, "
u"addon %s: %s: %s" % (user.email, a.slug, task_error[0], task_error[1]),
exc_info=False,
)
context = Context(
{
"APPLICATION": unicode(amo.APP_IDS[job.application].pretty),
"VERSION": job.target_version.version,
"PASSING_ADDONS": user.passing_addons,
"FAILING_ADDONS": user.failing_addons,
}
)
log.info(
u"Emailing %s%s for %d addons about "
"bulk validation job %s"
% (
user.email,
" [PREVIEW]" if dry_run else "",
len(user.passing_addons) + len(user.failing_addons),
job.pk,
)
)
args = (Template(data["subject"]).render(context), Template(data["text"]).render(context))
kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[user.email])
if dry_run:
job.preview_notify_mail(*args, **kwargs)
else:
stats["author_emailed"] += 1
send_mail(*args, **kwargs)
amo.log(
amo.LOG.BULK_VALIDATION_USER_EMAILED,
user,
details={
"passing": [a.id for a in user.passing_addons],
"failing": [a.id for a in user.failing_addons],
"target": job.target_version.version,
"application": app_id,
},
)
except:
task_error = sys.exc_info()
log.error(
u"Bulk validation email error for user %s: %s: %s" % (user.email, task_error[0], task_error[1]),
exc_info=False,
)
log.info(
"[%s@%s] bulk email stats for job %s: {%s}"
% (
len(users),
notify_compatibility.rate_limit,
job.pk,
", ".join("%s: %s" % (k, stats[k]) for k in sorted(stats.keys())),
)
)
def notify_success(version_pks, job_pk, data, **kw):
log.info("[%s@%s] Updating max version for job %s." % (len(version_pks), notify_success.rate_limit, job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
dry_run = data["preview_only"]
stats = collections.defaultdict(int)
stats["processed"] = 0
stats["is_dry_run"] = int(dry_run)
for version in Version.objects.filter(pk__in=version_pks):
stats["processed"] += 1
addon = version.addon
file_pks = version.files.values_list("pk", flat=True)
errors = ValidationResult.objects.filter(validation_job=job, file__pk__in=file_pks).values_list(
"errors", flat=True
)
if any(errors):
stats["invalid"] += 1
log.info(
"Version %s for addon %s not updated, "
"one of the files did not pass validation" % (version.pk, version.addon.pk)
)
continue
app_flag = False
for app in version.apps.filter(application=job.curr_max_version.application):
if (
app.max.version_int >= job.curr_max_version.version_int
and app.max.version_int < job.target_version.version_int
):
stats["bumped"] += 1
log.info(
"Updating version %s%s for addon %s from version %s "
"to version %s"
% (
version.pk,
" [DRY RUN]" if dry_run else "",
version.addon.pk,
job.curr_max_version.version,
job.target_version.version,
)
)
app.max = job.target_version
if not dry_run:
app.save()
app_flag = True
else:
stats["missed_targets"] += 1
log.info(
"Version %s for addon %s not updated, "
"current max version is %s not %s"
% (version.pk, version.addon.pk, app.max.version, job.curr_max_version.version)
)
if app_flag:
results = job.result_set.filter(file__version=version)
context = get_context(addon, version, job, results)
for author in addon.authors.all():
log.info(
u"Emailing %s%s for addon %s, version %s about "
"success from bulk validation job %s"
% (author.email, " [PREVIEW]" if dry_run else "", addon.pk, version.pk, job_pk)
)
args = (Template(data["subject"]).render(context), Template(data["text"]).render(context))
kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[author.email])
if dry_run:
job.preview_success_mail(*args, **kwargs)
else:
stats["author_emailed"] += 1
send_mail(*args, **kwargs)
app_id = job.target_version.application.pk
amo.log(
amo.LOG.MAX_APPVERSION_UPDATED,
version.addon,
version,
details={
"version": version.version,
"target": job.target_version.version,
"application": app_id,
},
)
log.info(
"[%s@%s] bulk update stats for job %s: {%s}"
% (
len(version_pks),
notify_success.rate_limit,
job_pk,
", ".join("%s: %s" % (k, stats[k]) for k in sorted(stats.keys())),
)
)
def notify_success(version_pks, job_pk, data, **kw):
log.info("[%s@None] Updating max version for job %s." % (len(version_pks), job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
for version in Version.objects.filter(pk__in=version_pks):
addon = version.addon
file_pks = version.files.values_list("pk", flat=True)
errors = ValidationResult.objects.filter(validation_job=job, file__pk__in=file_pks).values_list(
"errors", flat=True
)
if any(errors):
log.info(
"Version %s for addon %s not updated, "
"one of the files did not pass validation" % (version.pk, version.addon.pk)
)
continue
app_flag = False
dry_run = data["preview_only"]
for app in version.apps.filter(application=job.curr_max_version.application):
if app.max.version == job.curr_max_version.version and job.target_version.version != app.max.version:
log.info(
"Updating version %s%s for addon %s from version %s "
"to version %s"
% (
version.pk,
" [DRY RUN]" if dry_run else "",
version.addon.pk,
job.curr_max_version.version,
job.target_version.version,
)
)
app.max = job.target_version
if not dry_run:
app.save()
app_flag = True
else:
log.info(
"Version %s for addon %s not updated, "
"current max version is %s not %s"
% (version.pk, version.addon.pk, app.max.version, job.curr_max_version.version)
)
if app_flag:
results = job.result_set.filter(file__version=version)
context = get_context(addon, version, job, results)
for author in addon.authors.all():
log.info(
u"Emailing %s%s for addon %s, version %s about "
"success from bulk validation job %s"
% (author.email, " [PREVIEW]" if dry_run else "", addon.pk, version.pk, job_pk)
)
args = (Template(data["subject"]).render(context), Template(data["text"]).render(context))
kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[author.email])
if dry_run:
job.preview_success_mail(*args, **kwargs)
else:
send_mail(*args, **kwargs)
amo.log(
amo.LOG.BULK_VALIDATION_UPDATED,
version.addon,
version,
details={"version": version.version, "target": job.target_version.version},
)
def notify_success(version_pks, job_pk, data, **kw):
log.info('[%s@%s] Updating max version for job %s.' %
(len(version_pks), notify_success.rate_limit, job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
dry_run = data['preview_only']
stats = collections.defaultdict(int)
stats['processed'] = 0
stats['is_dry_run'] = int(dry_run)
for version in Version.objects.filter(pk__in=version_pks):
stats['processed'] += 1
addon = version.addon
file_pks = version.files.values_list('pk', flat=True)
errors = (ValidationResult.objects.filter(
validation_job=job, file__pk__in=file_pks).values_list('errors',
flat=True))
if any(errors):
stats['invalid'] += 1
log.info('Version %s for addon %s not updated, '
'one of the files did not pass validation' %
(version.pk, version.addon.pk))
continue
app_flag = False
for app in version.apps.filter(
application=job.curr_max_version.application):
if (app.max.version_int >= job.curr_max_version.version_int
and app.max.version_int < job.target_version.version_int):
stats['bumped'] += 1
log.info('Updating version %s%s for addon %s from version %s '
'to version %s' %
(version.pk, ' [DRY RUN]' if dry_run else '',
version.addon.pk, job.curr_max_version.version,
job.target_version.version))
app.max = job.target_version
if not dry_run:
app.save()
app_flag = True
else:
stats['missed_targets'] += 1
log.info('Version %s for addon %s not updated, '
'current max version is %s not %s' %
(version.pk, version.addon.pk, app.max.version,
job.curr_max_version.version))
if app_flag:
results = job.result_set.filter(file__version=version)
context = get_context(addon, version, job, results)
for author in addon.authors.all():
log.info(u'Emailing %s%s for addon %s, version %s about '
'success from bulk validation job %s' %
(author.email, ' [PREVIEW]' if dry_run else '',
addon.pk, version.pk, job_pk))
args = (Template(data['subject']).render(context),
Template(data['text']).render(context))
kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=[author.email])
if dry_run:
job.preview_success_mail(*args, **kwargs)
else:
stats['author_emailed'] += 1
send_mail(*args, **kwargs)
app_id = job.target_version.application.pk
amo.log(amo.LOG.MAX_APPVERSION_UPDATED,
version.addon,
version,
details={
'version': version.version,
'target': job.target_version.version,
'application': app_id
})
log.info('[%s@%s] bulk update stats for job %s: {%s}' %
(len(version_pks), notify_success.rate_limit, job_pk, ', '.join(
'%s: %s' % (k, stats[k]) for k in sorted(stats.keys()))))
def notify_success(version_pks, job_pk, data, **kw):
log.info('[%s@%s] Updating max version for job %s.'
% (len(version_pks), notify_success.rate_limit, job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
dry_run = data['preview_only']
stats = collections.defaultdict(int)
stats['processed'] = 0
stats['is_dry_run'] = int(dry_run)
for version in Version.objects.filter(pk__in=version_pks):
stats['processed'] += 1
addon = version.addon
file_pks = version.files.values_list('pk', flat=True)
errors = (ValidationResult.objects.filter(validation_job=job,
file__pk__in=file_pks)
.values_list('errors', flat=True))
if any(errors):
stats['invalid'] += 1
log.info('Version %s for addon %s not updated, '
'one of the files did not pass validation'
% (version.pk, version.addon.pk))
continue
app_flag = False
for app in version.apps.filter(
application=job.curr_max_version.application):
if (app.max.version_int >= job.curr_max_version.version_int and
app.max.version_int < job.target_version.version_int):
stats['bumped'] += 1
log.info('Updating version %s%s for addon %s from version %s '
'to version %s'
% (version.pk,
' [DRY RUN]' if dry_run else '',
version.addon.pk,
job.curr_max_version.version,
job.target_version.version))
app.max = job.target_version
if not dry_run:
app.save()
app_flag = True
else:
stats['missed_targets'] += 1
log.info('Version %s for addon %s not updated, '
'current max version is %s not %s'
% (version.pk, version.addon.pk,
app.max.version, job.curr_max_version.version))
if app_flag:
results = job.result_set.filter(file__version=version)
context = get_context(addon, version, job, results)
for author in addon.authors.all():
log.info(u'Emailing %s%s for addon %s, version %s about '
'success from bulk validation job %s'
% (author.email,
' [PREVIEW]' if dry_run else '',
addon.pk, version.pk, job_pk))
args = (Template(data['subject']).render(context),
Template(data['text']).render(context))
kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=[author.email])
if dry_run:
job.preview_success_mail(*args, **kwargs)
else:
stats['author_emailed'] += 1
send_mail(*args, **kwargs)
app_id = job.target_version.application.pk
amo.log(amo.LOG.MAX_APPVERSION_UPDATED,
version.addon, version,
details={'version': version.version,
'target': job.target_version.version,
'application': app_id})
log.info('[%s@%s] bulk update stats for job %s: {%s}'
% (len(version_pks), notify_success.rate_limit, job_pk,
', '.join('%s: %s' % (k, stats[k])
for k in sorted(stats.keys()))))
def _notify(context):
"""
Notify the admins or the developers what happened. Performing a sanity
check in case something went wrong.
"""
log.info('Completed run of paypal addon checks: %s' % context['checked'])
failure_list = []
if context['limit'] and context['rate'] > context['limit']:
# This is too cope with something horrible going wrong like, paypal
# goes into maintenance mode, or netops changes something and all of
# a sudden all the apps start failing their paypal checks.
#
# It would really suck if one errant current job disabled every app
# on the Marketplace. So this is an attempt to sanity check this.
for f in context['failed_addons']:
failure_list.append(absolutify(f.addon.get_url_path()))
context['failure_list'] = failure_list
log.info('Too many failed: %s%%, aborting.' % context['limit'])
template = 'market/emails/check_error.txt'
send_mail('Cron job error on checking addons',
env.get_template(template).render(context),
recipient_list=[settings.FLIGTAR],
from_email=settings.NOBODY_EMAIL)
else:
if not context['failed_addons']:
return
for f in context['failed_addons']:
addon = f.addon
url = absolutify(addon.get_url_path())
# Add this to a list so we can tell the admins who got disabled.
failure_list.append(url)
if not context['do_disable']:
continue
# Write to the developers log that it failed to pass and update
# the status of the addon.
amo.log(amo.LOG.PAYPAL_FAILED, addon, user=get_task_user())
addon.update(status=amo.STATUS_DISABLED)
authors = [u.email for u in addon.authors.all()]
if addon.is_webapp():
template = 'market/emails/check_developer_app.txt'
subject = _('App disabled on the Firefox Marketplace.')
else:
template = 'market/emails/check_developer_addon.txt'
subject = _('Add-on disabled on the Firefox Marketplace.')
# Now email the developer and tell them the bad news.
send_mail(subject,
env.get_template(template).render({
'addon':
addon,
'errors':
json.loads(f.failure_data)
}),
recipient_list=authors,
from_email=settings.NOBODY_EMAIL)
context['failure_list'] = failure_list
# Now email the admins and tell them what happened.
template = 'market/emails/check_summary.txt'
send_mail('Cron job disabled %s add-ons' % context['failed'],
env.get_template(template).render(context),
recipient_list=[settings.FLIGTAR],
from_email=settings.NOBODY_EMAIL)
def notify_success(version_pks, job_pk, data, **kw):
log.info('[%s@None] Updating max version for job %s.'
% (len(version_pks), job_pk))
job = ValidationJob.objects.get(pk=job_pk)
set_user(get_task_user())
for version in Version.objects.filter(pk__in=version_pks):
addon = version.addon
file_pks = version.files.values_list('pk', flat=True)
errors = (ValidationResult.objects.filter(validation_job=job,
file__pk__in=file_pks)
.values_list('errors', flat=True))
if any(errors):
log.info('Version %s for addon %s not updated, '
'one of the files did not pass validation'
% (version.pk, version.addon.pk))
continue
app_flag = False
dry_run = data['preview_only']
for app in version.apps.filter(application=
job.curr_max_version.application):
if (app.max.version == job.curr_max_version.version and
job.target_version.version != app.max.version):
log.info('Updating version %s%s for addon %s from version %s '
'to version %s'
% (version.pk,
' [DRY RUN]' if dry_run else '',
version.addon.pk,
job.curr_max_version.version,
job.target_version.version))
app.max = job.target_version
if not dry_run:
app.save()
app_flag = True
else:
log.info('Version %s for addon %s not updated, '
'current max version is %s not %s'
% (version.pk, version.addon.pk,
app.max.version, job.curr_max_version.version))
if app_flag:
results = job.result_set.filter(file__version=version)
context = get_context(addon, version, job, results)
for author in addon.authors.all():
log.info(u'Emailing %s%s for addon %s, version %s about '
'success from bulk validation job %s'
% (author.email,
' [PREVIEW]' if dry_run else '',
addon.pk, version.pk, job_pk))
args = (Template(data['subject']).render(context),
Template(data['text']).render(context))
kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=[author.email])
if dry_run:
job.preview_success_mail(*args, **kwargs)
else:
send_mail(*args, **kwargs)
amo.log(amo.LOG.BULK_VALIDATION_UPDATED,
version.addon, version,
details={'version': version.version,
'target': job.target_version.version})