def notify_failed(file_pks, job_pk, data, **kw): log.info('[%s@None] Notifying failed for job %s.' % (len(file_pks), job_pk)) job = ValidationJob.objects.get(pk=job_pk) set_user(get_task_user()) for result in ValidationResult.objects.filter(validation_job=job, file__pk__in=file_pks): file = result.file version = file.version addon = version.addon context = get_context(addon, version, job, [result], fileob=file) for author in addon.authors.all(): log.info( u'Emailing %s%s for addon %s, file %s about ' 'error from bulk validation job %s' % (author.email, ' [PREVIEW]' if data['preview_only'] else '', addon.pk, file.pk, job_pk)) args = (Template(data['subject']).render(context), Template(data['text']).render(context)) kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[author.email]) if data['preview_only']: job.preview_failure_mail(*args, **kwargs) else: send_mail(*args, **kwargs) amo.log(amo.LOG.BULK_VALIDATION_EMAILED, addon, version, details={ 'version': version.version, 'file': file.filename, 'target': job.target_version.version })
def notify_failed(file_pks, job_pk, data, **kw): log.info("[%s@None] Notifying failed for job %s." % (len(file_pks), job_pk)) job = ValidationJob.objects.get(pk=job_pk) set_user(get_task_user()) for result in ValidationResult.objects.filter(validation_job=job, file__pk__in=file_pks): file = result.file version = file.version addon = version.addon context = get_context(addon, version, job, [result], fileob=file) for author in addon.authors.all(): log.info( u"Emailing %s%s for addon %s, file %s about " "error from bulk validation job %s" % (author.email, " [PREVIEW]" if data["preview_only"] else "", addon.pk, file.pk, job_pk) ) args = (Template(data["subject"]).render(context), Template(data["text"]).render(context)) kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[author.email]) if data["preview_only"]: job.preview_failure_mail(*args, **kwargs) else: send_mail(*args, **kwargs) amo.log( amo.LOG.BULK_VALIDATION_EMAILED, addon, version, details={"version": version.version, "file": file.filename, "target": job.target_version.version}, )
def find_abuse_escalations(addon_id, **kw): amo.set_user(get_task_user()) weekago = datetime.date.today() - datetime.timedelta(days=7) for abuse in AbuseReport.recent_high_abuse_reports(1, weekago, addon_id): if EscalationQueue.objects.filter(addon=abuse.addon).exists(): # App is already in the queue, no need to re-add it. # TODO: If not in queue b/c of abuse reports, add an # amo.LOG.ESCALATED_HIGH_ABUSE for reviewers. log.info(u'[addon:%s] High abuse reports, but already escalated' % (abuse.addon,)) continue # We have an abuse report and this add-on isn't currently in the # escalated queue... let's see if it has been detected and dealt with # already. logs = (AppLog.objects.filter( activity_log__action=amo.LOG.ESCALATED_HIGH_ABUSE.id, addon=abuse.addon).order_by('-created')) if logs: abuse_since_log = AbuseReport.recent_high_abuse_reports( 1, logs[0].created, addon_id) # If no abuse reports have happened since the last logged abuse # report, do not add to queue. if not abuse_since_log: log.info(u'[addon:%s] High abuse reports, but none since last ' 'escalation' % abuse.addon) continue # If we haven't bailed out yet, escalate this app. msg = u'High number of abuse reports detected' EscalationQueue.objects.create(addon=abuse.addon) amo.log(amo.LOG.ESCALATED_HIGH_ABUSE, abuse.addon, abuse.addon.current_version, details={'comments': msg}) log.info(u'[addon:%s] %s' % (abuse.addon, msg))
def update_maxversions(version_pks, job_pk, data, **kw): log.info("[%s@%s] Updating max version for job %s." % (len(version_pks), update_maxversions.rate_limit, job_pk)) job = ValidationJob.objects.get(pk=job_pk) set_user(get_task_user()) dry_run = data["preview_only"] app_id = job.target_version.application stats = collections.defaultdict(int) stats["processed"] = 0 stats["is_dry_run"] = int(dry_run) for version in Version.objects.filter(pk__in=version_pks): stats["processed"] += 1 addon = version.addon file_pks = version.files.values_list("pk", flat=True) errors = ValidationResult.objects.filter(validation_job=job, file__pk__in=file_pks).values_list( "errors", flat=True ) if any(errors): stats["invalid"] += 1 log.info( "Version %s for addon %s not updated, " "one of the files did not pass validation" % (version.pk, version.addon.pk) ) continue for app in version.apps.filter( application=job.curr_max_version.application, max__version_int__gte=job.curr_max_version.version_int, max__version_int__lt=job.target_version.version_int, ): stats["bumped"] += 1 log.info( "Updating version %s%s for addon %s from version %s " "to version %s" % ( version.pk, " [DRY RUN]" if dry_run else "", version.addon.pk, job.curr_max_version.version, job.target_version.version, ) ) app.max = job.target_version if not dry_run: app.save() amo.log( amo.LOG.MAX_APPVERSION_UPDATED, version.addon, version, details={"version": version.version, "target": job.target_version.version, "application": app_id}, ) log.info( "[%s@%s] bulk update stats for job %s: {%s}" % ( len(version_pks), update_maxversions.rate_limit, job_pk, ", ".join("%s: %s" % (k, stats[k]) for k in sorted(stats.keys())), ) )
def notify_failed(file_pks, job_pk, data, **kw): log.info('[%s@None] Notifying failed for job %s.' % (len(file_pks), job_pk)) job = ValidationJob.objects.get(pk=job_pk) set_user(get_task_user()) for result in ValidationResult.objects.filter(validation_job=job, file__pk__in=file_pks): file = result.file version = file.version addon = version.addon context = get_context(addon, version, job, [result]) for author in addon.authors.all(): log.info(u'Emailing %s%s for addon %s, file %s about ' 'error from bulk validation job %s' % (author.email, ' [PREVIEW]' if data['preview_only'] else '', addon.pk, file.pk, job_pk)) args = (Template(data['subject']).render(context), Template(data['text']).render(context)) kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[author.email]) if data['preview_only']: job.preview_failure_mail(*args, **kwargs) else: send_mail(*args, **kwargs) amo.log(amo.LOG.BULK_VALIDATION_EMAILED, addon, version, details={'version': version.version, 'file': file.filename, 'target': job.target_version.version})
def update_manifests(ids, **kw): retry_secs = 3600 task_log.info('[%s@%s] Update manifests.' % (len(ids), update_manifests.rate_limit)) check_hash = kw.pop('check_hash', True) retries = kw.pop('retries', {}) # Since we'll be logging the updated manifest change to the users log, # we'll need to log in as user. amo.set_user(get_task_user()) for id in ids: _update_manifest(id, check_hash, retries) if retries: try: update_manifests.retry(args=(retries.keys(), ), kwargs={ 'check_hash': check_hash, 'retries': retries }, eta=datetime.datetime.now() + datetime.timedelta(seconds=retry_secs), max_retries=5) except RetryTaskError: _log(id, 'Retrying task in %d seconds.' % retry_secs) return retries
def test_update_pks_logs(self): self.create_result(self.job, self.create_file(self.version)) eq_(ActivityLog.objects.for_addons(self.addon).count(), 0) self.client.post(self.update_url, self.data) upd = amo.LOG.BULK_VALIDATION_UPDATED.id logs = ActivityLog.objects.for_addons(self.addon).filter(action=upd) eq_(logs.count(), 1) eq_(logs[0].user, get_task_user())
def wrapper(*args, **kw): old_user = get_user() set_user(get_task_user()) try: result = f(*args, **kw) finally: set_user(old_user) return result
def notify_compatibility(users, job, data, **kw): log.info('[%s@%s] Sending notification mail for job %s.' % (len(users), notify_compatibility.rate_limit, job.pk)) set_user(get_task_user()) dry_run = data['preview_only'] app_id = job.target_version.application.pk stats = collections.defaultdict(int) stats['processed'] = 0 stats['is_dry_run'] = int(dry_run) for user in users: stats['processed'] += 1 for a in chain(user.passing_addons, user.failing_addons): results = job.result_set.filter(file__version__addon=a) a.links = ' '.join(absolutify(reverse('devhub.bulk_compat_result', args=[a.slug, r.pk])) for r in results) a.compat_link = absolutify(reverse('devhub.versions.edit', args=[a.pk, a.current_version.pk])) context = Context({ 'APPLICATION': str(job.application), 'VERSION': job.target_version.version, 'PASSING_ADDONS': user.passing_addons, 'FAILING_ADDONS': user.failing_addons, }) log.info(u'Emailing %s%s for %d addons about ' 'bulk validation job %s' % (user.email, ' [PREVIEW]' if dry_run else '', len(user.passing_addons) + len(user.failing_addons), job.pk)) args = (Template(data['subject']).render(context), Template(data['text']).render(context)) kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[user.email]) if dry_run: job.preview_notify_mail(*args, **kwargs) else: stats['author_emailed'] += 1 send_mail(*args, **kwargs) amo.log(amo.LOG.BULK_VALIDATION_USER_EMAILED, user, details={'passing': [a.id for a in user.passing_addons], 'failing': [a.id for a in user.failing_addons], 'target': job.target_version.version, 'application': app_id}) log.info('[%s@%s] bulk email stats for job %s: {%s}' % (len(users), notify_compatibility.rate_limit, job.pk, ', '.join('%s: %s' % (k, stats[k]) for k in sorted(stats.keys()))))
def update_manifests(ids, **kw): task_log.info('[%s@%s] Update manifests.' % (len(ids), update_manifests.rate_limit)) # Since we'll be logging the updated manifest change to the users log, # we'll need to log in as user. amo.set_user(get_task_user()) for id in ids: _update_manifest(id)
def update_maxversions(version_pks, job_pk, data, **kw): log.info('[%s@%s] Updating max version for job %s.' % (len(version_pks), update_maxversions.rate_limit, job_pk)) job = ValidationJob.objects.get(pk=job_pk) set_user(get_task_user()) dry_run = data['preview_only'] app_id = job.target_version.application stats = collections.defaultdict(int) stats['processed'] = 0 stats['is_dry_run'] = int(dry_run) for version in Version.objects.filter(pk__in=version_pks): stats['processed'] += 1 addon = version.addon file_pks = version.files.values_list('pk', flat=True) errors = (ValidationResult.objects.filter( validation_job=job, file__pk__in=file_pks).values_list('errors', flat=True)) if any(errors): stats['invalid'] += 1 log.info('Version %s for addon %s not updated, ' 'one of the files did not pass validation' % (version.pk, version.addon.pk)) continue for app in version.apps.filter( application=job.curr_max_version.application, max__version_int__gte=job.curr_max_version.version_int, max__version_int__lt=job.target_version.version_int): stats['bumped'] += 1 log.info( 'Updating version %s%s for addon %s from version %s ' 'to version %s' % (version.pk, ' [DRY RUN]' if dry_run else '', version.addon.pk, job.curr_max_version.version, job.target_version.version)) app.max = job.target_version if not dry_run: app.save() amo.log(amo.LOG.MAX_APPVERSION_UPDATED, version.addon, version, details={ 'version': version.version, 'target': job.target_version.version, 'application': app_id }) log.info('[%s@%s] bulk update stats for job %s: {%s}' % (len(version_pks), update_maxversions.rate_limit, job_pk, ', '.join('%s: %s' % (k, stats[k]) for k in sorted(stats.keys()))))
def update_maxversions(version_pks, job_pk, data, **kw): log.info('[%s@%s] Updating max version for job %s.' % (len(version_pks), update_maxversions.rate_limit, job_pk)) job = ValidationJob.objects.get(pk=job_pk) set_user(get_task_user()) dry_run = data['preview_only'] app_id = job.target_version.application.pk stats = collections.defaultdict(int) stats['processed'] = 0 stats['is_dry_run'] = int(dry_run) for version in Version.objects.filter(pk__in=version_pks): stats['processed'] += 1 addon = version.addon file_pks = version.files.values_list('pk', flat=True) errors = (ValidationResult.objects.filter(validation_job=job, file__pk__in=file_pks) .values_list('errors', flat=True)) if any(errors): stats['invalid'] += 1 log.info('Version %s for addon %s not updated, ' 'one of the files did not pass validation' % (version.pk, version.addon.pk)) continue for app in version.apps.filter( application=job.curr_max_version.application, max__version_int__gte=job.curr_max_version.version_int, max__version_int__lt=job.target_version.version_int): stats['bumped'] += 1 log.info('Updating version %s%s for addon %s from version %s ' 'to version %s' % (version.pk, ' [DRY RUN]' if dry_run else '', version.addon.pk, job.curr_max_version.version, job.target_version.version)) app.max = job.target_version if not dry_run: app.save() amo.log(amo.LOG.MAX_APPVERSION_UPDATED, version.addon, version, details={'version': version.version, 'target': job.target_version.version, 'application': app_id}) log.info('[%s@%s] bulk update stats for job %s: {%s}' % (len(version_pks), update_maxversions.rate_limit, job_pk, ', '.join('%s: %s' % (k, stats[k]) for k in sorted(stats.keys()))))
def update_manifests(ids, **kw): task_log.info("[%s@%s] Update manifests." % (len(ids), update_manifests.rate_limit)) check_hash = kw.pop("check_hash", True) retries = kw.pop("retries", {}) # Since we'll be logging the updated manifest change to the users log, # we'll need to log in as user. amo.set_user(get_task_user()) for id in ids: _update_manifest(id, check_hash, retries) if retries: update_manifests.retry( args=(retries.keys(),), kwargs={"check_hash": check_hash, "retries": retries}, countdown=3600 ) return retries
def find_refund_escalations(addon_id, **kw): try: addon = Addon.objects.get(pk=addon_id) except Addon.DoesNotExist: log.info(u'[addon:%s] Task called but no addon found.' % addon_id) return refund_threshold = 0.05 amo.set_user(get_task_user()) weekago = datetime.date.today() - datetime.timedelta(days=7) ratio = Refund.recent_refund_ratio(addon.id, weekago) if ratio > refund_threshold: if EscalationQueue.objects.filter(addon=addon).exists(): # App is already in the queue, no need to re-add it. # TODO: If not in queue b/c of refunds, add an # amo.LOG.ESCALATED_HIGH_REFUNDS for reviewers. log.info(u'[addon:%s] High refunds, but already escalated' % addon) return # High refunds and app isn't in the escalation queue... let's see if # it has been detected and dealt with already. logs = (AppLog.objects.filter( activity_log__action=amo.LOG.ESCALATED_HIGH_REFUNDS.id, addon=addon).order_by('-created', '-id')) if logs: since_ratio = Refund.recent_refund_ratio(addon.id, logs[0].created) # If not high enough ratio since the last logged, do not add to # the queue. if not since_ratio > refund_threshold: log.info(u'[addon:%s] High refunds, but not enough since last ' 'escalation. Ratio: %.0f%%' % (addon, since_ratio * 100)) return # If we haven't bailed out yet, escalate this app. msg = u'High number of refund requests (%.0f%%) detected.' % ( (ratio * 100),) EscalationQueue.objects.create(addon=addon) amo.log(amo.LOG.ESCALATED_HIGH_REFUNDS, addon, addon.current_version, details={'comments': msg}) log.info(u'[addon:%s] %s' % (addon, msg))
def update_manifests(ids, **kw): task_log.info('[%s@%s] Update manifests.' % (len(ids), update_manifests.rate_limit)) # Since we'll be logging the updated manifest change to the users log, # we'll need to log in as user. set_user(get_task_user()) for id in ids: task_log.info('Fetching webapp manifest for: %s' % id) webapp = Webapp.objects.get(pk=id) file_ = webapp.get_latest_file() if not file_: task_log.info('Ignoring, no existing file for: %s' % id) continue # Fetch the data. temp = mkstemp(suffix='.webapp')[1] try: hash_ = _get_content_hash(temp, webapp.manifest_url) except: task_log.info('Failed to get manifest for: %s, %s' % (id, webapp.manifest_url), exc_info=True) rm_local_tmp_file(temp) continue # Try to create a new version, if needed. try: if file_.hash != hash_: task_log.info('Webapp manifest different for: %s, %s' % (id, webapp.manifest_url)) webapp.manifest_updated(temp) else: task_log.info('Webapp manifest the same for: %s, %s' % (id, webapp.manifest_url)) except: task_log.info('Failed to create version for: %s' % id, exc_info=True) finally: rm_local_tmp_file(temp)
def update_manifests(ids, **kw): retry_secs = 3600 task_log.info("[%s@%s] Update manifests." % (len(ids), update_manifests.rate_limit)) check_hash = kw.pop("check_hash", True) retries = kw.pop("retries", {}) # Since we'll be logging the updated manifest change to the users log, # we'll need to log in as user. amo.set_user(get_task_user()) for id in ids: _update_manifest(id, check_hash, retries) if retries: try: update_manifests.retry( args=(retries.keys(),), kwargs={"check_hash": check_hash, "retries": retries}, eta=datetime.datetime.now() + datetime.timedelta(seconds=retry_secs), max_retries=4, ) except RetryTaskError: _log(id, "Retrying task in %d seconds." % retry_secs) return retries
def notify_success(version_pks, job_pk, data, **kw): log.info('[%s@None] Updating max version for job %s.' % (len(version_pks), job_pk)) job = ValidationJob.objects.get(pk=job_pk) set_user(get_task_user()) for version in Version.objects.filter(pk__in=version_pks): addon = version.addon file_pks = version.files.values_list('pk', flat=True) errors = (ValidationResult.objects.filter( validation_job=job, file__pk__in=file_pks).values_list('errors', flat=True)) if any(errors): log.info('Version %s for addon %s not updated, ' 'one of the files did not pass validation' % (version.pk, version.addon.pk)) continue app_flag = False dry_run = data['preview_only'] for app in version.apps.filter( application=job.curr_max_version.application): if (app.max.version == job.curr_max_version.version and job.target_version.version != app.max.version): log.info('Updating version %s%s for addon %s from version %s ' 'to version %s' % (version.pk, ' [DRY RUN]' if dry_run else '', version.addon.pk, job.curr_max_version.version, job.target_version.version)) app.max = job.target_version if not dry_run: app.save() app_flag = True else: log.info('Version %s for addon %s not updated, ' 'current max version is %s not %s' % (version.pk, version.addon.pk, app.max.version, job.curr_max_version.version)) if app_flag: results = job.result_set.filter(file__version=version) context = get_context(addon, version, job, results) for author in addon.authors.all(): log.info(u'Emailing %s%s for addon %s, version %s about ' 'success from bulk validation job %s' % (author.email, ' [PREVIEW]' if dry_run else '', addon.pk, version.pk, job_pk)) args = (Template(data['subject']).render(context), Template(data['text']).render(context)) kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[author.email]) if dry_run: job.preview_success_mail(*args, **kwargs) else: send_mail(*args, **kwargs) amo.log(amo.LOG.BULK_VALIDATION_UPDATED, version.addon, version, details={ 'version': version.version, 'target': job.target_version.version })
def _notify(redis, context): """ Notify the admins or the developers what happened. Performing a sanity check in case something went wrong. """ log.info('Completed run of paypal addon checks: %s' % context['checked']) if context['limit'] and context['rate'] > context['limit']: # This is too cope with something horrible going wrong like, paypal # goes into maintenance mode, or netops changes something and all of # a sudden all the apps start failing their paypal checks. # # It would really suck if one errant current job disabled every app # on the marketplace. So this is an attempt to sanity check this. log.info('Too many failed: %s%%, aborting.' % context['limit']) template = 'market/emails/check_error.txt' send_mail('Cron job error on checking addons', env.get_template(template).render(context), recipient_list=[settings.FLIGTAR], from_email=settings.NOBODY_EMAIL) else: if not context['failed']: return failure_list = [] for k in xrange(context['failed']): data = json.loads(redis.lindex(failures, k)) addon = Addon.objects.get(pk=data[0]) url = absolutify(shared_url('detail', addon)) # Add this to a list so we can tell the admins who got disabled. failure_list.append(url) if not context['do_disable']: continue # Write to the developers log that it failed to pass and update # the status of the addon. amo.log(amo.LOG.PAYPAL_FAILED, addon, user=get_task_user()) addon.update(status=amo.STATUS_DISABLED) authors = [u.email for u in addon.authors.all()] if addon.is_webapp(): template = 'market/emails/check_developer_app.txt' subject = loc('App disabled on the Mozilla Market.') else: template = 'market/emails/check_developer_addon.txt' subject = loc('Add-on disabled on the Mozilla Market.') # Now email the developer and tell them the bad news. url = absolutify(shared_url('detail', addon)) send_mail(subject, env.get_template(template).render({ 'addon': addon, 'errors': data[1:] }), recipient_list=authors, from_email=settings.NOBODY_EMAIL) context['failure_list'] = failure_list # Now email the admins and tell them what happened. template = 'market/emails/check_summary.txt' send_mail('Cron job disabled %s add-ons' % context['failed'], env.get_template(template).render(context), recipient_list=[settings.FLIGTAR], from_email=settings.NOBODY_EMAIL)
def notify_compatibility_chunk(users, job, data, **kw): log.info("[%s@%s] Sending notification mail for job %s." % (len(users), notify_compatibility.rate_limit, job.pk)) set_user(get_task_user()) dry_run = data["preview_only"] app_id = job.target_version.application stats = collections.defaultdict(int) stats["processed"] = 0 stats["is_dry_run"] = int(dry_run) for user in users: stats["processed"] += 1 try: for a in chain(user.passing_addons, user.failing_addons): try: results = job.result_set.filter(file__version__addon=a) a.links = [absolutify(reverse("devhub.bulk_compat_result", args=[a.slug, r.pk])) for r in results] v = a.current_version or a.latest_version a.compat_link = absolutify(reverse("devhub.versions.edit", args=[a.pk, v.pk])) except: task_error = sys.exc_info() log.error( u"Bulk validation email error for user %s, " u"addon %s: %s: %s" % (user.email, a.slug, task_error[0], task_error[1]), exc_info=False, ) context = Context( { "APPLICATION": unicode(amo.APP_IDS[job.application].pretty), "VERSION": job.target_version.version, "PASSING_ADDONS": user.passing_addons, "FAILING_ADDONS": user.failing_addons, } ) log.info( u"Emailing %s%s for %d addons about " "bulk validation job %s" % ( user.email, " [PREVIEW]" if dry_run else "", len(user.passing_addons) + len(user.failing_addons), job.pk, ) ) args = (Template(data["subject"]).render(context), Template(data["text"]).render(context)) kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[user.email]) if dry_run: job.preview_notify_mail(*args, **kwargs) else: stats["author_emailed"] += 1 send_mail(*args, **kwargs) amo.log( amo.LOG.BULK_VALIDATION_USER_EMAILED, user, details={ "passing": [a.id for a in user.passing_addons], "failing": [a.id for a in user.failing_addons], "target": job.target_version.version, "application": app_id, }, ) except: task_error = sys.exc_info() log.error( u"Bulk validation email error for user %s: %s: %s" % (user.email, task_error[0], task_error[1]), exc_info=False, ) log.info( "[%s@%s] bulk email stats for job %s: {%s}" % ( len(users), notify_compatibility.rate_limit, job.pk, ", ".join("%s: %s" % (k, stats[k]) for k in sorted(stats.keys())), ) )
def notify_success(version_pks, job_pk, data, **kw): log.info("[%s@%s] Updating max version for job %s." % (len(version_pks), notify_success.rate_limit, job_pk)) job = ValidationJob.objects.get(pk=job_pk) set_user(get_task_user()) dry_run = data["preview_only"] stats = collections.defaultdict(int) stats["processed"] = 0 stats["is_dry_run"] = int(dry_run) for version in Version.objects.filter(pk__in=version_pks): stats["processed"] += 1 addon = version.addon file_pks = version.files.values_list("pk", flat=True) errors = ValidationResult.objects.filter(validation_job=job, file__pk__in=file_pks).values_list( "errors", flat=True ) if any(errors): stats["invalid"] += 1 log.info( "Version %s for addon %s not updated, " "one of the files did not pass validation" % (version.pk, version.addon.pk) ) continue app_flag = False for app in version.apps.filter(application=job.curr_max_version.application): if ( app.max.version_int >= job.curr_max_version.version_int and app.max.version_int < job.target_version.version_int ): stats["bumped"] += 1 log.info( "Updating version %s%s for addon %s from version %s " "to version %s" % ( version.pk, " [DRY RUN]" if dry_run else "", version.addon.pk, job.curr_max_version.version, job.target_version.version, ) ) app.max = job.target_version if not dry_run: app.save() app_flag = True else: stats["missed_targets"] += 1 log.info( "Version %s for addon %s not updated, " "current max version is %s not %s" % (version.pk, version.addon.pk, app.max.version, job.curr_max_version.version) ) if app_flag: results = job.result_set.filter(file__version=version) context = get_context(addon, version, job, results) for author in addon.authors.all(): log.info( u"Emailing %s%s for addon %s, version %s about " "success from bulk validation job %s" % (author.email, " [PREVIEW]" if dry_run else "", addon.pk, version.pk, job_pk) ) args = (Template(data["subject"]).render(context), Template(data["text"]).render(context)) kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[author.email]) if dry_run: job.preview_success_mail(*args, **kwargs) else: stats["author_emailed"] += 1 send_mail(*args, **kwargs) app_id = job.target_version.application.pk amo.log( amo.LOG.MAX_APPVERSION_UPDATED, version.addon, version, details={ "version": version.version, "target": job.target_version.version, "application": app_id, }, ) log.info( "[%s@%s] bulk update stats for job %s: {%s}" % ( len(version_pks), notify_success.rate_limit, job_pk, ", ".join("%s: %s" % (k, stats[k]) for k in sorted(stats.keys())), ) )
def notify_success(version_pks, job_pk, data, **kw): log.info("[%s@None] Updating max version for job %s." % (len(version_pks), job_pk)) job = ValidationJob.objects.get(pk=job_pk) set_user(get_task_user()) for version in Version.objects.filter(pk__in=version_pks): addon = version.addon file_pks = version.files.values_list("pk", flat=True) errors = ValidationResult.objects.filter(validation_job=job, file__pk__in=file_pks).values_list( "errors", flat=True ) if any(errors): log.info( "Version %s for addon %s not updated, " "one of the files did not pass validation" % (version.pk, version.addon.pk) ) continue app_flag = False dry_run = data["preview_only"] for app in version.apps.filter(application=job.curr_max_version.application): if app.max.version == job.curr_max_version.version and job.target_version.version != app.max.version: log.info( "Updating version %s%s for addon %s from version %s " "to version %s" % ( version.pk, " [DRY RUN]" if dry_run else "", version.addon.pk, job.curr_max_version.version, job.target_version.version, ) ) app.max = job.target_version if not dry_run: app.save() app_flag = True else: log.info( "Version %s for addon %s not updated, " "current max version is %s not %s" % (version.pk, version.addon.pk, app.max.version, job.curr_max_version.version) ) if app_flag: results = job.result_set.filter(file__version=version) context = get_context(addon, version, job, results) for author in addon.authors.all(): log.info( u"Emailing %s%s for addon %s, version %s about " "success from bulk validation job %s" % (author.email, " [PREVIEW]" if dry_run else "", addon.pk, version.pk, job_pk) ) args = (Template(data["subject"]).render(context), Template(data["text"]).render(context)) kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[author.email]) if dry_run: job.preview_success_mail(*args, **kwargs) else: send_mail(*args, **kwargs) amo.log( amo.LOG.BULK_VALIDATION_UPDATED, version.addon, version, details={"version": version.version, "target": job.target_version.version}, )
def notify_success(version_pks, job_pk, data, **kw): log.info('[%s@%s] Updating max version for job %s.' % (len(version_pks), notify_success.rate_limit, job_pk)) job = ValidationJob.objects.get(pk=job_pk) set_user(get_task_user()) dry_run = data['preview_only'] stats = collections.defaultdict(int) stats['processed'] = 0 stats['is_dry_run'] = int(dry_run) for version in Version.objects.filter(pk__in=version_pks): stats['processed'] += 1 addon = version.addon file_pks = version.files.values_list('pk', flat=True) errors = (ValidationResult.objects.filter( validation_job=job, file__pk__in=file_pks).values_list('errors', flat=True)) if any(errors): stats['invalid'] += 1 log.info('Version %s for addon %s not updated, ' 'one of the files did not pass validation' % (version.pk, version.addon.pk)) continue app_flag = False for app in version.apps.filter( application=job.curr_max_version.application): if (app.max.version_int >= job.curr_max_version.version_int and app.max.version_int < job.target_version.version_int): stats['bumped'] += 1 log.info('Updating version %s%s for addon %s from version %s ' 'to version %s' % (version.pk, ' [DRY RUN]' if dry_run else '', version.addon.pk, job.curr_max_version.version, job.target_version.version)) app.max = job.target_version if not dry_run: app.save() app_flag = True else: stats['missed_targets'] += 1 log.info('Version %s for addon %s not updated, ' 'current max version is %s not %s' % (version.pk, version.addon.pk, app.max.version, job.curr_max_version.version)) if app_flag: results = job.result_set.filter(file__version=version) context = get_context(addon, version, job, results) for author in addon.authors.all(): log.info(u'Emailing %s%s for addon %s, version %s about ' 'success from bulk validation job %s' % (author.email, ' [PREVIEW]' if dry_run else '', addon.pk, version.pk, job_pk)) args = (Template(data['subject']).render(context), Template(data['text']).render(context)) kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[author.email]) if dry_run: job.preview_success_mail(*args, **kwargs) else: stats['author_emailed'] += 1 send_mail(*args, **kwargs) app_id = job.target_version.application.pk amo.log(amo.LOG.MAX_APPVERSION_UPDATED, version.addon, version, details={ 'version': version.version, 'target': job.target_version.version, 'application': app_id }) log.info('[%s@%s] bulk update stats for job %s: {%s}' % (len(version_pks), notify_success.rate_limit, job_pk, ', '.join( '%s: %s' % (k, stats[k]) for k in sorted(stats.keys()))))
def notify_success(version_pks, job_pk, data, **kw): log.info('[%s@%s] Updating max version for job %s.' % (len(version_pks), notify_success.rate_limit, job_pk)) job = ValidationJob.objects.get(pk=job_pk) set_user(get_task_user()) dry_run = data['preview_only'] stats = collections.defaultdict(int) stats['processed'] = 0 stats['is_dry_run'] = int(dry_run) for version in Version.objects.filter(pk__in=version_pks): stats['processed'] += 1 addon = version.addon file_pks = version.files.values_list('pk', flat=True) errors = (ValidationResult.objects.filter(validation_job=job, file__pk__in=file_pks) .values_list('errors', flat=True)) if any(errors): stats['invalid'] += 1 log.info('Version %s for addon %s not updated, ' 'one of the files did not pass validation' % (version.pk, version.addon.pk)) continue app_flag = False for app in version.apps.filter( application=job.curr_max_version.application): if (app.max.version_int >= job.curr_max_version.version_int and app.max.version_int < job.target_version.version_int): stats['bumped'] += 1 log.info('Updating version %s%s for addon %s from version %s ' 'to version %s' % (version.pk, ' [DRY RUN]' if dry_run else '', version.addon.pk, job.curr_max_version.version, job.target_version.version)) app.max = job.target_version if not dry_run: app.save() app_flag = True else: stats['missed_targets'] += 1 log.info('Version %s for addon %s not updated, ' 'current max version is %s not %s' % (version.pk, version.addon.pk, app.max.version, job.curr_max_version.version)) if app_flag: results = job.result_set.filter(file__version=version) context = get_context(addon, version, job, results) for author in addon.authors.all(): log.info(u'Emailing %s%s for addon %s, version %s about ' 'success from bulk validation job %s' % (author.email, ' [PREVIEW]' if dry_run else '', addon.pk, version.pk, job_pk)) args = (Template(data['subject']).render(context), Template(data['text']).render(context)) kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[author.email]) if dry_run: job.preview_success_mail(*args, **kwargs) else: stats['author_emailed'] += 1 send_mail(*args, **kwargs) app_id = job.target_version.application.pk amo.log(amo.LOG.MAX_APPVERSION_UPDATED, version.addon, version, details={'version': version.version, 'target': job.target_version.version, 'application': app_id}) log.info('[%s@%s] bulk update stats for job %s: {%s}' % (len(version_pks), notify_success.rate_limit, job_pk, ', '.join('%s: %s' % (k, stats[k]) for k in sorted(stats.keys()))))
def _notify(context): """ Notify the admins or the developers what happened. Performing a sanity check in case something went wrong. """ log.info('Completed run of paypal addon checks: %s' % context['checked']) failure_list = [] if context['limit'] and context['rate'] > context['limit']: # This is too cope with something horrible going wrong like, paypal # goes into maintenance mode, or netops changes something and all of # a sudden all the apps start failing their paypal checks. # # It would really suck if one errant current job disabled every app # on the Marketplace. So this is an attempt to sanity check this. for f in context['failed_addons']: failure_list.append(absolutify(f.addon.get_url_path())) context['failure_list'] = failure_list log.info('Too many failed: %s%%, aborting.' % context['limit']) template = 'market/emails/check_error.txt' send_mail('Cron job error on checking addons', env.get_template(template).render(context), recipient_list=[settings.FLIGTAR], from_email=settings.NOBODY_EMAIL) else: if not context['failed_addons']: return for f in context['failed_addons']: addon = f.addon url = absolutify(addon.get_url_path()) # Add this to a list so we can tell the admins who got disabled. failure_list.append(url) if not context['do_disable']: continue # Write to the developers log that it failed to pass and update # the status of the addon. amo.log(amo.LOG.PAYPAL_FAILED, addon, user=get_task_user()) addon.update(status=amo.STATUS_DISABLED) authors = [u.email for u in addon.authors.all()] if addon.is_webapp(): template = 'market/emails/check_developer_app.txt' subject = _('App disabled on the Firefox Marketplace.') else: template = 'market/emails/check_developer_addon.txt' subject = _('Add-on disabled on the Firefox Marketplace.') # Now email the developer and tell them the bad news. send_mail(subject, env.get_template(template).render({ 'addon': addon, 'errors': json.loads(f.failure_data) }), recipient_list=authors, from_email=settings.NOBODY_EMAIL) context['failure_list'] = failure_list # Now email the admins and tell them what happened. template = 'market/emails/check_summary.txt' send_mail('Cron job disabled %s add-ons' % context['failed'], env.get_template(template).render(context), recipient_list=[settings.FLIGTAR], from_email=settings.NOBODY_EMAIL)
def notify_success(version_pks, job_pk, data, **kw): log.info('[%s@None] Updating max version for job %s.' % (len(version_pks), job_pk)) job = ValidationJob.objects.get(pk=job_pk) set_user(get_task_user()) for version in Version.objects.filter(pk__in=version_pks): addon = version.addon file_pks = version.files.values_list('pk', flat=True) errors = (ValidationResult.objects.filter(validation_job=job, file__pk__in=file_pks) .values_list('errors', flat=True)) if any(errors): log.info('Version %s for addon %s not updated, ' 'one of the files did not pass validation' % (version.pk, version.addon.pk)) continue app_flag = False dry_run = data['preview_only'] for app in version.apps.filter(application= job.curr_max_version.application): if (app.max.version == job.curr_max_version.version and job.target_version.version != app.max.version): log.info('Updating version %s%s for addon %s from version %s ' 'to version %s' % (version.pk, ' [DRY RUN]' if dry_run else '', version.addon.pk, job.curr_max_version.version, job.target_version.version)) app.max = job.target_version if not dry_run: app.save() app_flag = True else: log.info('Version %s for addon %s not updated, ' 'current max version is %s not %s' % (version.pk, version.addon.pk, app.max.version, job.curr_max_version.version)) if app_flag: results = job.result_set.filter(file__version=version) context = get_context(addon, version, job, results) for author in addon.authors.all(): log.info(u'Emailing %s%s for addon %s, version %s about ' 'success from bulk validation job %s' % (author.email, ' [PREVIEW]' if dry_run else '', addon.pk, version.pk, job_pk)) args = (Template(data['subject']).render(context), Template(data['text']).render(context)) kwargs = dict(from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=[author.email]) if dry_run: job.preview_success_mail(*args, **kwargs) else: send_mail(*args, **kwargs) amo.log(amo.LOG.BULK_VALIDATION_UPDATED, version.addon, version, details={'version': version.version, 'target': job.target_version.version})