def wrapper(task, akismet_results, id_or_path, **kwargs):
# This is necessary to prevent timeout exceptions from being set
# as our result, and replacing the partial validation results we'd
# prefer to return.
task.ignore_result = True
try:
data = fn(id_or_path, **kwargs)
results = json.loads(force_text(data))
if akismet_results:
annotations.annotate_akismet_spam_check(
results, akismet_results)
return results
except UnsupportedFileType as exc:
results = deepcopy(amo.VALIDATOR_SKELETON_RESULTS)
annotations.insert_validation_message(
results,
type_='error',
message=exc.message,
msg_id='unsupported_filetype',
compatibility_type=None)
return results
except Exception as exc:
log.exception('Unhandled error during validation: %r' % exc)
return deepcopy(amo.VALIDATOR_SKELETON_EXCEPTION_WEBEXT)
finally:
# But we do want to return a result after that exception has
# been handled.
task.ignore_result = False
def wrapper(task, akismet_results, id_, hash_, *args, **kw):
# This is necessary to prevent timeout exceptions from being set
# as our result, and replacing the partial validation results we'd
# prefer to return.
task.ignore_result = True
try:
data = fn(id_, hash_, *args, **kw)
result = json.loads(data)
if akismet_results:
annotations.annotate_akismet_spam_check(
result, akismet_results)
return result
except Exception as e:
log.exception('Unhandled error during validation: %r' % e)
is_webextension = kw.get('is_webextension', False)
if is_webextension:
return deepcopy(amo.VALIDATOR_SKELETON_EXCEPTION_WEBEXT)
return deepcopy(amo.VALIDATOR_SKELETON_EXCEPTION)
finally:
# But we do want to return a result after that exception has
# been handled.
task.ignore_result = False
def handle_upload_validation_result(
results, upload_pk, channel, is_mozilla_signed):
"""Annotate a set of validation results and save them to the given
FileUpload instance."""
upload = FileUpload.objects.get(pk=upload_pk)
# Check for API keys in submissions.
# Make sure it is extension-like, e.g. no search plugin
try:
results = check_for_api_keys_in_file(results=results, upload=upload)
except (ValidationError, BadZipfile, IOError):
pass
# Annotate results with akismet reports results if there are any.
reports = AkismetReport.objects.filter(upload_instance=upload)
akismet_results = [
(report.comment_type, report.result) for report in reports]
annotations.annotate_akismet_spam_check(results, akismet_results)
# Annotate results with potential webext warnings on new versions.
if upload.addon_id and upload.version:
annotations.annotate_webext_incompatibilities(
results=results, file_=None, addon=upload.addon,
version_string=upload.version, channel=channel)
upload.validation = json.dumps(results)
upload.save() # We want to hit the custom save().
# Track the time it took from first upload through validation
# until the results were processed and saved.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
delta = now_ts - upload_start
statsd.timing('devhub.validation_results_processed', delta)
if not storage.exists(upload.path):
# TODO: actually fix this so we can get stats. It seems that
# the file maybe gets moved but it needs more investigation.
log.warning('Scaled upload stats were not tracked. File is '
'missing: {}'.format(upload.path))
return
size = Decimal(storage.size(upload.path))
megabyte = Decimal(1024 * 1024)
# Stash separate metrics for small / large files.
quantifier = 'over' if size > megabyte else 'under'
statsd.timing(
'devhub.validation_results_processed_{}_1mb'.format(quantifier), delta)
# Scale the upload / processing time by package size (in MB)
# so we can normalize large XPIs which naturally take longer to validate.
scaled_delta = None
size_in_mb = size / megabyte
if size > 0:
# If the package is smaller than 1MB, don't scale it. This should
# help account for validator setup time.
unit = size_in_mb if size > megabyte else Decimal(1)
scaled_delta = Decimal(delta) / unit
statsd.timing('devhub.validation_results_processed_per_mb',
scaled_delta)
log.info('Time to process and save upload validation; '
'upload.pk={upload}; processing_time={delta}; '
'scaled_per_mb={scaled}; upload_size_in_mb={size_in_mb}; '
'created={created}; now={now}'
.format(delta=delta, upload=upload.pk,
created=upload.created, now=now,
scaled=scaled_delta, size_in_mb=size_in_mb))
def handle_upload_validation_result(
results, upload_pk, channel, is_mozilla_signed):
"""Annotate a set of validation results and save them to the given
FileUpload instance."""
upload = FileUpload.objects.get(pk=upload_pk)
# Check for API keys in submissions.
# Make sure it is extension-like, e.g. no LWT or search plugin
try:
results = check_for_api_keys_in_file(results=results, upload=upload)
except (ValidationError, BadZipfile, IOError):
pass
# Annotate results with akismet reports results if there are any.
reports = AkismetReport.objects.filter(upload_instance=upload)
akismet_results = [
(report.comment_type, report.result) for report in reports]
annotations.annotate_akismet_spam_check(results, akismet_results)
# Annotate results with potential webext warnings on new versions.
if upload.addon_id and upload.version:
annotations.annotate_webext_incompatibilities(
results=results, file_=None, addon=upload.addon,
version_string=upload.version, channel=channel)
upload.validation = json.dumps(results)
upload.save() # We want to hit the custom save().
# Track the time it took from first upload through validation
# until the results were processed and saved.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
delta = now_ts - upload_start
statsd.timing('devhub.validation_results_processed', delta)
if not storage.exists(upload.path):
# TODO: actually fix this so we can get stats. It seems that
# the file maybe gets moved but it needs more investigation.
log.warning('Scaled upload stats were not tracked. File is '
'missing: {}'.format(upload.path))
return
size = Decimal(storage.size(upload.path))
megabyte = Decimal(1024 * 1024)
# Stash separate metrics for small / large files.
quantifier = 'over' if size > megabyte else 'under'
statsd.timing(
'devhub.validation_results_processed_{}_1mb'.format(quantifier), delta)
# Scale the upload / processing time by package size (in MB)
# so we can normalize large XPIs which naturally take longer to validate.
scaled_delta = None
size_in_mb = size / megabyte
if size > 0:
# If the package is smaller than 1MB, don't scale it. This should
# help account for validator setup time.
unit = size_in_mb if size > megabyte else Decimal(1)
scaled_delta = Decimal(delta) / unit
statsd.timing('devhub.validation_results_processed_per_mb',
scaled_delta)
log.info('Time to process and save upload validation; '
'upload.pk={upload}; processing_time={delta}; '
'scaled_per_mb={scaled}; upload_size_in_mb={size_in_mb}; '
'created={created}; now={now}'
.format(delta=delta, upload=upload.pk,
created=upload.created, now=now,
scaled=scaled_delta, size_in_mb=size_in_mb))
def handle_upload_validation_result(
results, upload_pk, channel, is_mozilla_signed):
"""Annotate a set of validation results and save them to the given
FileUpload instance."""
upload = FileUpload.objects.get(pk=upload_pk)
if waffle.switch_is_active('enable-yara') and results['errors'] == 0:
# Run Yara. This cannot be asynchronous because we have no way to know
# whether the task will complete before we attach a `Version` to it
# later in the submission process... Because we cannot use `chord`
# reliably right now (requires Celery 4.2+), this task is actually not
# run as a task, it's a simple function call.
#
# TODO: use `run_yara` as a task in the submission chord once it is
# possible. See: https://github.com/mozilla/addons-server/issues/12216
run_yara(upload.pk)
if waffle.switch_is_active('enable-customs') and results['errors'] == 0:
# Run customs. This cannot be asynchronous because we have no way to
# know whether the task will complete before we attach a `Version` to
# it later in the submission process... Because we cannot use `chord`
# reliably right now (requires Celery 4.2+), this task is actually not
# run as a task, it's a simple function call.
#
# TODO: use `run_customs` as a task in the submission chord once it is
# possible. See: https://github.com/mozilla/addons-server/issues/12217
run_customs(upload.pk)
if waffle.switch_is_active('enable-wat') and results['errors'] == 0:
# Run wat. This cannot be asynchronous because we have no way to know
# whether the task will complete before we attach a `Version` to it
# later in the submission process... Because we cannot use `chord`
# reliably right now (requires Celery 4.2+), this task is actually not
# run as a task, it's a simple function call.
#
# TODO: use `run_wat` as a task in the submission chord once it is
# possible. See: https://github.com/mozilla/addons-server/issues/12224
run_wat(upload.pk)
# Check for API keys in submissions.
# Make sure it is extension-like, e.g. no search plugin
try:
results = check_for_api_keys_in_file(results=results, upload=upload)
except (ValidationError, BadZipfile, IOError):
pass
# Annotate results with akismet reports results if there are any.
reports = AkismetReport.objects.filter(upload_instance=upload)
akismet_results = [
(report.comment_type, report.result) for report in reports]
annotations.annotate_akismet_spam_check(results, akismet_results)
# Annotate results with potential webext warnings on new versions.
if upload.addon_id and upload.version:
annotations.annotate_webext_incompatibilities(
results=results, file_=None, addon=upload.addon,
version_string=upload.version, channel=channel)
upload.validation = json.dumps(results)
upload.save() # We want to hit the custom save().
# Track the time it took from first upload through validation
# until the results were processed and saved.
upload_start = utc_millesecs_from_epoch(upload.created)
now = datetime.datetime.now()
now_ts = utc_millesecs_from_epoch(now)
delta = now_ts - upload_start
statsd.timing('devhub.validation_results_processed', delta)
if not storage.exists(upload.path):
# TODO: actually fix this so we can get stats. It seems that
# the file maybe gets moved but it needs more investigation.
log.warning('Scaled upload stats were not tracked. File is '
'missing: {}'.format(upload.path))
return
size = Decimal(storage.size(upload.path))
megabyte = Decimal(1024 * 1024)
# Stash separate metrics for small / large files.
quantifier = 'over' if size > megabyte else 'under'
statsd.timing(
'devhub.validation_results_processed_{}_1mb'.format(quantifier), delta)
# Scale the upload / processing time by package size (in MB)
# so we can normalize large XPIs which naturally take longer to validate.
scaled_delta = None
size_in_mb = size / megabyte
if size > 0:
# If the package is smaller than 1MB, don't scale it. This should
# help account for validator setup time.
unit = size_in_mb if size > megabyte else Decimal(1)
scaled_delta = Decimal(delta) / unit
statsd.timing('devhub.validation_results_processed_per_mb',
scaled_delta)
log.info('Time to process and save upload validation; '
'upload.pk={upload}; processing_time={delta}; '
'scaled_per_mb={scaled}; upload_size_in_mb={size_in_mb}; '
'created={created}; now={now}'
.format(delta=delta, upload=upload.pk,
created=upload.created, now=now,
scaled=scaled_delta, size_in_mb=size_in_mb))