def run_scanner(results, upload_pk, scanner, api_url, api_key):
"""
Run a scanner on a FileUpload via RPC and store the results.
- `results` are the validation results passed in the validation chain. This
task is a validation task, which is why it must receive the validation
results as first argument.
- `upload_pk` is the FileUpload ID.
"""
scanner_name = SCANNERS.get(scanner)
log.info('Starting scanner "%s" task for FileUpload %s.', scanner_name,
upload_pk)
if not results['metadata']['is_webextension']:
log.info(
'Not running scanner "%s" for FileUpload %s, it is not a '
'webextension.', scanner_name, upload_pk)
return results
upload = FileUpload.objects.get(pk=upload_pk)
try:
if not os.path.exists(upload.path):
raise ValueError('File "{}" does not exist.'.format(upload.path))
scanner_result = ScannerResult(upload=upload, scanner=scanner)
with statsd.timer('devhub.{}'.format(scanner_name)):
json_payload = {
'api_key': api_key,
'download_url': upload.get_authenticated_download_url(),
}
response = requests.post(url=api_url,
json=json_payload,
timeout=settings.SCANNER_TIMEOUT)
try:
data = response.json()
except ValueError:
# Log the response body when JSON decoding has failed.
raise ValueError(response.text)
if response.status_code != 200 or 'error' in data:
raise ValueError(data)
scanner_result.results = data
scanner_result.save()
statsd.incr('devhub.{}.success'.format(scanner_name))
log.info('Ending scanner "%s" task for FileUpload %s.', scanner_name,
upload_pk)
except Exception:
statsd.incr('devhub.{}.failure'.format(scanner_name))
# We log the exception but we do not raise to avoid perturbing the
# submission flow.
log.exception('Error in scanner "%s" task for FileUpload %s.',
scanner_name, upload_pk)
return results
def run_scanner(results, upload_pk, scanner, api_url, api_key):
"""
Run a scanner on a FileUpload via RPC and store the results.
- `results` are the validation results passed in the validation chain. This
task is a validation task, which is why it must receive the validation
results as first argument.
- `upload_pk` is the FileUpload ID.
"""
scanner_name = SCANNERS.get(scanner)
log.info('Starting scanner "%s" task for FileUpload %s.', scanner_name,
upload_pk)
if not results['metadata']['is_webextension']:
log.info(
'Not running scanner "%s" for FileUpload %s, it is not a webextension.',
scanner_name,
upload_pk,
)
return results
upload = FileUpload.objects.get(pk=upload_pk)
try:
if not os.path.exists(upload.path):
raise ValueError('File "{}" does not exist.'.format(upload.path))
scanner_result = ScannerResult(upload=upload, scanner=scanner)
with statsd.timer('devhub.{}'.format(scanner_name)):
_run_scanner_for_url(
scanner_result,
upload.get_authenticated_download_url(),
scanner,
api_url,
api_key,
)
scanner_result.save()
if scanner_result.has_matches:
statsd.incr('devhub.{}.has_matches'.format(scanner_name))
for scanner_rule in scanner_result.matched_rules.all():
statsd.incr('devhub.{}.rule.{}.match'.format(
scanner_name, scanner_rule.id))
statsd.incr('devhub.{}.success'.format(scanner_name))
log.info('Ending scanner "%s" task for FileUpload %s.', scanner_name,
upload_pk)
except Exception as exc:
statsd.incr('devhub.{}.failure'.format(scanner_name))
log.exception('Error in scanner "%s" task for FileUpload %s.',
scanner_name, upload_pk)
if not waffle.switch_is_active('ignore-exceptions-in-scanner-tasks'):
raise exc
return results
def run_scanner(upload_pk, scanner, api_url, api_key):
"""
Run a scanner on a FileUpload via RPC and store the results.
"""
scanner_name = SCANNERS.get(scanner)
log.info('Starting scanner "%s" task for FileUpload %s.', scanner_name,
upload_pk)
upload = FileUpload.objects.get(pk=upload_pk)
if not upload.path.endswith('.xpi'):
log.info('Not running scanner "%s" for FileUpload %s, it is not a xpi '
'file.', scanner_name, upload_pk)
return
try:
if not os.path.exists(upload.path):
raise ValueError('File "{}" does not exist.' .format(upload.path))
result = ScannersResult()
result.upload = upload
result.scanner = scanner
with statsd.timer('devhub.{}'.format(scanner_name)):
json_payload = {
'api_key': api_key,
'download_url': upload.get_authenticated_download_url(),
}
response = requests.post(url=api_url,
json=json_payload,
timeout=settings.SCANNER_TIMEOUT)
try:
results = response.json()
except ValueError:
# Log the response body when JSON decoding has failed.
raise ValueError(response.text)
if 'error' in results:
raise ValueError(results)
result.results = results
result.save()
statsd.incr('devhub.{}.success'.format(scanner_name))
log.info('Ending scanner "%s" task for FileUpload %s.', scanner_name,
upload_pk)
except Exception:
statsd.incr('devhub.{}.failure'.format(scanner_name))
# We log the exception but we do not raise to avoid perturbing the
# submission flow.
log.exception('Error in scanner "%s" task for FileUpload %s.',
scanner_name, upload_pk)
def get_queryset(self):
label = self.request.query_params.get('label', None)
scanner = next(
(key for key in SCANNERS
if SCANNERS.get(key) == self.request.query_params.get('scanner')),
None,
)
bad_results = ScannerResult.objects.exclude(version=None)
good_results = ScannerResult.objects.exclude(version=None)
if scanner:
bad_results = bad_results.filter(scanner=scanner)
good_results = good_results.filter(scanner=scanner)
bad_filters = Q(state=TRUE_POSITIVE) | Q(
version__versionlog__activity_log__action__in=(
amo.LOG.BLOCKLIST_BLOCK_ADDED.id,
amo.LOG.BLOCKLIST_BLOCK_EDITED.id,
))
good_results = (
good_results.filter(
Q(version__versionlog__activity_log__action__in=(
amo.LOG.CONFIRM_AUTO_APPROVED.id,
amo.LOG.APPROVE_VERSION.id,
))
& ~Q(version__versionlog__activity_log__user_id=settings.
TASK_USER_ID # noqa
)).exclude(bad_filters).distinct().annotate(label=Value(
LABEL_GOOD, output_field=CharField())).all())
bad_results = (bad_results.filter(bad_filters).distinct().annotate(
label=Value(LABEL_BAD, output_field=CharField())).all())
queryset = ScannerResult.objects.none()
if not label:
queryset = good_results.union(bad_results)
elif label == LABEL_GOOD:
queryset = good_results
elif label == LABEL_BAD:
queryset = bad_results
return queryset.order_by('-pk')
def get_scanner_name(self):
return SCANNERS.get(self.scanner)
