def run_scanner(results, upload_pk, scanner, api_url, api_key): """ Run a scanner on a FileUpload via RPC and store the results. - `results` are the validation results passed in the validation chain. This task is a validation task, which is why it must receive the validation results as first argument. - `upload_pk` is the FileUpload ID. """ scanner_name = SCANNERS.get(scanner) log.info('Starting scanner "%s" task for FileUpload %s.', scanner_name, upload_pk) if not results['metadata']['is_webextension']: log.info( 'Not running scanner "%s" for FileUpload %s, it is not a ' 'webextension.', scanner_name, upload_pk) return results upload = FileUpload.objects.get(pk=upload_pk) try: if not os.path.exists(upload.path): raise ValueError('File "{}" does not exist.'.format(upload.path)) scanner_result = ScannerResult(upload=upload, scanner=scanner) with statsd.timer('devhub.{}'.format(scanner_name)): json_payload = { 'api_key': api_key, 'download_url': upload.get_authenticated_download_url(), } response = requests.post(url=api_url, json=json_payload, timeout=settings.SCANNER_TIMEOUT) try: data = response.json() except ValueError: # Log the response body when JSON decoding has failed. raise ValueError(response.text) if response.status_code != 200 or 'error' in data: raise ValueError(data) scanner_result.results = data scanner_result.save() statsd.incr('devhub.{}.success'.format(scanner_name)) log.info('Ending scanner "%s" task for FileUpload %s.', scanner_name, upload_pk) except Exception: statsd.incr('devhub.{}.failure'.format(scanner_name)) # We log the exception but we do not raise to avoid perturbing the # submission flow. log.exception('Error in scanner "%s" task for FileUpload %s.', scanner_name, upload_pk) return results
def run_scanner(results, upload_pk, scanner, api_url, api_key): """ Run a scanner on a FileUpload via RPC and store the results. - `results` are the validation results passed in the validation chain. This task is a validation task, which is why it must receive the validation results as first argument. - `upload_pk` is the FileUpload ID. """ scanner_name = SCANNERS.get(scanner) log.info('Starting scanner "%s" task for FileUpload %s.', scanner_name, upload_pk) if not results['metadata']['is_webextension']: log.info( 'Not running scanner "%s" for FileUpload %s, it is not a webextension.', scanner_name, upload_pk, ) return results upload = FileUpload.objects.get(pk=upload_pk) try: if not os.path.exists(upload.path): raise ValueError('File "{}" does not exist.'.format(upload.path)) scanner_result = ScannerResult(upload=upload, scanner=scanner) with statsd.timer('devhub.{}'.format(scanner_name)): _run_scanner_for_url( scanner_result, upload.get_authenticated_download_url(), scanner, api_url, api_key, ) scanner_result.save() if scanner_result.has_matches: statsd.incr('devhub.{}.has_matches'.format(scanner_name)) for scanner_rule in scanner_result.matched_rules.all(): statsd.incr('devhub.{}.rule.{}.match'.format( scanner_name, scanner_rule.id)) statsd.incr('devhub.{}.success'.format(scanner_name)) log.info('Ending scanner "%s" task for FileUpload %s.', scanner_name, upload_pk) except Exception as exc: statsd.incr('devhub.{}.failure'.format(scanner_name)) log.exception('Error in scanner "%s" task for FileUpload %s.', scanner_name, upload_pk) if not waffle.switch_is_active('ignore-exceptions-in-scanner-tasks'): raise exc return results
def run_scanner(upload_pk, scanner, api_url, api_key): """ Run a scanner on a FileUpload via RPC and store the results. """ scanner_name = SCANNERS.get(scanner) log.info('Starting scanner "%s" task for FileUpload %s.', scanner_name, upload_pk) upload = FileUpload.objects.get(pk=upload_pk) if not upload.path.endswith('.xpi'): log.info('Not running scanner "%s" for FileUpload %s, it is not a xpi ' 'file.', scanner_name, upload_pk) return try: if not os.path.exists(upload.path): raise ValueError('File "{}" does not exist.' .format(upload.path)) result = ScannersResult() result.upload = upload result.scanner = scanner with statsd.timer('devhub.{}'.format(scanner_name)): json_payload = { 'api_key': api_key, 'download_url': upload.get_authenticated_download_url(), } response = requests.post(url=api_url, json=json_payload, timeout=settings.SCANNER_TIMEOUT) try: results = response.json() except ValueError: # Log the response body when JSON decoding has failed. raise ValueError(response.text) if 'error' in results: raise ValueError(results) result.results = results result.save() statsd.incr('devhub.{}.success'.format(scanner_name)) log.info('Ending scanner "%s" task for FileUpload %s.', scanner_name, upload_pk) except Exception: statsd.incr('devhub.{}.failure'.format(scanner_name)) # We log the exception but we do not raise to avoid perturbing the # submission flow. log.exception('Error in scanner "%s" task for FileUpload %s.', scanner_name, upload_pk)
def get_queryset(self): label = self.request.query_params.get('label', None) scanner = next( (key for key in SCANNERS if SCANNERS.get(key) == self.request.query_params.get('scanner')), None, ) bad_results = ScannerResult.objects.exclude(version=None) good_results = ScannerResult.objects.exclude(version=None) if scanner: bad_results = bad_results.filter(scanner=scanner) good_results = good_results.filter(scanner=scanner) bad_filters = Q(state=TRUE_POSITIVE) | Q( version__versionlog__activity_log__action__in=( amo.LOG.BLOCKLIST_BLOCK_ADDED.id, amo.LOG.BLOCKLIST_BLOCK_EDITED.id, )) good_results = ( good_results.filter( Q(version__versionlog__activity_log__action__in=( amo.LOG.CONFIRM_AUTO_APPROVED.id, amo.LOG.APPROVE_VERSION.id, )) & ~Q(version__versionlog__activity_log__user_id=settings. TASK_USER_ID # noqa )).exclude(bad_filters).distinct().annotate(label=Value( LABEL_GOOD, output_field=CharField())).all()) bad_results = (bad_results.filter(bad_filters).distinct().annotate( label=Value(LABEL_BAD, output_field=CharField())).all()) queryset = ScannerResult.objects.none() if not label: queryset = good_results.union(bad_results) elif label == LABEL_GOOD: queryset = good_results elif label == LABEL_BAD: queryset = bad_results return queryset.order_by('-pk')
def get_scanner_name(self): return SCANNERS.get(self.scanner)