def validate_file_path(path, channel): """Run the validator against a file at the given path, and return the results. Should only be called directly by Validator or `validate_file` task. Search plugins don't call the linter but get linted by `annotate_search_plugin_validation`. All legacy extensions (including dictionaries, themes etc) are disabled via `annotate_legacy_addon_restrictions` except if they're signed by Mozilla. """ if path.endswith('.xml'): # search plugins are validated directly by addons-server # so that we don't have to call the linter or validator results = deepcopy(amo.VALIDATOR_SKELETON_RESULTS) annotations.annotate_search_plugin_validation( results=results, file_path=path, channel=channel) return json.dumps(results) # Annotate results with potential legacy add-ons restrictions. data = parse_addon(path, minimal=True) is_webextension = data.get('is_webextension') is True is_mozilla_signed = data.get('is_mozilla_signed_extension', False) if not is_webextension: results = deepcopy(amo.VALIDATOR_SKELETON_RESULTS) annotations.annotate_legacy_addon_restrictions( path=path, results=results, parsed_data=data, error=not is_mozilla_signed) return json.dumps(results) return run_addons_linter(path, channel=channel)
def test_opensearch_validation(fixture, success, message): """Tests that the OpenSearch validation doesn't find anything worrying.""" fixture_path = os.path.join( settings.ROOT, 'src/olympia/files/fixtures/files/opensearch/', fixture) results = { 'messages': [], 'errors': 0, 'metadata': {} } annotations.annotate_search_plugin_validation( results, fixture_path, channel=amo.RELEASE_CHANNEL_LISTED) if success: assert not results['errors'] assert not results['messages'] else: assert results['errors'] assert results['messages'] expected = 'OpenSearch: {}'.format(message) assert any( message['message'] == expected for message in results['messages'])
def handle_file_validation_result(results, file_id, *args): """Annotate a set of validation results and save them to the given File instance.""" file_ = File.objects.get(pk=file_id) annotations.annotate_webext_incompatibilities( results=results, file_=file_, addon=file_.version.addon, version_string=file_.version.version, channel=file_.version.channel) annotations.annotate_search_plugin_validation( results=results, file_path=file_.current_file_path, channel=file_.version.channel) return FileValidation.from_json(file_, results).pk
def validate_file_path(path, channel): """Run the validator against a file at the given path, and return the results, which should be a json string. Should only be called directly by `validate_upload` or `validate_file` tasks. Search plugins don't call the linter but get linted by `annotate_search_plugin_validation`. All legacy extensions (including dictionaries, themes etc) are disabled via `annotate_legacy_addon_restrictions` except if they're signed by Mozilla. """ if path.endswith('.xml'): # search plugins are validated directly by addons-server # so that we don't have to call the linter or validator results = deepcopy(amo.VALIDATOR_SKELETON_RESULTS) annotations.annotate_search_plugin_validation(results=results, file_path=path, channel=channel) return json.dumps(results) # Annotate results with potential legacy add-ons restrictions. try: data = parse_addon(path, minimal=True) except NoManifestFound: # If no manifest is found, return empty data; the check below # explicitly looks for is_webextension is False, so it will not be # considered a legacy extension, and the linter will pick it up and # will know what message to return to the developer. data = {} except InvalidManifest: # Similarly, if we can't parse the manifest, let the linter pick that # up. data = {} is_legacy_extension = data.get('is_webextension', None) is False is_mozilla_signed = data.get('is_mozilla_signed_extension', None) is True if is_legacy_extension: results = deepcopy(amo.VALIDATOR_SKELETON_RESULTS) annotations.annotate_legacy_addon_restrictions( path=path, results=results, parsed_data=data, error=not is_mozilla_signed) return json.dumps(results) return run_addons_linter(path, channel=channel)
def test_opensearch_validation_rel_self_url(): """Tests that rel=self urls are ignored for unlisted addons.""" fixture_path = os.path.join(settings.ROOT, 'src/olympia/files/fixtures/files', 'opensearch/rel_self_url.xml') results = {'messages': [], 'errors': 0, 'metadata': {}} annotations.annotate_search_plugin_validation( results, fixture_path, channel=amo.RELEASE_CHANNEL_UNLISTED) assert not results['errors'] annotations.annotate_search_plugin_validation( results, fixture_path, channel=amo.RELEASE_CHANNEL_LISTED) assert results['errors']
def validate_file_path(path, channel): """Run the validator against a file at the given path, and return the results, which should be a json string. Should only be called directly by `validate_upload` or `validate_file` tasks. Search plugins don't call the linter but get linted by `annotate_search_plugin_validation`. All legacy extensions (including dictionaries, themes etc) are disabled via `annotate_legacy_addon_restrictions` except if they're signed by Mozilla. """ if path.endswith('.xml'): # search plugins are validated directly by addons-server # so that we don't have to call the linter or validator results = deepcopy(amo.VALIDATOR_SKELETON_RESULTS) annotations.annotate_search_plugin_validation( results=results, file_path=path, channel=channel) return json.dumps(results) # Annotate results with potential legacy add-ons restrictions. try: data = parse_addon(path, minimal=True) except NoManifestFound: # If no manifest is found, return empty data; the check below # explicitly looks for is_webextension is False, so it will not be # considered a legacy extension, and the linter will pick it up and # will know what message to return to the developer. data = {} except InvalidManifest: # Similarly, if we can't parse the manifest, let the linter pick that # up. data = {} is_legacy_extension = data.get('is_webextension', None) is False is_mozilla_signed = data.get('is_mozilla_signed_extension', None) is True if is_legacy_extension: results = deepcopy(amo.VALIDATOR_SKELETON_RESULTS) annotations.annotate_legacy_addon_restrictions( path=path, results=results, parsed_data=data, error=not is_mozilla_signed) return json.dumps(results) return run_addons_linter(path, channel=channel)
def handle_upload_validation_result( results, upload_pk, channel, is_mozilla_signed): """Annotate a set of validation results and save them to the given FileUpload instance.""" upload = FileUpload.objects.get(pk=upload_pk) # Restrictions applying to new legacy submissions apply if: # - It's the very first upload (there is no addon id yet) # - It's the first upload in that channel is_new_upload = ( not upload.addon_id or not upload.addon.find_latest_version(channel=channel, exclude=())) # Annotate results with potential legacy add-ons restrictions. if not is_mozilla_signed: results = annotations.annotate_legacy_addon_restrictions( results=results, is_new_upload=is_new_upload) annotations.annotate_legacy_langpack_restriction(results=results) annotations.annotate_search_plugin_validation( results=results, file_path=upload.path, channel=channel) # Check for API keys in submissions. # Make sure it is extension-like, e.g. no LWT or search plugin try: results = check_for_api_keys_in_file(results=results, upload=upload) except (ValidationError, BadZipfile, IOError): pass # Annotate results with potential webext warnings on new versions. if upload.addon_id and upload.version: results = annotations.annotate_webext_incompatibilities( results=results, file_=None, addon=upload.addon, version_string=upload.version, channel=channel) upload.validation = json.dumps(results) upload.save() # We want to hit the custom save(). # Track the time it took from first upload through validation # until the results were processed and saved. upload_start = utc_millesecs_from_epoch(upload.created) now = datetime.datetime.now() now_ts = utc_millesecs_from_epoch(now) delta = now_ts - upload_start statsd.timing('devhub.validation_results_processed', delta) if not storage.exists(upload.path): # TODO: actually fix this so we can get stats. It seems that # the file maybe gets moved but it needs more investigation. log.warning('Scaled upload stats were not tracked. File is ' 'missing: {}'.format(upload.path)) return size = Decimal(storage.size(upload.path)) megabyte = Decimal(1024 * 1024) # Stash separate metrics for small / large files. quantifier = 'over' if size > megabyte else 'under' statsd.timing( 'devhub.validation_results_processed_{}_1mb'.format(quantifier), delta) # Scale the upload / processing time by package size (in MB) # so we can normalize large XPIs which naturally take longer to validate. scaled_delta = None size_in_mb = size / megabyte if size > 0: # If the package is smaller than 1MB, don't scale it. This should # help account for validator setup time. unit = size_in_mb if size > megabyte else Decimal(1) scaled_delta = Decimal(delta) / unit statsd.timing('devhub.validation_results_processed_per_mb', scaled_delta) log.info('Time to process and save upload validation; ' 'upload.pk={upload}; processing_time={delta}; ' 'scaled_per_mb={scaled}; upload_size_in_mb={size_in_mb}; ' 'created={created}; now={now}' .format(delta=delta, upload=upload.pk, created=upload.created, now=now, scaled=scaled_delta, size_in_mb=size_in_mb))