def _gen_buckets(self, rules): bucket_agg = collections.defaultdict(list) fallback_rules = [] # Variables that the buckets will share between them self.buckets_vars = (collections.defaultdict(list), collections.defaultdict(list)) self.bucket_groups = { } # type: typing.Dict[token.Token, JBlockBucketGroup] for r in rules: if isinstance(r, parser.JBlockRule): rule = r else: rule = parser.JBlockRule(r) if rule.is_html_rule: continue if not rule.matching_supported(self.supported_options): continue t = self._pick_token(rule) if t is None: fallback_rules.append(rule) else: bucket_agg[t].append(rule) for k, v in bucket_agg.items(): self.bucket_groups[k] = self._rule_list_to_bucket_group(k, v) self.fallback_bucket_group = self._rule_list_to_bucket_group( "FALLBACK", fallback_rules)
def test_rule_with_options(rule_text, results): rule = parser.JBlockRule(rule_text) rules = bucket.JBlockBuckets([rule_text]) for url, params, match in results: assert rule.match_url(url, params) == match assert rules.should_block(url, params) == match
def test_documented(rule_text, results): rule = parser.JBlockRule(rule_text) for url in results["blocks"]: assert rule.match_url(url) for url in results["doesn't block"]: assert not rule.match_url(url)
def test_documented_examples(rule_text, results): rule = parser.JBlockRule(rule_text) rules = bucket.JBlockBuckets([rule_text]) for url in results["blocks"]: assert rule.match_url(url) assert rules.should_block(url) for url in results["doesn't block"]: assert not rule.match_url(url) assert not rules.should_block(url)
def test_bulk_regexp_match(): tokens = [] with open("tests/data/easylist.txt") as f: for line in f: p = parser.JBlockRule(line) if not p.matching_supported(): continue r = p.regex t = token.TokenConverter.regex_to_tokens(r) if not t: tokens.append(line) print(tokens)
def test_options_extraction(text, result): rule = parser.JBlockRule(text) assert rule.options == result
def test_empty_rules(): rules = bucket.JBlockBuckets(["adv", "", " \t", parser.JBlockRule("adv2")]) assert len(rules) == 2
def test_rules_instantiation(): rule = parser.JBlockRule("adv") rules = bucket.JBlockBuckets([rule]) assert rule.match_url("http://example.com/adv") assert rules.should_block("http://example.com/adv")
def test_pattern_token_basic(pattern, tokens): t = parser.JBlockRule(pattern).to_tokens() assert t == tokens
def test_pattern_token_basic(pattern, tokens): re = parser.JBlockRule(pattern).regex t = token.TokenConverter.regex_to_tokens(re) assert t == tokens