def _gen_buckets(self, rules):
bucket_agg = collections.defaultdict(list)
fallback_rules = []
# Variables that the buckets will share between them
self.buckets_vars = (collections.defaultdict(list),
collections.defaultdict(list))
self.bucket_groups = {
} # type: typing.Dict[token.Token, JBlockBucketGroup]
for r in rules:
if isinstance(r, parser.JBlockRule):
rule = r
else:
rule = parser.JBlockRule(r)
if rule.is_html_rule:
continue
if not rule.matching_supported(self.supported_options):
continue
t = self._pick_token(rule)
if t is None:
fallback_rules.append(rule)
else:
bucket_agg[t].append(rule)
for k, v in bucket_agg.items():
self.bucket_groups[k] = self._rule_list_to_bucket_group(k, v)
self.fallback_bucket_group = self._rule_list_to_bucket_group(
"FALLBACK", fallback_rules)
def test_rule_with_options(rule_text, results):
rule = parser.JBlockRule(rule_text)
rules = bucket.JBlockBuckets([rule_text])
for url, params, match in results:
assert rule.match_url(url, params) == match
assert rules.should_block(url, params) == match
def test_documented(rule_text, results):
rule = parser.JBlockRule(rule_text)
for url in results["blocks"]:
assert rule.match_url(url)
for url in results["doesn't block"]:
assert not rule.match_url(url)
def test_documented_examples(rule_text, results):
rule = parser.JBlockRule(rule_text)
rules = bucket.JBlockBuckets([rule_text])
for url in results["blocks"]:
assert rule.match_url(url)
assert rules.should_block(url)
for url in results["doesn't block"]:
assert not rule.match_url(url)
assert not rules.should_block(url)
def test_bulk_regexp_match():
tokens = []
with open("tests/data/easylist.txt") as f:
for line in f:
p = parser.JBlockRule(line)
if not p.matching_supported():
continue
r = p.regex
t = token.TokenConverter.regex_to_tokens(r)
if not t:
tokens.append(line)
print(tokens)
def test_options_extraction(text, result):
rule = parser.JBlockRule(text)
assert rule.options == result
def test_empty_rules():
rules = bucket.JBlockBuckets(["adv", "", " \t", parser.JBlockRule("adv2")])
assert len(rules) == 2
def test_rules_instantiation():
rule = parser.JBlockRule("adv")
rules = bucket.JBlockBuckets([rule])
assert rule.match_url("http://example.com/adv")
assert rules.should_block("http://example.com/adv")
def test_pattern_token_basic(pattern, tokens): t = parser.JBlockRule(pattern).to_tokens() assert t == tokens
def test_pattern_token_basic(pattern, tokens): re = parser.JBlockRule(pattern).regex t = token.TokenConverter.regex_to_tokens(re) assert t == tokens
