def link_to_features(link):
text = normalize(get_link_text(link))
href = get_link_href(link)
p = urlsplit(href)
query_parsed = parse_qsl(p.query)
query_param_names = [k.lower() for k, v in query_parsed]
query_param_names_ngrams = _as_list(
ngrams_wb(" ".join([normalize(name) for name in query_param_names]), 3,
5, True))
elem = get_selector_root(link)
elem_target = _elem_attr(elem, 'target')
elem_rel = _elem_attr(elem, 'rel')
# Classes of link itself and all its children.
# It is common to have e.g. span elements with fontawesome
# arrow icon classes inside <a> links.
self_and_children_classes = ' '.join(link.xpath(".//@class").extract())
parent_classes = ' '.join(link.xpath('../@class').extract())
css_classes = normalize(parent_classes + ' ' + self_and_children_classes)
return {
'bias':
3.0,
'isdigit':
text.isdigit(),
'isalpha':
text.isalpha(),
'elem-target':
elem_target,
'elem-rel':
elem_rel,
'num-tokens%s' % _num_tokens_feature(text):
1.0,
'text':
_as_list(ngrams_wb(replace_digits(text), 2, 5),
AUTOPAGER_LIMITS.max_text_features),
'text-exact':
replace_digits(text.strip()[:20].strip()),
'class':
_as_list(ngrams_wb(css_classes, 4, 5),
AUTOPAGER_LIMITS.max_css_features),
'query':
query_param_names_ngrams,
'path-has-page':
'page' in p.path.lower(),
'path-has-pageXX':
re.search(r'[/-](?:p|page\w?)/?\d+', p.path.lower()) is not None,
'path-has-number':
any(part.isdigit() for part in p.path.split('/')),
'href-has-year':
re.search('20\d\d', href) is not None,
}
def page_to_features(xseq):
features = [link_to_features(a) for a in xseq]
around = get_text_around_selector_list(xseq, max_length=15)
# weight is less than 1 because there is a lot of duplicate information
# in these ngrams and so we want to regularize them stronger
# (as if they are a single feature, not many features)
k = 0.2
for feat, (before, after) in zip(features, around):
feat['text-before'] = {
n: k
for n in ngrams_wb(normalize(before), 5, 5)
}
feat['text-after'] = {n: k for n in ngrams_wb(normalize(after), 5, 5)}
return features
def link_to_features(link):
text = normalize(get_link_text(link))
href = get_link_href(link)
p = urlsplit(href)
query_parsed = parse_qsl(p.query)
query_param_names = [k.lower() for k, v in query_parsed]
query_param_names_ngrams = ngrams_wb(
" ".join([normalize(name) for name in query_param_names]), 3, 5, True
)
elem = link.root
elem_target = _elem_attr(elem, 'target')
elem_rel = _elem_attr(elem, 'rel')
# Classes of link itself and all its children.
# It is common to have e.g. span elements with fontawesome
# arrow icon classes inside <a> links.
self_and_children_classes = ' '.join(link.xpath(".//@class").extract())
parent_classes = ' '.join(link.xpath('../@class').extract())
css_classes = normalize(self_and_children_classes + ' ' + parent_classes)
return {
'bias': 3.0,
'isdigit': text.isdigit(),
'isalpha': text.isalpha(),
'elem-target': elem_target,
'elem-rel': elem_rel,
'num-tokens%s' % _num_tokens_feature(text): 1.0,
'text': ngrams_wb(replace_digits(text), 2, 5),
'text-exact': replace_digits(text.strip()[:20].strip()),
'class': ngrams_wb(css_classes, 4, 5),
'query': query_param_names_ngrams,
'path-has-page': 'page' in p.path.lower(),
'path-has-pageXX': re.search(r'[/-](?:p|page\w?)/?\d+', p.path.lower()) is not None,
'path-has-number': any(part.isdigit() for part in p.path.split('/')),
'href-has-year': re.search('20\d\d', href) is not None,
}
def test_normalize():
assert normalize("Hello,\n world!") == "hello, world!"
def _elem_attr(elem, attr):
return normalize(elem.get(attr, ''))
def _elem_attr(elem, attr):
return normalize(elem.get(attr, ''))
def test_normalize():
assert normalize("Hello,\n world!") == "hello, world!"