def iter_response_link_dicts(response: TextResponse,
limit_by_domain: bool = True) -> Iterator[Dict]:
page_url = response.url
domain_from = get_domain(response.url)
base_url = get_base_url(response)
for link in extract_link_dicts(response.selector, base_url):
link['domain_to'] = get_domain(link['url'])
if limit_by_domain and link['domain_to'] != domain_from:
continue
link['domain_from'] = domain_from
link['page_url'] = page_url
yield link
def _links_to_requests(
self,
response: TextResponse,
links: List[Dict],
links_matrix: sp.csr_matrix,
) -> Iterator[scrapy.Request]:
indices_and_links = list(self.le.deduplicate_links_enumerated(links))
if not indices_and_links:
return
indices, links_to_follow = zip(*indices_and_links)
AS = links_matrix[list(indices)]
scores = self.Q.predict(AS)
for link, v, score in zip(links_to_follow, AS, scores):
url = link['url']
next_domain = get_domain(url)
meta = {
'link_vector': v,
# 'link': link, # turn it on for debugging
'scheduler_slot': next_domain,
}
priority = score_to_priority(score)
req = scrapy.Request(url, priority=priority, meta=meta)
set_request_domain(req, next_domain)
yield req
def _parse(self, response):
if self.is_seed(response) and not hasattr(response, 'text'):
# bad seed
return [], 0
as_t = response.meta.get('link_vector')
if not hasattr(response, 'text'):
# learn to avoid non-html responses
self.Q.add_experience(as_t=as_t, AS_t1=None, r_t1=0)
self.update_node(response, {'reward': 0})
return [], 0
page_vector = self._page_vector(response) if self.use_pages else None
links = self._extract_links(response)
links_matrix = self.link_vectorizer.transform(links) if links else None
links_matrix = self.Q.join_As(links_matrix, page_vector)
if links_matrix is not None:
links_matrix = links_matrix.astype(np.float32) # saving memory
reward = 0
if not self.is_seed(response):
reward = self.goal.get_reward(response)
self.update_node(response, {'reward': reward})
self.total_reward += reward
self.rewards.append(reward)
self.Q.add_experience(as_t=as_t, AS_t1=links_matrix, r_t1=reward)
domain = get_domain(response.url)
self.crawled_domains.add(domain)
if reward > 0.5:
self.relevant_domains.add(domain)
return (list(self._links_to_requests(response, links,
links_matrix)), reward)
def _examples(self):
examples = [
[
'forgot password',
'http://example.com/wp-login.php?action=lostpassword'
],
['registration', 'http://example.com/register'],
['register', 'http://example.com/reg'],
['sign up', 'http://example.com/users/new'],
['my account', 'http://example.com/account/my?sess=GJHFHJS21123'],
['my little pony', 'http://example.com?category=25?sort=1&'],
['comment', 'http://example.com/blog?p=2'],
['sign in', 'http://example.com/users/login'],
['login', 'http://example.com/users/login'],
['forum', 'http://example.com/mybb'],
['forums', 'http://example.com/mybb'],
['forums', 'http://other-domain.com/mybb'],
[
'sadhjgrhgsfd',
'http://example.com/new-to-exhibiting/discover-your-stand-position/'
],
['забыли пароль', 'http://example.com/users/send-password/'],
]
examples_repr = ["{:20s} {}".format(txt, url) for txt, url in examples]
links = [{
'inside_text': txt,
'url': url,
'domain_from': 'example',
'domain_to': get_domain(url),
} for txt, url in examples]
A = self.link_vectorizer.transform(links)
s = self.page_vectorizer.transform([""]) if self.use_pages else None
AS = self.Q.join_As(A, s)
return examples_repr, AS
def process_request(self, request, spider):
if not request.meta.get('domain'):
return
domain = request.meta['domain']
if get_domain(request.url) != domain:
logger.info("Dropped request {}: it doesn't belong to {}".format(
request, domain))
self.signals.send_catch_log(offdomain_request_dropped,
request=request)
raise IgnoreRequest()
def _extract_urls(self, html: str, url: str, sel: parsel.Selector,
base_url: str) -> List[Tuple[float, str]]:
links = list(extract_link_dicts(sel, base_url))
if not links:
return []
domain_from = get_domain(url)
for link in links:
link['domain_from'] = domain_from
link['domain_to'] = get_domain(link['url'])
if self.page_vectorizer:
page_vec = self.page_vectorizer.transform([html])
else:
page_vec = None
link_matrix = self.link_vectorizer.transform(links)
AS = self.Q.join_As(link_matrix, page_vec)
scores = self.Q.predict(AS)
urls = [link['url'] for link in links]
return list(zip(scores, urls))