def fetch_download(self, db: DatabaseHandler, download: dict) -> Optional[Response]:
download = decode_object_from_bytes_if_needed(download)
url = self._download_url(download=download)
if not is_http_url(url):
raise McCrawlerFetcherSoftError(f"URL is not HTTP(s): {url}")
download['download_time'] = sql_now()
download['state'] = 'fetching'
try:
db.update_by_id(table='downloads', object_id=download['downloads_id'], update_hash=download)
except McTupleAlreadyMovedError as ex:
# Some attempts to set the download's row to "fetching" fail with:
#
# "tuple to be locked was already moved to another partition due to concurrent update"
#
# If that happens, we assume that some other fetcher instance somehow got to the download first and do
# nothing
log.warning(f"Some other fetcher got to download {download['downloads_id']} first: {ex}")
return None
except Exception as ex:
# Raise further on misc. errors
raise ex
ua = UserAgent()
response = ua.get_follow_http_html_redirects(url)
return response
def fetch_download(self, db: DatabaseHandler, download: dict) -> Response:
download = decode_object_from_bytes_if_needed(download)
url = self._download_url(download=download)
if not is_http_url(url):
raise McCrawlerFetcherSoftError(f"URL is not HTTP(s): {url}")
download['download_time'] = sql_now()
download['state'] = 'fetching'
db.update_by_id(table='downloads',
object_id=download['downloads_id'],
update_hash=download)
ua = UserAgent()
response = ua.get_follow_http_html_redirects(url)
return response
def all_url_variants(db: DatabaseHandler, url: str) -> List[str]:
"""Given the URL, return all URL variants that we can think of:
1) Normal URL (the one passed as a parameter)
2) URL after redirects (i.e., fetch the URL, see if it gets redirected somewhere)
3) Canonical URL (after removing #fragments, session IDs, tracking parameters, etc.)
4) Canonical URL after redirects (do the redirect check first, then strip the tracking parameters from the URL)
5) URL from <link rel="canonical" /> (if any)
6) Any alternative URLs from topic_merged_stories or topic_links"""
url = decode_object_from_bytes_if_needed(url)
if url is None:
raise McAllURLVariantsException("URL is None.")
url = fix_common_url_mistakes(url)
if not is_http_url(url):
log.warning("URL %s is not a valid HTTP URL." % url)
return [
url,
]
# Get URL after HTTP / HTML redirects
ua = UserAgent()
response = ua.get_follow_http_html_redirects(url)
url_after_redirects = response.request().url()
data_after_redirects = response.decoded_content()
urls = {
# Normal URL (don't touch anything)
'normal': url,
# Normal URL after redirects
'after_redirects': url_after_redirects,
# Canonical URL
'normalized': normalize_url(url),
# Canonical URL after redirects
'after_redirects_normalized': normalize_url(url_after_redirects),
}
# If <link rel="canonical" /> is present, try that one too
if data_after_redirects is not None:
url_link_rel_canonical = link_canonical_url_from_html(
html=data_after_redirects, base_url=url_after_redirects)
if url_link_rel_canonical is not None and len(
url_link_rel_canonical) > 0:
log.debug(
('Found <link rel="canonical" /> for URL %(url_after_redirects)s '
'(original URL: %(url)s): %(url_link_rel_canonical)s') % {
"url_after_redirects": url_after_redirects,
"url": url,
"url_link_rel_canonical": url_link_rel_canonical,
})
urls['after_redirects_canonical'] = url_link_rel_canonical
# If URL gets redirected to the homepage (e.g.
# http://m.wired.com/threatlevel/2011/12/sopa-watered-down-amendment/ leads
# to http://www.wired.com/), don't use those redirects
if not is_homepage_url(url):
urls = {
key: urls[key]
for key in urls.keys() if not is_homepage_url(urls[key])
}
distinct_urls = list(set(urls.values()))
topic_urls = __get_topic_url_variants(db=db, urls=distinct_urls)
distinct_urls = distinct_urls + topic_urls
distinct_urls = list(set(distinct_urls))
# Remove URLs that can't be variants of the initial URL
for invalid_url_variant_regex in __INVALID_URL_VARIANT_REGEXES:
distinct_urls = [
x for x in distinct_urls
if not re.search(pattern=invalid_url_variant_regex, string=x)
]
return distinct_urls
def all_url_variants(db: DatabaseHandler, url: str) -> List[str]:
"""Given the URL, return all URL variants that we can think of:
1) Normal URL (the one passed as a parameter)
2) URL after redirects (i.e., fetch the URL, see if it gets redirected somewhere)
3) Canonical URL (after removing #fragments, session IDs, tracking parameters, etc.)
4) Canonical URL after redirects (do the redirect check first, then strip the tracking parameters from the URL)
5) URL from <link rel="canonical" /> (if any)
6) Any alternative URLs from topic_merged_stories or topic_links"""
url = decode_object_from_bytes_if_needed(url)
if url is None:
raise McAllURLVariantsException("URL is None.")
url = fix_common_url_mistakes(url)
if not is_http_url(url):
log.warning("URL %s is not a valid HTTP URL." % url)
return [
url,
]
# Get URL after HTTP / HTML redirects
ua = UserAgent()
response = ua.get_follow_http_html_redirects(url)
url_after_redirects = response.request().url()
data_after_redirects = response.decoded_content()
urls = {
# Normal URL (don't touch anything)
'normal': url,
# Normal URL after redirects
'after_redirects': url_after_redirects,
# Canonical URL
'normalized': normalize_url(url),
# Canonical URL after redirects
'after_redirects_normalized': normalize_url(url_after_redirects),
}
# If <link rel="canonical" /> is present, try that one too
if data_after_redirects is not None:
url_link_rel_canonical = link_canonical_url_from_html(html=data_after_redirects, base_url=url_after_redirects)
if url_link_rel_canonical is not None and len(url_link_rel_canonical) > 0:
log.debug(
(
'Found <link rel="canonical" /> for URL %(url_after_redirects)s '
'(original URL: %(url)s): %(url_link_rel_canonical)s'
) % {
"url_after_redirects": url_after_redirects,
"url": url,
"url_link_rel_canonical": url_link_rel_canonical,
}
)
urls['after_redirects_canonical'] = url_link_rel_canonical
# If URL gets redirected to the homepage (e.g.
# http://m.wired.com/threatlevel/2011/12/sopa-watered-down-amendment/ leads
# to http://www.wired.com/), don't use those redirects
if not is_homepage_url(url):
urls = {key: urls[key] for key in urls.keys() if not is_homepage_url(urls[key])}
distinct_urls = list(set(urls.values()))
topic_urls = __get_topic_url_variants(db=db, urls=distinct_urls)
distinct_urls = distinct_urls + topic_urls
distinct_urls = list(set(distinct_urls))
# Remove URLs that can't be variants of the initial URL
for invalid_url_variant_regex in __INVALID_URL_VARIANT_REGEXES:
distinct_urls = [x for x in distinct_urls if not re.search(pattern=invalid_url_variant_regex, string=x)]
return distinct_urls