def test_crawl_when_response_is_recieved(session, http_client, status_code,
mock_s3, requests_mock):
url = URL.from_string("http://example.com/" + random_string())
upsert_url(session, url)
requests_mock.add(responses.GET,
url.to_string(),
body=b"hello",
status=status_code,
stream=True)
request = Request(verb=HTTPVerb.GET, url=url)
response = crawler.crawl(session, http_client, request)
sql_request = session.query(SQLACrawlRequest).get(response.crawl_uuid)
sql_response = session.query(CrawlResponse).get(response.crawl_uuid)
assert sql_request.requested == datetime(2018, 1, 3, tzinfo=timezone.utc)
assert sql_request.got_response
assert sql_response.status_code == status_code
assert sql_response.crawl_uuid == sql_response.crawl_uuid
assert sql_response.headers == {"content-type": "text/plain"}
s3_obj = (file_storage.get_s3().Object(
environ["QM_RESPONSE_BODY_BUCKET_NAME"],
str(sql_response.body_uuid)).get())
response_body = s3_obj["Body"].read()
assert response_body == gzip.compress(b"hello")
def test_crawl_hn_api_no_discussions(session, bg_client, requests_mock):
url = random_url()
upsert_url(session, url)
session.commit()
api_url = get_hn_api_url(url)
requests_mock.add(
responses.GET,
url=api_url.to_string(),
json=make_algolia_resp(hits=[]),
status=200,
)
event = FetchDiscussionsCommand(url_uuid=url.url_uuid,
source=DiscussionSource.HN)
bg_client.send(PickleMessage.from_obj(event))
discussion = (session.query(SQLDiscussion).filter(
SQLDiscussion.discussion_source_id == DiscussionSource.HN.value).
filter(SQLDiscussion.url_uuid == url.url_uuid).first())
assert discussion is None
discussion_fetch = (session.query(SQLDiscussionFetch).filter(
SQLDiscussionFetch.url_uuid == url.url_uuid).one())
assert discussion_fetch.discussion_source_id == DiscussionSource.HN.value
assert discussion_fetch.status_code == 200
assert discussion_fetch.retrieved == datetime(2018,
1,
3,
tzinfo=timezone.utc)
def test_new_icon_found_domain_but_is_already_indexed(
session, requests_mock, bg_client: TestAdapter[PickleMessage],
mock_s3):
icon_url = URL.from_string(
f"http://{random_string()}.example.com/favicon.ico")
image_buff = random_image_fileobj()
hash_bytes = hashlib.blake2b(image_buff.read()).digest()
image_buff.seek(0)
requests_mock.add(
responses.GET,
url=icon_url.to_string(),
body=image_buff.read(),
status=200,
stream=True,
)
requests_mock.start()
upsert_url(session, icon_url)
session.commit()
event = NewIconFound(icon_url_uuid=icon_url.url_uuid)
bg_client.send(PickleMessage.from_obj(event))
bg_client.send(PickleMessage.from_obj(event))
icon, domain_icon = (session.query(
Icon, DomainIcon).join(DomainIcon).filter(
DomainIcon.scheme == icon_url.scheme,
DomainIcon.netloc == icon_url.netloc).one())
assert icon.source_blake2b_hash == hash_bytes
assert domain_icon.scheme == icon_url.scheme
assert domain_icon.netloc == icon_url.netloc
def test_crawl_hn_api(session, bg_client: TestAdapter[PickleMessage], mock_s3,
requests_mock):
url = random_url()
upsert_url(session, url)
session.commit()
hn_id = random_numeric_id()
api_url = get_hn_api_url(url)
requests_mock.add(
responses.GET,
url=api_url.to_string(),
json=make_algolia_resp(
hits=[make_algolia_hit(objectID=hn_id, url=url.to_string())]),
status=200,
)
event = FetchDiscussionsCommand(url_uuid=url.url_uuid,
source=DiscussionSource.HN)
bg_client.send(PickleMessage.from_obj(event))
discussion = (session.query(SQLDiscussion).filter(
SQLDiscussion.discussion_source_id == DiscussionSource.HN.value
).filter(SQLDiscussion.external_discussion_id == str(hn_id)).one())
assert discussion.external_discussion_id == str(hn_id)
assert discussion.discussion_source_id == DiscussionSource.HN.value
assert discussion.comment_count == 1
assert discussion.url_uuid == url.url_uuid
def test_index_requested_new_page_and_known_page_icon_url(
session, bg_worker, mock_s3, requests_mock):
"""Test that when a page uses an icon url we already have in the index, we reuse it."""
icon_url = URL.from_string(
f"http://{random_string()}.example.com/favicon.png")
icon_uuid = uuid4()
html = f"""
<html>
<head>
<link rel="icon" type="image/png" href="{icon_url.to_string()}">
</head>
</html>
"""
sqla_url, crawl_req, crawl_resp = make_crawl_with_response(
session, response_body=BytesIO(html.encode("utf-8")))
image_buff = random_image_fileobj()
hash_bytes = hashlib.blake2b(image_buff.read()).digest()
upsert_url(session, icon_url)
session.add(Icon(icon_uuid=icon_uuid, source_blake2b_hash=hash_bytes))
session.add(IconSource(icon_uuid=icon_uuid, url_uuid=icon_url.url_uuid))
session.commit()
bg_worker.send(
PickleMessage.from_obj(IndexRequested(crawl_resp.crawl_uuid)))
url_icon = (session.query(URLIcon).filter(
URLIcon.url_uuid == sqla_url.url_uuid).one())
assert url_icon.url_uuid == sqla_url.url_uuid
assert url_icon.icon_uuid == icon_uuid
def icon_message_if_necessary(
session: Session, metadata: HTMLMetadata) -> Optional[NewIconFound]:
icon = best_icon(metadata)
icon_uuid = icon_at_url(session, icon.url)
if icon_uuid is not None:
log.debug("already have icon for %s (%s)", metadata.url, icon.url)
upsert_icon_for_url(session, metadata.url, icon_uuid)
return None
else:
upsert_url(session, icon.url)
message = NewIconFound(icon.url.url_uuid)
if icon.scope == IconScope.PAGE:
message.page_url_uuid = metadata.url.url_uuid
return message
def test_recrawl_of_hn_api(session, bg_client: TestAdapter[PickleMessage],
mock_s3, requests_mock):
url = random_url()
upsert_url(session, url)
session.commit()
hn_id = random_numeric_id()
api_url = get_hn_api_url(url)
requests_mock.add(
responses.GET,
url=api_url.to_string(),
json=make_algolia_resp(
hits=[make_algolia_hit(objectID=hn_id, url=url.to_string())]),
status=200,
)
event = FetchDiscussionsCommand(url_uuid=url.url_uuid,
source=DiscussionSource.HN)
bg_client.send(PickleMessage.from_obj(event))
# And again, but with a different comment count
requests_mock.remove(responses.GET, url=api_url.to_string())
requests_mock.add(
responses.GET,
url=api_url.to_string(),
json=make_algolia_resp(hits=[
make_algolia_hit(
objectID=hn_id,
url=url.to_string(),
num_comments=5,
title="Other example",
created_at_i=int(datetime(2018, 1, 4).timestamp()),
)
]),
status=200,
)
# and again
bg_client.send(PickleMessage.from_obj(event))
discussion = (session.query(SQLDiscussion).filter(
SQLDiscussion.discussion_source_id == DiscussionSource.HN.value
).filter(SQLDiscussion.external_discussion_id == str(hn_id)).one())
assert discussion.comment_count == 5
assert discussion.url_uuid == url.url_uuid
assert discussion.title == "Other example"
assert discussion.created_at == datetime(2018, 1, 4, tzinfo=timezone.utc)
def test_crawl_when_no_response(session, http_client, requests_mock):
url = URL.from_string("http://example.com/" + random_string())
upsert_url(session, url)
requests_mock.add(
responses.GET,
url.to_string(),
body=requests.exceptions.ConnectTimeout("connect timeout"),
)
response = crawler.crawl(session, http_client,
Request(verb=HTTPVerb.GET, url=url))
sql_request = session.query(SQLACrawlRequest).get(response.crawl_uuid)
sql_response = session.query(CrawlResponse).get(response.crawl_uuid)
assert sql_request is not None
assert sql_response is None
def test_new_icon_found_for_page_url_duplicated_by_content(
session, requests_mock, bg_client: TestAdapter[PickleMessage],
mock_s3):
"""Test that when a new page icon is found that is the same icon by hash as
an existing icon, that it is recorded."""
page_url_1 = URL.from_string(
f"http://{random_string()}.example.com/index.html")
page_url_2 = page_url_1.follow("/otherindex.html")
icon_url_1 = page_url_1.follow("favicon1.png")
icon_url_2 = page_url_2.follow("favicon2.png")
image_buff = random_image_fileobj()
hash_bytes = hashlib.blake2b(image_buff.read()).digest()
image_buff.seek(0)
requests_mock.add(
responses.GET,
url=icon_url_1.to_string(),
body=image_buff.read(),
status=200,
stream=True,
)
image_buff.seek(0)
requests_mock.add(
responses.GET,
url=icon_url_2.to_string(),
body=image_buff.read(),
status=200,
stream=True,
)
requests_mock.start()
upsert_url(session, page_url_1)
upsert_url(session, page_url_2)
upsert_url(session, icon_url_1)
upsert_url(session, icon_url_2)
session.commit()
event = NewIconFound(icon_url_uuid=icon_url_1.url_uuid,
page_url_uuid=page_url_1.url_uuid)
bg_client.send(PickleMessage.from_obj(event))
event = NewIconFound(icon_url_uuid=icon_url_2.url_uuid,
page_url_uuid=page_url_2.url_uuid)
bg_client.send(PickleMessage.from_obj(event))
url_icon_obj_1, url_icon_obj_2 = (session.query(URLIcon).join(
SQLAUrl, URLIcon.url_uuid == SQLAUrl.url_uuid).filter(
SQLAUrl.netloc == page_url_1.netloc).order_by(SQLAUrl.path).all())
assert url_icon_obj_1.icon == url_icon_obj_2.icon
assert url_icon_obj_1.icon.source_blake2b_hash == hash_bytes
def test_new_icon_found_for_page_icon(session, requests_mock,
bg_client: TestAdapter[PickleMessage],
mock_s3):
"""Test that when a new page icon is found (that doesn't match any existing
icons) that it is retrieved, indexed and stored.
"""
url = URL.from_string(f"http://{random_string()}.example.com/")
icon_url = url.follow("/favicon.png")
image_buff = random_image_fileobj()
hash_bytes = hashlib.blake2b(image_buff.read()).digest()
image_buff.seek(0)
requests_mock.add(
responses.GET,
url=icon_url.to_string(),
body=image_buff.read(),
status=200,
stream=True,
)
requests_mock.start()
upsert_url(session, url)
upsert_url(session, icon_url)
session.commit()
event = NewIconFound(icon_url_uuid=icon_url.url_uuid,
page_url_uuid=url.url_uuid)
bg_client.send(PickleMessage.from_obj(event))
icon, url_icon = (session.query(Icon, URLIcon).join(URLIcon).filter(
URLIcon.url_uuid == url.url_uuid).first())
assert icon.source_blake2b_hash == hash_bytes
assert url_icon.url_uuid == url.url_uuid
icon_bucket = file_storage.get_icon_bucket()
(s3_obj, ) = list(
icon_bucket.objects.filter(Prefix=f"{icon.icon_uuid}.png"))
assert s3_obj.key == f"{icon.icon_uuid}.png"
response = s3_obj.get()
assert response["ResponseMetadata"]["HTTPHeaders"][
"content-type"] == "image/png"
def test_icon_uuids_url_icon(session, test_user, canonical_url):
bm1 = make_bookmark()
set_bookmark(session, test_user.user_uuid, bm1)
if canonical_url:
canonical_url = bm1.url.follow("canonical.html")
upsert_url(session, canonical_url)
session.add(
CanonicalUrl(
non_canonical_url_uuid=bm1.url.url_uuid,
canonical_url_uuid=canonical_url.url_uuid,
)
)
icon_url = URL.from_string("http://example.com/" + random_string() + "/icon.png")
upsert_url(session, icon_url)
random_hash = random_bytes(64)
icon_uuid = record_page_icon(session, icon_url, bm1.url, random_hash)
(bm1_view,) = (f for f in BookmarkViewQueryBuilder(session, test_user).execute())
assert bm1_view.icon_uuid == icon_uuid
def test_new_icon_found_for_page_url_duplicated_by_url(
session, bg_client: TestAdapter[PickleMessage], mock_s3,
requests_mock):
"""Test that when a new page icon is found that is the same icon by hash as
an existing icon, that it is recorded."""
page_url_1 = URL.from_string(
f"http://{random_string()}.example.com/index.html")
page_url_2 = page_url_1.follow("/otherindex.html")
icon_url = page_url_1.follow("favicon1.png")
hash_bytes = bytes(random.getrandbits(8) for _ in range(64))
upsert_url(session, page_url_1)
upsert_url(session, page_url_2)
upsert_url(session, icon_url)
icon_uuid = uuid4()
session.add(Icon(icon_uuid=icon_uuid, source_blake2b_hash=hash_bytes))
session.add(IconSource(icon_uuid=icon_uuid, url_uuid=icon_url.url_uuid))
session.add(URLIcon(url_uuid=page_url_1.url_uuid, icon_uuid=icon_uuid))
session.commit()
event = NewIconFound(icon_url_uuid=icon_url.url_uuid,
page_url_uuid=page_url_2.url_uuid)
bg_client.send(PickleMessage.from_obj(event))
url_icon_obj_1, url_icon_obj_2 = (session.query(URLIcon).join(
SQLAUrl, URLIcon.url_uuid == SQLAUrl.url_uuid).filter(
SQLAUrl.netloc == page_url_1.netloc).order_by(SQLAUrl.path).all())
assert url_icon_obj_1.icon == url_icon_obj_2.icon
assert url_icon_obj_1.icon.source_blake2b_hash == hash_bytes
def test_new_icon_found_domain(session, requests_mock,
bg_client: TestAdapter[PickleMessage], mock_s3):
icon_url = URL.from_string(
f"http://{random_string()}.example.com/favicon.ico")
image_buff = random_image_fileobj()
hash_bytes = hashlib.blake2b(image_buff.read()).digest()
image_buff.seek(0)
requests_mock.add(
responses.GET,
url=icon_url.to_string(),
body=image_buff.read(),
status=200,
stream=True,
)
requests_mock.start()
upsert_url(session, icon_url)
session.commit()
event = NewIconFound(icon_url_uuid=icon_url.url_uuid)
bg_client.send(PickleMessage.from_obj(event))
icon, domain_icon = (session.query(
Icon, DomainIcon).join(DomainIcon).filter(
DomainIcon.scheme == icon_url.scheme,
DomainIcon.netloc == icon_url.netloc).first())
assert icon.source_blake2b_hash == hash_bytes
assert domain_icon.scheme == icon_url.scheme
assert domain_icon.netloc == icon_url.netloc
icon_bucket = file_storage.get_icon_bucket()
(s3_obj, ) = list(
icon_bucket.objects.filter(Prefix=f"{icon.icon_uuid}.png"))
assert s3_obj.key == f"{icon.icon_uuid}.png"
response = s3_obj.get()
assert response["ResponseMetadata"]["HTTPHeaders"][
"content-type"] == "image/png"
def test_bookmark_created(session, bg_worker, mock_s3, requests_mock,
test_user):
url = URL.from_string("http://example.com/" + random_string())
upsert_url(session, url)
session.commit()
requests_mock.add(
responses.GET,
url=url.to_string(),
body="Hello!",
status=200,
stream=True,
)
bg_worker.send(
PickleMessage.from_obj(
BookmarkCreated(user_uuid=test_user.user_uuid,
url_uuid=url.url_uuid)))
response_exists = session.query(
session.query(SQLACrawlResponse).join(SQLACrawlRequest).join(SQLAUrl).
filter(SQLAUrl.url_uuid == url.url_uuid).exists()).scalar()
assert response_exists
def test_multi_page_hn_api(session, bg_client: TestAdapter[PickleMessage],
mock_s3, requests_mock):
url = random_url()
upsert_url(session, url)
session.commit()
api_url1 = get_hn_api_url(url)
api_url2 = URL.from_string(api_url1.to_string() + "&page=1")
requests_mock.add(
responses.GET,
url=api_url1.to_string(),
json=make_algolia_resp(nbPages=2,
hitsPerPage=1,
hits=[make_algolia_hit(url=url.to_string())]),
status=200,
)
requests_mock.add(
responses.GET,
url=api_url2.to_string(),
json=make_algolia_resp(
nbPages=2,
page=1,
hitsPerPage=1,
hits=[make_algolia_hit(url=url.to_string())],
),
status=200,
)
event = FetchDiscussionsCommand(url_uuid=url.url_uuid,
source=DiscussionSource.HN)
bg_client.send(PickleMessage.from_obj(event))
discussion_count = (session.query(SQLDiscussion).filter(
SQLDiscussion.discussion_source_id == DiscussionSource.HN.value).
filter(SQLDiscussion.url_uuid == url.url_uuid).count())
assert discussion_count == 2
