def test_crawl_when_response_is_recieved(session, http_client, status_code, mock_s3, requests_mock): url = URL.from_string("http://example.com/" + random_string()) upsert_url(session, url) requests_mock.add(responses.GET, url.to_string(), body=b"hello", status=status_code, stream=True) request = Request(verb=HTTPVerb.GET, url=url) response = crawler.crawl(session, http_client, request) sql_request = session.query(SQLACrawlRequest).get(response.crawl_uuid) sql_response = session.query(CrawlResponse).get(response.crawl_uuid) assert sql_request.requested == datetime(2018, 1, 3, tzinfo=timezone.utc) assert sql_request.got_response assert sql_response.status_code == status_code assert sql_response.crawl_uuid == sql_response.crawl_uuid assert sql_response.headers == {"content-type": "text/plain"} s3_obj = (file_storage.get_s3().Object( environ["QM_RESPONSE_BODY_BUCKET_NAME"], str(sql_response.body_uuid)).get()) response_body = s3_obj["Body"].read() assert response_body == gzip.compress(b"hello")
def test_crawl_hn_api_no_discussions(session, bg_client, requests_mock): url = random_url() upsert_url(session, url) session.commit() api_url = get_hn_api_url(url) requests_mock.add( responses.GET, url=api_url.to_string(), json=make_algolia_resp(hits=[]), status=200, ) event = FetchDiscussionsCommand(url_uuid=url.url_uuid, source=DiscussionSource.HN) bg_client.send(PickleMessage.from_obj(event)) discussion = (session.query(SQLDiscussion).filter( SQLDiscussion.discussion_source_id == DiscussionSource.HN.value). filter(SQLDiscussion.url_uuid == url.url_uuid).first()) assert discussion is None discussion_fetch = (session.query(SQLDiscussionFetch).filter( SQLDiscussionFetch.url_uuid == url.url_uuid).one()) assert discussion_fetch.discussion_source_id == DiscussionSource.HN.value assert discussion_fetch.status_code == 200 assert discussion_fetch.retrieved == datetime(2018, 1, 3, tzinfo=timezone.utc)
def test_new_icon_found_domain_but_is_already_indexed( session, requests_mock, bg_client: TestAdapter[PickleMessage], mock_s3): icon_url = URL.from_string( f"http://{random_string()}.example.com/favicon.ico") image_buff = random_image_fileobj() hash_bytes = hashlib.blake2b(image_buff.read()).digest() image_buff.seek(0) requests_mock.add( responses.GET, url=icon_url.to_string(), body=image_buff.read(), status=200, stream=True, ) requests_mock.start() upsert_url(session, icon_url) session.commit() event = NewIconFound(icon_url_uuid=icon_url.url_uuid) bg_client.send(PickleMessage.from_obj(event)) bg_client.send(PickleMessage.from_obj(event)) icon, domain_icon = (session.query( Icon, DomainIcon).join(DomainIcon).filter( DomainIcon.scheme == icon_url.scheme, DomainIcon.netloc == icon_url.netloc).one()) assert icon.source_blake2b_hash == hash_bytes assert domain_icon.scheme == icon_url.scheme assert domain_icon.netloc == icon_url.netloc
def test_crawl_hn_api(session, bg_client: TestAdapter[PickleMessage], mock_s3, requests_mock): url = random_url() upsert_url(session, url) session.commit() hn_id = random_numeric_id() api_url = get_hn_api_url(url) requests_mock.add( responses.GET, url=api_url.to_string(), json=make_algolia_resp( hits=[make_algolia_hit(objectID=hn_id, url=url.to_string())]), status=200, ) event = FetchDiscussionsCommand(url_uuid=url.url_uuid, source=DiscussionSource.HN) bg_client.send(PickleMessage.from_obj(event)) discussion = (session.query(SQLDiscussion).filter( SQLDiscussion.discussion_source_id == DiscussionSource.HN.value ).filter(SQLDiscussion.external_discussion_id == str(hn_id)).one()) assert discussion.external_discussion_id == str(hn_id) assert discussion.discussion_source_id == DiscussionSource.HN.value assert discussion.comment_count == 1 assert discussion.url_uuid == url.url_uuid
def test_index_requested_new_page_and_known_page_icon_url( session, bg_worker, mock_s3, requests_mock): """Test that when a page uses an icon url we already have in the index, we reuse it.""" icon_url = URL.from_string( f"http://{random_string()}.example.com/favicon.png") icon_uuid = uuid4() html = f""" <html> <head> <link rel="icon" type="image/png" href="{icon_url.to_string()}"> </head> </html> """ sqla_url, crawl_req, crawl_resp = make_crawl_with_response( session, response_body=BytesIO(html.encode("utf-8"))) image_buff = random_image_fileobj() hash_bytes = hashlib.blake2b(image_buff.read()).digest() upsert_url(session, icon_url) session.add(Icon(icon_uuid=icon_uuid, source_blake2b_hash=hash_bytes)) session.add(IconSource(icon_uuid=icon_uuid, url_uuid=icon_url.url_uuid)) session.commit() bg_worker.send( PickleMessage.from_obj(IndexRequested(crawl_resp.crawl_uuid))) url_icon = (session.query(URLIcon).filter( URLIcon.url_uuid == sqla_url.url_uuid).one()) assert url_icon.url_uuid == sqla_url.url_uuid assert url_icon.icon_uuid == icon_uuid
def icon_message_if_necessary( session: Session, metadata: HTMLMetadata) -> Optional[NewIconFound]: icon = best_icon(metadata) icon_uuid = icon_at_url(session, icon.url) if icon_uuid is not None: log.debug("already have icon for %s (%s)", metadata.url, icon.url) upsert_icon_for_url(session, metadata.url, icon_uuid) return None else: upsert_url(session, icon.url) message = NewIconFound(icon.url.url_uuid) if icon.scope == IconScope.PAGE: message.page_url_uuid = metadata.url.url_uuid return message
def test_recrawl_of_hn_api(session, bg_client: TestAdapter[PickleMessage], mock_s3, requests_mock): url = random_url() upsert_url(session, url) session.commit() hn_id = random_numeric_id() api_url = get_hn_api_url(url) requests_mock.add( responses.GET, url=api_url.to_string(), json=make_algolia_resp( hits=[make_algolia_hit(objectID=hn_id, url=url.to_string())]), status=200, ) event = FetchDiscussionsCommand(url_uuid=url.url_uuid, source=DiscussionSource.HN) bg_client.send(PickleMessage.from_obj(event)) # And again, but with a different comment count requests_mock.remove(responses.GET, url=api_url.to_string()) requests_mock.add( responses.GET, url=api_url.to_string(), json=make_algolia_resp(hits=[ make_algolia_hit( objectID=hn_id, url=url.to_string(), num_comments=5, title="Other example", created_at_i=int(datetime(2018, 1, 4).timestamp()), ) ]), status=200, ) # and again bg_client.send(PickleMessage.from_obj(event)) discussion = (session.query(SQLDiscussion).filter( SQLDiscussion.discussion_source_id == DiscussionSource.HN.value ).filter(SQLDiscussion.external_discussion_id == str(hn_id)).one()) assert discussion.comment_count == 5 assert discussion.url_uuid == url.url_uuid assert discussion.title == "Other example" assert discussion.created_at == datetime(2018, 1, 4, tzinfo=timezone.utc)
def test_crawl_when_no_response(session, http_client, requests_mock): url = URL.from_string("http://example.com/" + random_string()) upsert_url(session, url) requests_mock.add( responses.GET, url.to_string(), body=requests.exceptions.ConnectTimeout("connect timeout"), ) response = crawler.crawl(session, http_client, Request(verb=HTTPVerb.GET, url=url)) sql_request = session.query(SQLACrawlRequest).get(response.crawl_uuid) sql_response = session.query(CrawlResponse).get(response.crawl_uuid) assert sql_request is not None assert sql_response is None
def test_new_icon_found_for_page_url_duplicated_by_content( session, requests_mock, bg_client: TestAdapter[PickleMessage], mock_s3): """Test that when a new page icon is found that is the same icon by hash as an existing icon, that it is recorded.""" page_url_1 = URL.from_string( f"http://{random_string()}.example.com/index.html") page_url_2 = page_url_1.follow("/otherindex.html") icon_url_1 = page_url_1.follow("favicon1.png") icon_url_2 = page_url_2.follow("favicon2.png") image_buff = random_image_fileobj() hash_bytes = hashlib.blake2b(image_buff.read()).digest() image_buff.seek(0) requests_mock.add( responses.GET, url=icon_url_1.to_string(), body=image_buff.read(), status=200, stream=True, ) image_buff.seek(0) requests_mock.add( responses.GET, url=icon_url_2.to_string(), body=image_buff.read(), status=200, stream=True, ) requests_mock.start() upsert_url(session, page_url_1) upsert_url(session, page_url_2) upsert_url(session, icon_url_1) upsert_url(session, icon_url_2) session.commit() event = NewIconFound(icon_url_uuid=icon_url_1.url_uuid, page_url_uuid=page_url_1.url_uuid) bg_client.send(PickleMessage.from_obj(event)) event = NewIconFound(icon_url_uuid=icon_url_2.url_uuid, page_url_uuid=page_url_2.url_uuid) bg_client.send(PickleMessage.from_obj(event)) url_icon_obj_1, url_icon_obj_2 = (session.query(URLIcon).join( SQLAUrl, URLIcon.url_uuid == SQLAUrl.url_uuid).filter( SQLAUrl.netloc == page_url_1.netloc).order_by(SQLAUrl.path).all()) assert url_icon_obj_1.icon == url_icon_obj_2.icon assert url_icon_obj_1.icon.source_blake2b_hash == hash_bytes
def test_new_icon_found_for_page_icon(session, requests_mock, bg_client: TestAdapter[PickleMessage], mock_s3): """Test that when a new page icon is found (that doesn't match any existing icons) that it is retrieved, indexed and stored. """ url = URL.from_string(f"http://{random_string()}.example.com/") icon_url = url.follow("/favicon.png") image_buff = random_image_fileobj() hash_bytes = hashlib.blake2b(image_buff.read()).digest() image_buff.seek(0) requests_mock.add( responses.GET, url=icon_url.to_string(), body=image_buff.read(), status=200, stream=True, ) requests_mock.start() upsert_url(session, url) upsert_url(session, icon_url) session.commit() event = NewIconFound(icon_url_uuid=icon_url.url_uuid, page_url_uuid=url.url_uuid) bg_client.send(PickleMessage.from_obj(event)) icon, url_icon = (session.query(Icon, URLIcon).join(URLIcon).filter( URLIcon.url_uuid == url.url_uuid).first()) assert icon.source_blake2b_hash == hash_bytes assert url_icon.url_uuid == url.url_uuid icon_bucket = file_storage.get_icon_bucket() (s3_obj, ) = list( icon_bucket.objects.filter(Prefix=f"{icon.icon_uuid}.png")) assert s3_obj.key == f"{icon.icon_uuid}.png" response = s3_obj.get() assert response["ResponseMetadata"]["HTTPHeaders"][ "content-type"] == "image/png"
def test_icon_uuids_url_icon(session, test_user, canonical_url): bm1 = make_bookmark() set_bookmark(session, test_user.user_uuid, bm1) if canonical_url: canonical_url = bm1.url.follow("canonical.html") upsert_url(session, canonical_url) session.add( CanonicalUrl( non_canonical_url_uuid=bm1.url.url_uuid, canonical_url_uuid=canonical_url.url_uuid, ) ) icon_url = URL.from_string("http://example.com/" + random_string() + "/icon.png") upsert_url(session, icon_url) random_hash = random_bytes(64) icon_uuid = record_page_icon(session, icon_url, bm1.url, random_hash) (bm1_view,) = (f for f in BookmarkViewQueryBuilder(session, test_user).execute()) assert bm1_view.icon_uuid == icon_uuid
def test_new_icon_found_for_page_url_duplicated_by_url( session, bg_client: TestAdapter[PickleMessage], mock_s3, requests_mock): """Test that when a new page icon is found that is the same icon by hash as an existing icon, that it is recorded.""" page_url_1 = URL.from_string( f"http://{random_string()}.example.com/index.html") page_url_2 = page_url_1.follow("/otherindex.html") icon_url = page_url_1.follow("favicon1.png") hash_bytes = bytes(random.getrandbits(8) for _ in range(64)) upsert_url(session, page_url_1) upsert_url(session, page_url_2) upsert_url(session, icon_url) icon_uuid = uuid4() session.add(Icon(icon_uuid=icon_uuid, source_blake2b_hash=hash_bytes)) session.add(IconSource(icon_uuid=icon_uuid, url_uuid=icon_url.url_uuid)) session.add(URLIcon(url_uuid=page_url_1.url_uuid, icon_uuid=icon_uuid)) session.commit() event = NewIconFound(icon_url_uuid=icon_url.url_uuid, page_url_uuid=page_url_2.url_uuid) bg_client.send(PickleMessage.from_obj(event)) url_icon_obj_1, url_icon_obj_2 = (session.query(URLIcon).join( SQLAUrl, URLIcon.url_uuid == SQLAUrl.url_uuid).filter( SQLAUrl.netloc == page_url_1.netloc).order_by(SQLAUrl.path).all()) assert url_icon_obj_1.icon == url_icon_obj_2.icon assert url_icon_obj_1.icon.source_blake2b_hash == hash_bytes
def test_new_icon_found_domain(session, requests_mock, bg_client: TestAdapter[PickleMessage], mock_s3): icon_url = URL.from_string( f"http://{random_string()}.example.com/favicon.ico") image_buff = random_image_fileobj() hash_bytes = hashlib.blake2b(image_buff.read()).digest() image_buff.seek(0) requests_mock.add( responses.GET, url=icon_url.to_string(), body=image_buff.read(), status=200, stream=True, ) requests_mock.start() upsert_url(session, icon_url) session.commit() event = NewIconFound(icon_url_uuid=icon_url.url_uuid) bg_client.send(PickleMessage.from_obj(event)) icon, domain_icon = (session.query( Icon, DomainIcon).join(DomainIcon).filter( DomainIcon.scheme == icon_url.scheme, DomainIcon.netloc == icon_url.netloc).first()) assert icon.source_blake2b_hash == hash_bytes assert domain_icon.scheme == icon_url.scheme assert domain_icon.netloc == icon_url.netloc icon_bucket = file_storage.get_icon_bucket() (s3_obj, ) = list( icon_bucket.objects.filter(Prefix=f"{icon.icon_uuid}.png")) assert s3_obj.key == f"{icon.icon_uuid}.png" response = s3_obj.get() assert response["ResponseMetadata"]["HTTPHeaders"][ "content-type"] == "image/png"
def test_bookmark_created(session, bg_worker, mock_s3, requests_mock, test_user): url = URL.from_string("http://example.com/" + random_string()) upsert_url(session, url) session.commit() requests_mock.add( responses.GET, url=url.to_string(), body="Hello!", status=200, stream=True, ) bg_worker.send( PickleMessage.from_obj( BookmarkCreated(user_uuid=test_user.user_uuid, url_uuid=url.url_uuid))) response_exists = session.query( session.query(SQLACrawlResponse).join(SQLACrawlRequest).join(SQLAUrl). filter(SQLAUrl.url_uuid == url.url_uuid).exists()).scalar() assert response_exists
def test_multi_page_hn_api(session, bg_client: TestAdapter[PickleMessage], mock_s3, requests_mock): url = random_url() upsert_url(session, url) session.commit() api_url1 = get_hn_api_url(url) api_url2 = URL.from_string(api_url1.to_string() + "&page=1") requests_mock.add( responses.GET, url=api_url1.to_string(), json=make_algolia_resp(nbPages=2, hitsPerPage=1, hits=[make_algolia_hit(url=url.to_string())]), status=200, ) requests_mock.add( responses.GET, url=api_url2.to_string(), json=make_algolia_resp( nbPages=2, page=1, hitsPerPage=1, hits=[make_algolia_hit(url=url.to_string())], ), status=200, ) event = FetchDiscussionsCommand(url_uuid=url.url_uuid, source=DiscussionSource.HN) bg_client.send(PickleMessage.from_obj(event)) discussion_count = (session.query(SQLDiscussion).filter( SQLDiscussion.discussion_source_id == DiscussionSource.HN.value). filter(SQLDiscussion.url_uuid == url.url_uuid).count()) assert discussion_count == 2