def test_url_recheck_with_all_valid(session, test_user):
runner = CliRunner()
bookmark = make_bookmark()
sut.set_bookmark(session, test_user.user_uuid, bookmark)
session.commit()
result = runner.invoke(sut.url_recheck)
assert result.exit_code == 0
def make_fulltext_indexed_bookmark(
session: Session, user: User, bookmark: sut.Bookmark, full_text: str
):
# FIXME: this really shows the need for a library of common db functions
url_uuid = sut.set_bookmark(session, user.user_uuid, bookmark)
crawl_uuid = uuid4()
body_uuid = uuid4()
crawl_req = sut.CrawlRequest(
crawl_uuid=crawl_uuid,
url_uuid=url_uuid,
requested=datetime(2018, 1, 3),
got_response=True,
)
crawl_resp = sut.CrawlResponse(
crawl_uuid=crawl_uuid,
headers={"content-type": "text/html"},
body_uuid=body_uuid,
status_code=200,
)
fulltext_obj = sut.FullText(
url_uuid=url_uuid,
crawl_uuid=crawl_uuid,
inserted=datetime.utcnow().replace(tzinfo=timezone.utc),
full_text=full_text,
tsvector=func.to_tsvector(full_text),
)
session.add_all([crawl_req, crawl_resp, fulltext_obj])
def test_create_bookmark_form_add_tag(
signed_in_client,
viewfunc,
tags_for_param,
add_tag,
remove_tag,
expected_tags,
session,
test_user,
cache,
):
if viewfunc == "quarchive.edit_bookmark_form":
bookmark = make_bookmark()
sut.set_bookmark(session, cache, test_user.user_uuid, bookmark)
url = bookmark.url.to_string()
else:
url = "http://example.com"
title = "Example"
description = "A sample website"
unread = "on"
tags = ",".join(tags_for_param)
add_tag = add_tag
params = {
"username": test_user.username,
"url": url,
"title": title,
"description": description,
"unread": unread,
"tags": tags,
"add-tag": add_tag,
"remove-tag": remove_tag,
}
if viewfunc == "quarchive.edit_bookmark_form":
params["url_uuid"] = bookmark.url.url_uuid
response = signed_in_client.get(flask.url_for(viewfunc, **params), )
assert response.status_code == 200
form_as_dict = get_form_as_dict(response)
assert form_as_dict["url"].attrib["value"] == url
assert form_as_dict["title"].attrib["value"] == title
assert form_as_dict["description"].text == description
assert "checked" in form_as_dict["unread"].attrib
assert form_as_dict["tags"].attrib["value"] == ",".join(expected_tags)
assert "value" not in form_as_dict["add-tag"].attrib
def test_pinboard_uses_merge(session, tmpdir, test_user):
runner = CliRunner()
existing_bookmark = make_bookmark(
created=datetime(2018, 2, 1, tzinfo=timezone.utc),
updated=datetime(2018, 2, 1, tzinfo=timezone.utc),
description="as of 2018-02",
)
sut.set_bookmark(session, test_user.user_uuid, existing_bookmark)
session.commit()
pinboard_bookmarks = [
dict(
href=existing_bookmark.url.to_string(),
extended="",
description="as of 2018-01-01",
time=datetime(2018, 1, 12, tzinfo=timezone.utc).isoformat(),
toread=False,
deleted=False,
)
]
json_path = tmpdir.join("pinboard.json")
with open(str(json_path), "w") as json_file:
json.dump(pinboard_bookmarks, json_file)
runner.invoke(
sut.pinboard_import,
[str(test_user.user_uuid), str(json_path), "--as-of", "2018-01-01"],
catch_exceptions=False,
)
url_uuid = existing_bookmark.url.url_uuid
assert (
session.query(sut.SQLABookmark)
.filter(sut.SQLABookmark.url_uuid == url_uuid)
.count()
== 1
)
final_bookmark = sut.get_bookmark_by_url(
session, test_user.user_uuid, existing_bookmark.url.to_string()
)
assert final_bookmark is not None
assert final_bookmark.created == datetime(2018, 1, 12, tzinfo=timezone.utc)
assert final_bookmark.updated == datetime(2018, 2, 1, tzinfo=timezone.utc)
def test_enqueue_crawls_for_uncrawled_urls(session, eager_celery, mock_s3, test_user):
bookmark = make_bookmark()
sut.set_bookmark(session, test_user.user_uuid, bookmark)
session.commit()
url = bookmark.url
responses.add(
responses.GET, re.compile(r"http://example.com/.*"), body=b"hello", stream=True
)
sut.enqueue_crawls_for_uncrawled_urls()
resp_query = (
session.query(sut.CrawlResponse)
.join(sut.CrawlRequest)
.join(sut.SQLAUrl)
.filter(
sut.SQLAUrl.scheme == url.scheme,
sut.SQLAUrl.netloc == url.netloc,
sut.SQLAUrl.path == url.path,
sut.SQLAUrl.query == url.query,
sut.SQLAUrl.fragment == url.fragment,
)
)
assert resp_query.count() == 1
def bad_uuid_bookmark(session, test_user):
url_uuid = UUID("f" * 32)
bad_uuid_url = sut.URL(
url_uuid=url_uuid,
scheme="http",
netloc="example.com",
path="/bad-uuid",
query="",
fragment="",
)
bookmark = make_bookmark(url=bad_uuid_url, tag_triples=frozenset())
bookmark_uuid = sut.set_bookmark(session, test_user.user_uuid, bookmark)
session.commit()
yield
# teardown is required to avoid leaving a dirty db around
delete_bookmark(session, test_user.user_uuid, bookmark_uuid)
delete_url(session, url_uuid)
session.commit()
def bad_canonicalisation_bookmark(session, test_user):
# Needs a trailing slash
url_uuid = uuid5(UUID_URL_NAMESPACE, "http://example.com")
bad_uuid_url = sut.URL(
url_uuid=url_uuid,
scheme="http",
netloc="example.com",
path="",
query="",
fragment="",
)
bookmark = make_bookmark(url=bad_uuid_url, tag_triples=frozenset())
bookmark_uuid = sut.set_bookmark(session, test_user.user_uuid, bookmark)
session.commit()
yield
# teardown is required to avoid leaving a dirty db around
delete_bookmark(session, test_user.user_uuid, bookmark_uuid)
delete_url(session, url_uuid)
session.commit()
def bad_scheme_bookmark(session, test_user):
url_uuid = uuid5(UUID_URL_NAMESPACE, "about:blank")
bad_scheme_url = sut.URL(
url_uuid=url_uuid,
scheme="about",
netloc="",
path="blank",
query="",
fragment="",
)
bookmark = make_bookmark(url=bad_scheme_url, tag_triples=frozenset())
bookmark_uuid = sut.set_bookmark(session, test_user.user_uuid, bookmark)
session.commit()
yield
# teardown is required to avoid leaving a dirty db around
delete_bookmark(session, test_user.user_uuid, bookmark_uuid)
delete_url(session, url_uuid)
session.commit()
