def test_document_find_by_uris_no_matches():
document = Document()
document.document_uris.append(DocumentURI(
claimant='https://en.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page'))
db.Session.add(document)
db.Session.flush()
actual = Document.find_by_uris(db.Session, ['https://de.wikipedia.org/wiki/Hauptseite'])
assert actual.count() == 0
def test_document_title():
doc = Document()
DocumentMeta(type='title', value='The Title', document=doc, claimant='http://example.com')
db.Session.add(doc)
db.Session.flush()
assert doc.title == 'The Title'
def test_document_title_meta_not_found():
doc = Document()
DocumentMeta(type='other', value='something', document=doc, claimant='http://example.com')
db.Session.add(doc)
db.Session.flush()
assert doc.title is None
def create_or_update_document_objects(es_ann):
es_doc = es_ann.document
if not es_doc:
return
uris = [u.uri for u in es_doc.document_uris]
documents = Document.find_or_create_by_uris(Session,
es_ann.target_uri,
uris,
created=es_doc.created,
updated=es_doc.updated)
if documents.count() > 1:
document = merge_documents(Session, documents, updated=es_doc.updated)
else:
document = documents.first()
document.updated = es_doc.updated
for uri_ in es_doc.document_uris:
create_or_update_document_uri(uri_, document)
for meta in es_doc.meta:
create_or_update_document_meta(meta, document)
def expand_uri(request, uri):
"""
Return all URIs which refer to the same underlying document as `uri`.
This function determines whether we already have "document" records for the
passed URI, and if so returns the set of all URIs which we currently
believe refer to the same document.
:param request: the request object
:type request: pyramid.request.Request
:param uri: a URI associated with the document
:type id: str
:returns: a list of equivalent URIs
:rtype: list
"""
doc = None
if _postgres_enabled(request):
doc = Document.find_by_uris(request.db, [uri]).one_or_none()
else:
doc = elastic.Document.get_by_uri(uri)
if doc is None:
return [uri]
# We check if the match was a "canonical" link. If so, all annotations
# created on that page are guaranteed to have that as their target.source
# field, so we don't need to expand to other URIs and risk false positives.
docuris = doc.document_uris
for docuri in docuris:
if docuri.uri == uri and docuri.type == 'rel-canonical':
return [uri]
return [docuri.uri for docuri in docuris]
def test_document(annotation):
document = Document(document_uris=[
DocumentURI(claimant=annotation.target_uri, uri=annotation.target_uri)
])
db.Session.add(document)
db.Session.flush()
assert annotation.document == document
def test_document_not_found(annotation):
document = Document(document_uris=[
DocumentURI(claimant='something-else', uri='something-else')
])
db.Session.add(document)
db.Session.flush()
assert annotation.document is None
def test_document_find_by_uris():
document1 = Document()
uri1 = 'https://de.wikipedia.org/wiki/Hauptseite'
document1.document_uris.append(DocumentURI(claimant=uri1, uri=uri1))
document2 = Document()
uri2 = 'https://en.wikipedia.org/wiki/Main_Page'
document2.document_uris.append(DocumentURI(claimant=uri2, uri=uri2))
uri3 = 'https://en.wikipedia.org'
document2.document_uris.append(DocumentURI(claimant=uri3, uri=uri2))
db.Session.add_all([document1, document2])
db.Session.flush()
actual = Document.find_by_uris(db.Session, [
'https://en.wikipedia.org/wiki/Main_Page',
'https://m.en.wikipedia.org/wiki/Main_Page'])
assert actual.count() == 1
assert actual.first() == document2
def test_expand_uri_document_uris(self, db_session):
document = Document(document_uris=[
DocumentURI(uri='http://foo.com/', claimant='http://bar.com'),
DocumentURI(uri='http://bar.com/', claimant='http://bar.com'),
])
db_session.add(document)
db_session.flush()
assert storage.expand_uri(db_session, 'http://foo.com/') == [
'http://foo.com/', 'http://bar.com/'
]
def test_document_find_or_create_by_uris():
document = Document()
docuri1 = DocumentURI(
claimant='https://en.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page',
document=document)
docuri2 = DocumentURI(
claimant='https://en.wikipedia.org/wiki/http/en.m.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page',
document=document)
db.Session.add(docuri1)
db.Session.add(docuri2)
db.Session.flush()
actual = Document.find_or_create_by_uris(db.Session,
'https://en.wikipedia.org/wiki/Main_Page',
['https://en.wikipedia.org/wiki/http/en.m.wikipedia.org/wiki/Main_Page',
'https://m.en.wikipedia.org/wiki/Main_Page'])
assert actual.count() == 1
assert actual.first() == document
def merge_data(request):
master = Document(document_uris=[DocumentURI(
claimant='https://en.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page',
type='self-claim')],
meta=[DocumentMeta(
claimant='https://en.wikipedia.org/wiki/Main_Page',
type='title',
value='Wikipedia, the free encyclopedia')])
duplicate = Document(document_uris=[DocumentURI(
claimant='https://m.en.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page',
type='rel-canonical')],
meta=[DocumentMeta(
claimant='https://m.en.wikipedia.org/wiki/Main_Page',
type='title',
value='Wikipedia, the free encyclopedia')])
db.Session.add_all([master, duplicate])
db.Session.flush()
return (master, duplicate)
def test_og_document(render_app_html, annotation_document, document_title):
annotation = Annotation(id='123', userid='foo', target_uri='http://example.com')
document = Document()
annotation_document.return_value = document
document_title.return_value = 'WikiHow — How to Make a ☆Starmap☆'
render_app_html.return_value = '<html></html>'
request = _dummy_request()
main.annotation_page(annotation, request)
args, kwargs = render_app_html.call_args
test = lambda d: 'foo' in d['content'] and 'Starmap' in d['content']
assert any(test(d) for d in kwargs['extra']['meta_attrs'])
def test_expand_uri_postgres_document_uris(postgres_enabled):
request = DummyRequest(db=db.Session)
postgres_enabled.return_value = True
document = Document(document_uris=[
DocumentURI(uri='http://foo.com/', claimant='http://bar.com'),
DocumentURI(uri='http://bar.com/', claimant='http://bar.com'),
])
db.Session.add(document)
db.Session.flush()
assert storage.expand_uri(
request, 'http://foo.com/') == ['http://foo.com/', 'http://bar.com/']
def test_expand_uri_document_doesnt_expand_canonical_uris(
self, db_session):
document = Document(document_uris=[
DocumentURI(uri='http://foo.com/', claimant='http://example.com'),
DocumentURI(uri='http://bar.com/', claimant='http://example.com'),
DocumentURI(uri='http://example.com/',
type='rel-canonical',
claimant='http://example.com'),
])
db_session.add(document)
db_session.flush()
assert storage.expand_uri(
db_session, "http://example.com/") == ["http://example.com/"]
def test_document_find_or_create_by_uris_no_results():
document = Document()
docuri = DocumentURI(
claimant='https://en.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page',
document=document)
db.Session.add(docuri)
db.Session.flush()
documents = Document.find_or_create_by_uris(db.Session,
'https://en.wikipedia.org/wiki/Pluto',
['https://m.en.wikipedia.org/wiki/Pluto'])
assert documents.count() == 1
actual = documents.first()
assert isinstance(actual, Document)
assert len(actual.document_uris) == 1
docuri = actual.document_uris[0]
assert docuri.claimant == 'https://en.wikipedia.org/wiki/Pluto'
assert docuri.uri == 'https://en.wikipedia.org/wiki/Pluto'
assert docuri.type == 'self-claim'
def test_expand_uri_postgres_document_doesnt_expand_canonical_uris(
postgres_enabled):
request = DummyRequest(db=db.Session)
postgres_enabled.return_value = True
document = Document(document_uris=[
DocumentURI(uri='http://foo.com/', claimant='http://example.com'),
DocumentURI(uri='http://bar.com/', claimant='http://example.com'),
DocumentURI(uri='http://example.com/',
type='rel-canonical',
claimant='http://example.com'),
])
db.Session.add(document)
db.Session.flush()
assert storage.expand_uri(
request, "http://example.com/") == ["http://example.com/"]
def create_or_update_document_objects(es_ann):
es_doc = es_ann.document
if not es_doc:
return
uris = [u.uri for u in es_doc.document_uris]
documents = Document.find_or_create_by_uris(Session, es_ann.target_uri, uris,
created=es_doc.created,
updated=es_doc.updated)
if documents.count() > 1:
document = merge_documents(Session, documents, updated=es_doc.updated)
else:
document = documents.first()
document.updated = es_doc.updated
for uri_ in es_doc.document_uris:
create_or_update_document_uri(uri_, document)
for meta in es_doc.meta:
create_or_update_document_meta(meta, document)