def test_it_updates_an_existing_DocumentMeta_if_there_is_one(self, db_session):
claimant = 'http://example.com/claimant'
type_ = 'title'
value = 'the title'
document_ = document.Document()
created = yesterday()
updated = now()
document_meta = document.DocumentMeta(
claimant=claimant,
type=type_,
value=value,
document=document_,
created=created,
updated=updated,
)
db_session.add(document_meta)
new_updated = now()
document.create_or_update_document_meta(
session=db_session,
claimant=claimant,
type=type_,
value='new value',
document=document.Document(), # This should be ignored.
created=now(), # This should be ignored.
updated=new_updated,
)
assert document_meta.value == 'new value'
assert document_meta.updated == new_updated
assert document_meta.created == created, "It shouldn't update created"
assert document_meta.document == document_, (
"It shouldn't update document")
assert len(db_session.query(document.DocumentMeta).all()) == 1, (
"It shouldn't have added any new objects to the db")
def merge_data(self, db_session, request):
master = document.Document(
document_uris=[
document.DocumentURI(
claimant='https://en.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page',
type='self-claim')
],
meta=[
document.DocumentMeta(
claimant='https://en.wikipedia.org/wiki/Main_Page',
type='title',
value='Wikipedia, the free encyclopedia')
])
duplicate = document.Document(
document_uris=[
document.DocumentURI(
claimant='https://m.en.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page',
type='rel-canonical')
],
meta=[
document.DocumentMeta(
claimant='https://m.en.wikipedia.org/wiki/Main_Page',
type='title',
value='Wikipedia, the free encyclopedia')
])
db_session.add_all([master, duplicate])
db_session.flush()
return (master, duplicate)
def test_with_one_matching_Document(self, db_session):
# One Document with a non-matching DocumentURI pointing to it.
# find_by_uris() should not return this Document.
document1 = document.Document()
uri1 = 'https://de.wikipedia.org/wiki/Hauptseite'
document1.document_uris.append(
document.DocumentURI(claimant=uri1, uri=uri1))
# A second Document with one matching and one non-matching DocumentURI
# pointing to it. find_by_uris() should return this Document.
document2 = document.Document()
uri2 = 'https://en.wikipedia.org/wiki/Main_Page'
document2.document_uris.append(
document.DocumentURI(claimant=uri2, uri=uri2))
uri3 = 'https://en.wikipedia.org'
document2.document_uris.append(
document.DocumentURI(claimant=uri3, uri=uri2))
db_session.add_all([document1, document2])
db_session.flush()
actual = document.Document.find_by_uris(db_session, [
'https://en.wikipedia.org/wiki/Main_Page',
'https://m.en.wikipedia.org/wiki/Main_Page'])
assert actual.count() == 1
assert actual.first() == document2
def merge_data(self, db_session, request):
master = document.Document(
document_uris=[
document.DocumentURI(
claimant='https://en.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page',
type='self-claim')
],
meta=[
document.DocumentMeta(
claimant='https://en.wikipedia.org/wiki/Main_Page',
type='title',
value='Wikipedia, the free encyclopedia')
])
duplicate_1 = document.Document(
document_uris=[
document.DocumentURI(
claimant='https://m.en.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page',
type='rel-canonical')
],
meta=[
document.DocumentMeta(
claimant='https://m.en.wikipedia.org/wiki/Main_Page',
type='title',
value='Wikipedia, the free encyclopedia')
])
duplicate_2 = document.Document(
document_uris=[
document.DocumentURI(
claimant='https://en.wikipedia.org/wiki/Home',
uri='https://en.wikipedia.org/wiki/Main_Page',
type='rel-canonical')
],
meta=[
document.DocumentMeta(
claimant='https://en.wikipedia.org/wiki/Home',
type='title',
value='Wikipedia, the free encyclopedia')
])
db_session.add_all([master, duplicate_1, duplicate_2])
db_session.flush()
master_ann_1 = models.Annotation(userid='luke', document_id=master.id)
master_ann_2 = models.Annotation(userid='alice', document_id=master.id)
duplicate_1_ann_1 = models.Annotation(userid='lucy',
document_id=duplicate_1.id)
duplicate_1_ann_2 = models.Annotation(userid='bob',
document_id=duplicate_1.id)
duplicate_2_ann_1 = models.Annotation(userid='amy',
document_id=duplicate_2.id)
duplicate_2_ann_2 = models.Annotation(userid='dan',
document_id=duplicate_2.id)
db_session.add_all([
master_ann_1, master_ann_2, duplicate_1_ann_1, duplicate_1_ann_2,
duplicate_2_ann_1, duplicate_2_ann_2
])
return (master, duplicate_1, duplicate_2)
def test_with_one_existing_Document(self, db_session):
"""
When there's one matching Document it should return that Document.
When searching with two URIs that match two DocumentURIs that both
point to the same Document, it should return that Document.
"""
document_ = document.Document()
docuri1 = document.DocumentURI(
claimant='https://en.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page',
document=document_)
docuri2 = document.DocumentURI(
claimant='https://en.wikipedia.org/wiki/http/en.m.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page',
document=document_)
db_session.add(docuri1)
db_session.add(docuri2)
db_session.flush()
actual = document.Document.find_or_create_by_uris(db_session,
'https://en.wikipedia.org/wiki/Main_Page',
['https://en.wikipedia.org/wiki/http/en.m.wikipedia.org/wiki/Main_Page',
'https://m.en.wikipedia.org/wiki/Main_Page'])
assert actual.count() == 1
assert actual.first() == document_
def test_it_updates_the_existing_DocumentURI_if_there_is_one(self, db_session):
claimant = 'http://example.com/example_claimant.html'
uri = 'http://example.com/example_uri.html'
type_ = 'self-claim'
content_type = ''
document_ = document.Document()
created = yesterday()
updated = yesterday()
document_uri = document.DocumentURI(
claimant=claimant,
uri=uri,
type=type_,
content_type=content_type,
document=document_,
created=created,
updated=updated,
)
db_session.add(document_uri)
now_ = now()
document.create_or_update_document_uri(
session=db_session,
claimant=claimant,
uri=uri,
type=type_,
content_type=content_type,
document=document_,
created=now_,
updated=now_,
)
assert document_uri.created == created
assert document_uri.updated == now_
assert len(db_session.query(document.DocumentURI).all()) == 1, (
"It shouldn't have added any new objects to the db")
def test_with_no_existing_documents(self, db_session):
"""When there are no matching Documents it creates and returns one."""
document_ = document.Document()
docuri = document.DocumentURI(
claimant='https://en.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page',
document=document_)
db_session.add(docuri)
db_session.flush()
documents = document.Document.find_or_create_by_uris(
db_session,
'https://en.wikipedia.org/wiki/Pluto',
['https://m.en.wikipedia.org/wiki/Pluto'])
assert documents.count() == 1
actual = documents.first()
assert isinstance(actual, document.Document)
assert len(actual.document_uris) == 1
docuri = actual.document_uris[0]
assert docuri.claimant == 'https://en.wikipedia.org/wiki/Pluto'
assert docuri.uri == 'https://en.wikipedia.org/wiki/Pluto'
assert docuri.type == 'self-claim'
def test_content_type_defaults_to_empty_string(self, db_session):
document_uri = document.DocumentURI(claimant='http://www.example.com',
uri='http://www.example.com',
type='bar',
content_type=None,
document=document.Document())
db_session.add(document_uri)
db_session.flush()
assert document_uri.content_type == ''
def test_no_matches(self, db_session):
document_ = document.Document()
document_.document_uris.append(document.DocumentURI(
claimant='https://en.wikipedia.org/wiki/Main_Page',
uri='https://en.wikipedia.org/wiki/Main_Page'))
db_session.add(document_)
db_session.flush()
actual = document.Document.find_by_uris(
db_session, ['https://de.wikipedia.org/wiki/Hauptseite'])
assert actual.count() == 0
def test_it_returns_None_if_there_are_no_title_DocumentMetas(
self, db_session):
doc = document.Document()
document.DocumentMeta(type='other',
value='something',
document=doc,
claimant='http://example.com')
db_session.add(doc)
db_session.flush()
assert doc.title is None
def test_it_returns_the_value_of_the_one_title_DocumentMeta(
self, db_session):
"""When there's only one DocumentMeta it should return its title."""
doc = document.Document()
document.DocumentMeta(type='title',
value=['The Title'],
document=doc,
claimant='http://example.com')
db_session.add(doc)
db_session.flush()
assert doc.title == 'The Title'
def test_you_cannot_set_content_type_to_null(self, db_session):
document_uri = document.DocumentURI(claimant='http://www.example.com',
uri='http://www.example.com',
type='foo',
content_type='bar',
document=document.Document())
db_session.add(document_uri)
db_session.flush()
document_uri.content_type = None
with pytest.raises(sa.exc.IntegrityError):
db_session.flush()
def test_you_cannot_add_duplicate_document_uris(self, db_session):
"""
You can't add duplicate DocumentURI's to the database.
You can't add DocumentURI's with the same claimant, uri, type and
content_type, even if they have different documents.
"""
db_session.add_all([
document.DocumentURI(claimant='http://www.example.com',
uri='http://www.example.com',
type='foo',
content_type='bar',
document=document.Document()),
document.DocumentURI(claimant='http://www.example.com',
uri='http://www.example.com',
type='foo',
content_type='bar',
document=document.Document())
])
with pytest.raises(sa.exc.IntegrityError):
db_session.commit()
def test_it_returns_the_value_of_the_first_title_DocumentMeta(
self, db_session):
doc = document.Document()
document.DocumentMeta(type='title',
value=['The US Title'],
document=doc,
claimant='http://example.com')
document.DocumentMeta(type='title',
value=['The UK Title'],
document=doc,
claimant='http://example.co.uk')
db_session.add(doc)
db_session.flush()
assert doc.title == 'The US Title'
def test_it_skips_denormalizing_http_s_uri_to_document(self, db_session):
document_ = document.Document(web_uri='http://example.com/first_uri.html')
db_session.add(document_)
document.create_or_update_document_uri(
session=db_session,
claimant='http://example.com/example_claimant.html',
uri='http://example.com/second_uri.html',
type='self-claim',
content_type='',
document=document_,
created=now(),
updated=now(),
)
document_ = db_session.query(document.Document).get(document_.id)
assert document_.web_uri == 'http://example.com/first_uri.html'
def test_raises_retryable_error_when_flush_fails(self, db_session, monkeypatch):
document_ = document.Document()
def err():
raise sa.exc.IntegrityError(None, None, None)
monkeypatch.setattr(db_session, 'flush', err)
with pytest.raises(transaction.interfaces.TransientError):
with db_session.no_autoflush: # prevent premature IntegrityError
document.create_or_update_document_meta(
session=db_session,
claimant='http://example.com',
type='title',
value='My Title',
document=document_,
created=now(),
updated=now(),
)
def test_it_creates_a_new_DocumentURI_if_there_is_no_existing_one(
self, db_session):
claimant = 'http://example.com/example_claimant.html'
uri = 'http://example.com/example_uri.html'
type_ = 'self-claim'
content_type = ''
document_ = document.Document()
created = yesterday()
updated = yesterday()
# Add one non-matching DocumentURI to the database.
db_session.add(
document.DocumentURI(
claimant=claimant,
uri=uri,
type=type_,
# Different content_type means this DocumentURI should not match
# the query.
content_type='different',
document=document_,
created=created,
updated=updated,
))
document.create_or_update_document_uri(
session=db_session,
claimant=claimant,
uri=uri,
type=type_,
content_type=content_type,
document=document_,
created=now(),
updated=now(),
)
document_uri = db_session.query(document.DocumentURI).all()[-1]
assert document_uri.claimant == claimant
assert document_uri.uri == uri
assert document_uri.type == type_
assert document_uri.content_type == content_type
assert document_uri.document == document_
assert document_uri.created > created
assert document_uri.updated > updated
def test_it_denormalizes_https_uri_to_document_when_empty(self, db_session):
uri = 'https://example.com/example_uri.html'
document_ = document.Document(web_uri='')
db_session.add(document_)
document.create_or_update_document_uri(
session=db_session,
claimant='http://example.com/example_claimant.html',
uri=uri,
type='self-claim',
content_type='',
document=document_,
created=now(),
updated=now(),
)
document_ = db_session.query(document.Document).get(document_.id)
assert document_.web_uri == uri
def test_it_skips_denormalizing_title_to_document_when_already_set(self, db_session):
claimant = 'http://example.com/claimant'
type_ = 'title'
value = ['the title']
document_ = document.Document(title='foobar')
created = yesterday()
updated = now()
db_session.add(document_)
document.create_or_update_document_meta(
session=db_session,
claimant=claimant,
type=type_,
value=value,
document=document_,
created=created,
updated=updated,
)
document_ = db_session.query(document.Document).get(document_.id)
assert document_.title == 'foobar'
def test_it_creates_a_new_DocumentMeta_if_there_is_no_existing_one(
self, db_session):
claimant = 'http://example.com/claimant'
type_ = 'title'
value = 'the title'
document_ = document.Document()
created = yesterday()
updated = now()
# Add one non-matching DocumentMeta to the database.
# This should be ignored.
db_session.add(
document.DocumentMeta(
claimant=claimant,
# Different type means this should not match the query.
type='different',
value=value,
document=document_,
created=created,
updated=updated,
))
document.create_or_update_document_meta(
session=db_session,
claimant=claimant,
type=type_,
value=value,
document=document_,
created=created,
updated=updated,
)
document_meta = db_session.query(document.DocumentMeta).all()[-1]
assert document_meta.claimant == claimant
assert document_meta.type == type_
assert document_meta.value == value
assert document_meta.document == document_
assert document_meta.created == created
assert document_meta.updated == updated
def mock_document():
"""Return a mock Document object."""
return mock.Mock(spec=document.Document())
