def check_abnormal_remove_data_from_exist_and_cache(self, op):
db = ExistDB()
c = Chunk(data="<div/>", is_normal=False)
c.clean()
cache.delete(c.c_hash)
method = getattr(c, op)
db.removeCollection(self.chunk_collection_path, True)
db.removeCollection(self.display_collection_path, True)
c.save()
keys = [c.display_key(kind) for kind in self.prepare_kinds]
for kind in self.prepare_kinds:
self.assertIsNone(cache.get(c.display_key(kind)))
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
0)
self.assertEqual(len(list_collection(db,
self.display_collection_path)),
0)
with mock.patch('lexicography.models.ExistDB.removeDocument') as \
remove_mock:
method()
self.assertEqual(remove_mock.call_count, 0)
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
0)
self.assertEqual(len(list_collection(db,
self.display_collection_path)),
0)
for key in keys:
self.assertIsNone(cache.get(key))
def check_abnormal_remove_data_from_exist_and_cache(self, op):
db = ExistDB()
c = Chunk(data="<div/>", is_normal=False)
c.clean()
cache.delete(c.c_hash)
method = getattr(c, op)
db.removeCollection(self.chunk_collection_path, True)
db.removeCollection(self.display_collection_path, True)
c.save()
keys = [c.display_key(kind) for kind in self.prepare_kinds]
for kind in self.prepare_kinds:
self.assertIsNone(cache.get(c.display_key(kind)))
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
0)
self.assertEqual(
len(list_collection(db, self.display_collection_path)), 0)
with mock.patch('lexicography.models.ExistDB.removeDocument') as \
remove_mock:
method()
self.assertEqual(remove_mock.call_count, 0)
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
0)
self.assertEqual(
len(list_collection(db, self.display_collection_path)), 0)
for key in keys:
self.assertIsNone(cache.get(key))
def setUp(self):
self.foo = foo = user_model.objects.create(username="******",
password="******")
scribe = Group.objects.get(name='scribe')
foo.groups.add(scribe)
cache.clear()
db = ExistDB()
db.removeCollection(self.chunk_collection_path, True)
db.removeCollection(self.display_collection_path, True)
return super(ChunkTransactionTestCase, self).setUp()
def setUp(self):
self.foo = foo = user_model.objects.create(
username="******", password="******")
scribe = Group.objects.get(name='scribe')
foo.groups.add(scribe)
cache.clear()
db = ExistDB()
db.removeCollection(self.chunk_collection_path, True)
db.removeCollection(self.display_collection_path, True)
return super(ChunkTransactionTestCase, self).setUp()
def check_deletes_documents(self, op, collection, *args):
c = Chunk(data="<div/>", is_normal=True)
c.save()
entry = self.make_reachable(c)
# If it does not have metadata yet, that's fine.
try:
c.chunkmetadata.delete()
except ChunkMetadata.DoesNotExist:
pass
# We have to delete the collection because merely saving the
# chunk causes it to be synced, but this is not what we are
# testing here. We want to make sure that calling
# op will perform the sync.
db = ExistDB()
db.removeCollection(collection, True)
self.assertEqual(len(list_collection(db, collection)), 0)
op = getattr(self.manager, op)
op(*args)
self.assertEqual(len(list_collection(db, collection)), 1)
# Make sure our chunk was not collected.
self.assertEqual(self.manager.count(), 1)
# Now we delete the chunk in SQL because we do not want the
# ``delete`` method to be called, as it would take care of
# removing the document itself. (And yes, we do interpolate
# the table name. This is safe as ``Entry._meta.db_table`` is
# a value under our control.)
with connection.cursor() as cursor:
cr = entry.latest
cursor.execute(
"DELETE FROM {} WHERE id = %s".format(entry._meta.db_table),
[entry.pk])
# We have to do this ourselves because Django's cascading
# delete is implemented at the ORM level, not the database
# level.
cursor.execute(
"DELETE FROM {} WHERE id = %s".format(cr._meta.db_table),
[cr.pk])
# Check that no collection or syncing has occurred.
self.assertEqual(self.manager.count(), 1)
self.assertEqual(len(list_collection(db, collection)), 1)
op(*args)
# Make sure our chunk was collected.
self.assertEqual(self.manager.count(), 0)
self.assertEqual(len(list_collection(db, collection)), 0)
def check_deletes_documents(self, op, collection, *args):
c = Chunk(data="<div/>", is_normal=True)
c.save()
entry = self.make_reachable(c)
# If it does not have metadata yet, that's fine.
try:
c.chunkmetadata.delete()
except ChunkMetadata.DoesNotExist:
pass
# We have to delete the collection because merely saving the
# chunk causes it to be synced, but this is not what we are
# testing here. We want to make sure that calling
# op will perform the sync.
db = ExistDB()
db.removeCollection(collection, True)
self.assertEqual(len(list_collection(db, collection)), 0)
op = getattr(self.manager, op)
op(*args)
self.assertEqual(len(list_collection(db, collection)), 1)
# Make sure our chunk was not collected.
self.assertEqual(self.manager.count(), 1)
# Now we delete the chunk in SQL because we do not want the
# ``delete`` method to be called, as it would take care of
# removing the document itself. (And yes, we do interpolate
# the table name. This is safe as ``Entry._meta.db_table`` is
# a value under our control.)
with connection.cursor() as cursor:
cr = entry.latest
cursor.execute(
"DELETE FROM {} WHERE id = %s".format(entry._meta.db_table),
[entry.pk])
# We have to do this ourselves because Django's cascading
# delete is implemented at the ORM level, not the database
# level.
cursor.execute(
"DELETE FROM {} WHERE id = %s".format(cr._meta.db_table),
[cr.pk])
# Check that no collection or syncing has occurred.
self.assertEqual(self.manager.count(), 1)
self.assertEqual(len(list_collection(db, collection)), 1)
op(*args)
# Make sure our chunk was collected.
self.assertEqual(self.manager.count(), 0)
self.assertEqual(len(list_collection(db, collection)), 0)
def check_skip_abnormal_chunks(self, op, collection, *args):
c = Chunk(data="", is_normal=False)
c.save()
# We have to delete the collection because merely saving the
# chunk causes it to be synced, but this is not what we are
# testing here. We want to make sure that calling
# sync_with_exist will perform the sync.
db = ExistDB()
db.removeCollection(collection, True)
self.assertEqual(len(list_collection(db, collection)), 0)
getattr(c, op)(*args)
self.assertEqual(len(list_collection(db, collection)), 0)
def check_skip_abnormal_chunks(self, op, collection, *args):
c = Chunk(data="", is_normal=False)
c.save()
# We have to delete the collection because merely saving the
# chunk causes it to be synced, but this is not what we are
# testing here. We want to make sure that calling
# sync_with_exist will perform the sync.
db = ExistDB()
db.removeCollection(collection, True)
self.assertEqual(len(list_collection(db, collection)), 0)
getattr(c, op)(*args)
self.assertEqual(len(list_collection(db, collection)), 0)
def test_no_exist_document(self):
"""
When the exist document is missing, raise an error. We want an
error because it indicates something really broken about our
internal state. We should never have metadata without a
corresponding XML file.
"""
cr = ChangeRecord.objects.get(pk=1)
chunk = cr.c_hash
self.assertIsNotNone(cache.get(chunk.display_key("xml")))
cache.clear()
db = ExistDB()
db.removeCollection(get_collection_path("display"), True)
with self.assertRaises(ExistDBException):
tasks.fetch_xml(chunk.c_hash)
def test_sync_handles_overwrites(self):
"""
``sync_with_exist`` will not overwrite documents already in eXist.
"""
db = ExistDB()
db.removeCollection(self.chunk_collection_path, True)
c = Chunk(data="<div/>", is_normal=True)
c.save()
c.sync_with_exist()
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
1)
with mock.patch('lexicography.models.ExistDB.load') as load_mock:
c.sync_with_exist()
self.assertEqual(load_mock.call_count, 0,
"load should not have been called!")
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
1)
def test_sync_handles_overwrites(self):
"""
``sync_with_exist`` will not overwrite documents already in eXist.
"""
db = ExistDB()
db.removeCollection(self.chunk_collection_path, True)
c = Chunk(data="<div/>", is_normal=True)
c.save()
c.sync_with_exist()
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
1)
with mock.patch('lexicography.models.ExistDB.load') as load_mock:
c.sync_with_exist()
self.assertEqual(load_mock.call_count, 0,
"load should not have been called!")
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
1)
def check_remove_data_from_exist_and_cache(self, op):
"""
Check that invoking ``op`` will remove the data from the eXist
database and the cache.
"""
db = ExistDB()
c = Chunk(data="<div/>", is_normal=True)
c.clean()
method = op if callable(op) else getattr(c, op)
cache.delete(c.c_hash)
db.removeCollection(self.chunk_collection_path, True)
db.removeCollection(self.display_collection_path, True)
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
0)
self.assertEqual(len(list_collection(db,
self.display_collection_path)),
0)
for kind in self.prepare_kinds:
self.assertIsNone(cache.get(c.display_key(kind)))
c.save()
c._create_cached_data()
keys = [c.display_key(kind) for kind in self.prepare_kinds]
# Only the "xml" data is created on save.
self.assertIsNotNone(cache.get(c.display_key("xml")))
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
1)
self.assertEqual(len(list_collection(db,
self.display_collection_path)),
1)
method()
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
0)
self.assertEqual(len(list_collection(db,
self.display_collection_path)),
0)
for key in keys:
self.assertIsNone(cache.get(key))
def __call__(self, command, _options):
"""
Load initial data into a new database. This is necessary for BTW
to run.
"""
assert_running()
from django.utils import translation
translation.activate('en-us')
db = ExistDB()
chunk_collection_path = get_collection_path("chunks")
if db.hasCollection(chunk_collection_path):
db.removeCollection(chunk_collection_path)
Chunk.objects.sync_with_exist()
display_path = get_collection_path("display")
if db.hasCollection(display_path):
db.removeCollection(display_path)
Chunk.objects.prepare("xml", True)
def __call__(self, command, _options):
"""
Load initial data into a new database. This is necessary for BTW
to run.
"""
assert_running()
from django.utils import translation
translation.activate('en-us')
db = ExistDB()
chunk_collection_path = get_collection_path("chunks")
if db.hasCollection(chunk_collection_path):
db.removeCollection(chunk_collection_path)
Chunk.objects.sync_with_exist()
display_path = get_collection_path("display")
if db.hasCollection(display_path):
db.removeCollection(display_path)
Chunk.objects.prepare("xml", include_unpublished=False)
def check_remove_data_from_exist_and_cache(self, op):
"""
Check that invoking ``op`` will remove the data from the eXist
database and the cache.
"""
db = ExistDB()
c = Chunk(data="<div/>", is_normal=True)
c.clean()
method = op if isinstance(op, Callable) else getattr(c, op)
cache.delete(c.c_hash)
db.removeCollection(self.chunk_collection_path, True)
db.removeCollection(self.display_collection_path, True)
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
0)
self.assertEqual(
len(list_collection(db, self.display_collection_path)), 0)
for kind in self.prepare_kinds:
self.assertIsNone(cache.get(c.display_key(kind)))
c.save()
c._create_cached_data()
keys = [c.display_key(kind) for kind in self.prepare_kinds]
# Only the "xml" data is created on save.
self.assertIsNotNone(cache.get(c.display_key("xml")))
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
1)
self.assertEqual(
len(list_collection(db, self.display_collection_path)), 1)
method()
self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
0)
self.assertEqual(
len(list_collection(db, self.display_collection_path)), 0)
for key in keys:
self.assertIsNone(cache.get(key))
def check_syncs_normal_chunks(self, op, collection, *args):
c = Chunk(data="<div/>", is_normal=True)
c.save()
self.make_reachable(c)
# If it does not have metadata yet, that's fine.
try:
c.chunkmetadata.delete()
except ChunkMetadata.DoesNotExist:
pass
# We have to delete the collection because merely saving the
# chunk causes it to be synced, but this is not what we are
# testing here. We want to make sure that calling
# sync_with_exist will perform the sync.
db = ExistDB()
db.removeCollection(collection, True)
self.assertEqual(len(list_collection(db, collection)), 0)
getattr(self.manager, op)(*args)
self.assertEqual(len(list_collection(db, collection)), 1)
# Make sure our chunk was not collected.
self.assertEqual(self.manager.count(), 1)
def check_syncs_normal_chunks(self, op, collection, *args):
c = Chunk(data="<div/>", is_normal=True)
c.save()
self.make_reachable(c)
# If it does not have metadata yet, that's fine.
try:
c.chunkmetadata.delete()
except ChunkMetadata.DoesNotExist:
pass
# We have to delete the collection because merely saving the
# chunk causes it to be synced, but this is not what we are
# testing here. We want to make sure that calling
# sync_with_exist will perform the sync.
db = ExistDB()
db.removeCollection(collection, True)
self.assertEqual(len(list_collection(db, collection)), 0)
getattr(self.manager, op)(*args)
self.assertEqual(len(list_collection(db, collection)), 1)
# Make sure our chunk was not collected.
self.assertEqual(self.manager.count(), 1)