Example #1
0
    def check_abnormal_remove_data_from_exist_and_cache(self, op):
        db = ExistDB()
        c = Chunk(data="<div/>", is_normal=False)
        c.clean()
        cache.delete(c.c_hash)
        method = getattr(c, op)
        db.removeCollection(self.chunk_collection_path, True)
        db.removeCollection(self.display_collection_path, True)

        c.save()
        keys = [c.display_key(kind) for kind in self.prepare_kinds]

        for kind in self.prepare_kinds:
            self.assertIsNone(cache.get(c.display_key(kind)))
        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         0)
        self.assertEqual(len(list_collection(db,
                                             self.display_collection_path)),
                         0)

        with mock.patch('lexicography.models.ExistDB.removeDocument') as \
                remove_mock:
            method()
            self.assertEqual(remove_mock.call_count, 0)

        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         0)
        self.assertEqual(len(list_collection(db,
                                             self.display_collection_path)),
                         0)
        for key in keys:
            self.assertIsNone(cache.get(key))
Example #2
0
    def check_abnormal_remove_data_from_exist_and_cache(self, op):
        db = ExistDB()
        c = Chunk(data="<div/>", is_normal=False)
        c.clean()
        cache.delete(c.c_hash)
        method = getattr(c, op)
        db.removeCollection(self.chunk_collection_path, True)
        db.removeCollection(self.display_collection_path, True)

        c.save()
        keys = [c.display_key(kind) for kind in self.prepare_kinds]

        for kind in self.prepare_kinds:
            self.assertIsNone(cache.get(c.display_key(kind)))
        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         0)
        self.assertEqual(
            len(list_collection(db, self.display_collection_path)), 0)

        with mock.patch('lexicography.models.ExistDB.removeDocument') as \
                remove_mock:
            method()
            self.assertEqual(remove_mock.call_count, 0)

        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         0)
        self.assertEqual(
            len(list_collection(db, self.display_collection_path)), 0)
        for key in keys:
            self.assertIsNone(cache.get(key))
Example #3
0
 def setUp(self):
     self.foo = foo = user_model.objects.create(username="******",
                                                password="******")
     scribe = Group.objects.get(name='scribe')
     foo.groups.add(scribe)
     cache.clear()
     db = ExistDB()
     db.removeCollection(self.chunk_collection_path, True)
     db.removeCollection(self.display_collection_path, True)
     return super(ChunkTransactionTestCase, self).setUp()
Example #4
0
 def setUp(self):
     self.foo = foo = user_model.objects.create(
         username="******", password="******")
     scribe = Group.objects.get(name='scribe')
     foo.groups.add(scribe)
     cache.clear()
     db = ExistDB()
     db.removeCollection(self.chunk_collection_path, True)
     db.removeCollection(self.display_collection_path, True)
     return super(ChunkTransactionTestCase, self).setUp()
Example #5
0
    def check_deletes_documents(self, op, collection, *args):
        c = Chunk(data="<div/>", is_normal=True)
        c.save()
        entry = self.make_reachable(c)

        # If it does not have metadata yet, that's fine.
        try:
            c.chunkmetadata.delete()
        except ChunkMetadata.DoesNotExist:
            pass

        # We have to delete the collection because merely saving the
        # chunk causes it to be synced, but this is not what we are
        # testing here. We want to make sure that calling
        # op will perform the sync.
        db = ExistDB()
        db.removeCollection(collection, True)
        self.assertEqual(len(list_collection(db, collection)), 0)

        op = getattr(self.manager, op)
        op(*args)

        self.assertEqual(len(list_collection(db, collection)), 1)

        # Make sure our chunk was not collected.
        self.assertEqual(self.manager.count(), 1)

        # Now we delete the chunk in SQL because we do not want the
        # ``delete`` method to be called, as it would take care of
        # removing the document itself. (And yes, we do interpolate
        # the table name. This is safe as ``Entry._meta.db_table`` is
        # a value under our control.)
        with connection.cursor() as cursor:
            cr = entry.latest
            cursor.execute(
                "DELETE FROM {} WHERE id = %s".format(entry._meta.db_table),
                [entry.pk])
            # We have to do this ourselves because Django's cascading
            # delete is implemented at the ORM level, not the database
            # level.
            cursor.execute(
                "DELETE FROM {} WHERE id = %s".format(cr._meta.db_table),
                [cr.pk])

        # Check that no collection or syncing has occurred.
        self.assertEqual(self.manager.count(), 1)
        self.assertEqual(len(list_collection(db, collection)), 1)

        op(*args)

        # Make sure our chunk was collected.
        self.assertEqual(self.manager.count(), 0)
        self.assertEqual(len(list_collection(db, collection)), 0)
Example #6
0
    def check_deletes_documents(self, op, collection, *args):
        c = Chunk(data="<div/>", is_normal=True)
        c.save()
        entry = self.make_reachable(c)

        # If it does not have metadata yet, that's fine.
        try:
            c.chunkmetadata.delete()
        except ChunkMetadata.DoesNotExist:
            pass

        # We have to delete the collection because merely saving the
        # chunk causes it to be synced, but this is not what we are
        # testing here. We want to make sure that calling
        # op will perform the sync.
        db = ExistDB()
        db.removeCollection(collection, True)
        self.assertEqual(len(list_collection(db, collection)), 0)

        op = getattr(self.manager, op)
        op(*args)

        self.assertEqual(len(list_collection(db, collection)), 1)

        # Make sure our chunk was not collected.
        self.assertEqual(self.manager.count(), 1)

        # Now we delete the chunk in SQL because we do not want the
        # ``delete`` method to be called, as it would take care of
        # removing the document itself. (And yes, we do interpolate
        # the table name. This is safe as ``Entry._meta.db_table`` is
        # a value under our control.)
        with connection.cursor() as cursor:
            cr = entry.latest
            cursor.execute(
                "DELETE FROM {} WHERE id = %s".format(entry._meta.db_table),
                [entry.pk])
            # We have to do this ourselves because Django's cascading
            # delete is implemented at the ORM level, not the database
            # level.
            cursor.execute(
                "DELETE FROM {} WHERE id = %s".format(cr._meta.db_table),
                [cr.pk])

        # Check that no collection or syncing has occurred.
        self.assertEqual(self.manager.count(), 1)
        self.assertEqual(len(list_collection(db, collection)), 1)

        op(*args)

        # Make sure our chunk was collected.
        self.assertEqual(self.manager.count(), 0)
        self.assertEqual(len(list_collection(db, collection)), 0)
Example #7
0
    def check_skip_abnormal_chunks(self, op, collection, *args):
        c = Chunk(data="", is_normal=False)
        c.save()

        # We have to delete the collection because merely saving the
        # chunk causes it to be synced, but this is not what we are
        # testing here. We want to make sure that calling
        # sync_with_exist will perform the sync.
        db = ExistDB()
        db.removeCollection(collection, True)
        self.assertEqual(len(list_collection(db, collection)), 0)

        getattr(c, op)(*args)

        self.assertEqual(len(list_collection(db, collection)), 0)
Example #8
0
    def check_skip_abnormal_chunks(self, op, collection, *args):
        c = Chunk(data="", is_normal=False)
        c.save()

        # We have to delete the collection because merely saving the
        # chunk causes it to be synced, but this is not what we are
        # testing here. We want to make sure that calling
        # sync_with_exist will perform the sync.
        db = ExistDB()
        db.removeCollection(collection, True)
        self.assertEqual(len(list_collection(db, collection)), 0)

        getattr(c, op)(*args)

        self.assertEqual(len(list_collection(db, collection)), 0)
Example #9
0
    def test_no_exist_document(self):
        """
        When the exist document is missing, raise an error. We want an
        error because it indicates something really broken about our
        internal state. We should never have metadata without a
        corresponding XML file.
        """
        cr = ChangeRecord.objects.get(pk=1)
        chunk = cr.c_hash
        self.assertIsNotNone(cache.get(chunk.display_key("xml")))

        cache.clear()
        db = ExistDB()
        db.removeCollection(get_collection_path("display"), True)

        with self.assertRaises(ExistDBException):
            tasks.fetch_xml(chunk.c_hash)
Example #10
0
    def test_sync_handles_overwrites(self):
        """
        ``sync_with_exist`` will not overwrite documents already in eXist.
        """
        db = ExistDB()
        db.removeCollection(self.chunk_collection_path, True)
        c = Chunk(data="<div/>", is_normal=True)
        c.save()
        c.sync_with_exist()

        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         1)

        with mock.patch('lexicography.models.ExistDB.load') as load_mock:
            c.sync_with_exist()
            self.assertEqual(load_mock.call_count, 0,
                             "load should not have been called!")

        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         1)
Example #11
0
    def test_sync_handles_overwrites(self):
        """
        ``sync_with_exist`` will not overwrite documents already in eXist.
        """
        db = ExistDB()
        db.removeCollection(self.chunk_collection_path, True)
        c = Chunk(data="<div/>", is_normal=True)
        c.save()
        c.sync_with_exist()

        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         1)

        with mock.patch('lexicography.models.ExistDB.load') as load_mock:
            c.sync_with_exist()
            self.assertEqual(load_mock.call_count, 0,
                             "load should not have been called!")

        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         1)
Example #12
0
    def check_remove_data_from_exist_and_cache(self, op):
        """
        Check that invoking ``op`` will remove the data from the eXist
        database and the cache.
        """
        db = ExistDB()
        c = Chunk(data="<div/>", is_normal=True)
        c.clean()
        method = op if callable(op) else getattr(c, op)
        cache.delete(c.c_hash)
        db.removeCollection(self.chunk_collection_path, True)
        db.removeCollection(self.display_collection_path, True)
        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         0)
        self.assertEqual(len(list_collection(db,
                                             self.display_collection_path)),
                         0)

        for kind in self.prepare_kinds:
            self.assertIsNone(cache.get(c.display_key(kind)))

        c.save()
        c._create_cached_data()
        keys = [c.display_key(kind) for kind in self.prepare_kinds]

        # Only the "xml" data is created on save.
        self.assertIsNotNone(cache.get(c.display_key("xml")))
        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         1)
        self.assertEqual(len(list_collection(db,
                                             self.display_collection_path)),
                         1)

        method()
        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         0)
        self.assertEqual(len(list_collection(db,
                                             self.display_collection_path)),
                         0)
        for key in keys:
            self.assertIsNone(cache.get(key))
Example #13
0
    def __call__(self, command, _options):
        """
        Load initial data into a new database. This is necessary for BTW
        to run.
        """
        assert_running()

        from django.utils import translation
        translation.activate('en-us')

        db = ExistDB()
        chunk_collection_path = get_collection_path("chunks")

        if db.hasCollection(chunk_collection_path):
            db.removeCollection(chunk_collection_path)

        Chunk.objects.sync_with_exist()

        display_path = get_collection_path("display")
        if db.hasCollection(display_path):
            db.removeCollection(display_path)
        Chunk.objects.prepare("xml", True)
Example #14
0
    def __call__(self, command, _options):
        """
        Load initial data into a new database. This is necessary for BTW
        to run.
        """
        assert_running()

        from django.utils import translation
        translation.activate('en-us')

        db = ExistDB()
        chunk_collection_path = get_collection_path("chunks")

        if db.hasCollection(chunk_collection_path):
            db.removeCollection(chunk_collection_path)

        Chunk.objects.sync_with_exist()

        display_path = get_collection_path("display")
        if db.hasCollection(display_path):
            db.removeCollection(display_path)
        Chunk.objects.prepare("xml", include_unpublished=False)
Example #15
0
    def check_remove_data_from_exist_and_cache(self, op):
        """
        Check that invoking ``op`` will remove the data from the eXist
        database and the cache.
        """
        db = ExistDB()
        c = Chunk(data="<div/>", is_normal=True)
        c.clean()
        method = op if isinstance(op, Callable) else getattr(c, op)
        cache.delete(c.c_hash)
        db.removeCollection(self.chunk_collection_path, True)
        db.removeCollection(self.display_collection_path, True)
        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         0)
        self.assertEqual(
            len(list_collection(db, self.display_collection_path)), 0)

        for kind in self.prepare_kinds:
            self.assertIsNone(cache.get(c.display_key(kind)))

        c.save()
        c._create_cached_data()
        keys = [c.display_key(kind) for kind in self.prepare_kinds]

        # Only the "xml" data is created on save.
        self.assertIsNotNone(cache.get(c.display_key("xml")))
        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         1)
        self.assertEqual(
            len(list_collection(db, self.display_collection_path)), 1)

        method()
        self.assertEqual(len(list_collection(db, self.chunk_collection_path)),
                         0)
        self.assertEqual(
            len(list_collection(db, self.display_collection_path)), 0)
        for key in keys:
            self.assertIsNone(cache.get(key))
Example #16
0
    def check_syncs_normal_chunks(self, op, collection, *args):
        c = Chunk(data="<div/>", is_normal=True)
        c.save()
        self.make_reachable(c)
        # If it does not have metadata yet, that's fine.
        try:
            c.chunkmetadata.delete()
        except ChunkMetadata.DoesNotExist:
            pass

        # We have to delete the collection because merely saving the
        # chunk causes it to be synced, but this is not what we are
        # testing here. We want to make sure that calling
        # sync_with_exist will perform the sync.
        db = ExistDB()
        db.removeCollection(collection, True)
        self.assertEqual(len(list_collection(db, collection)), 0)

        getattr(self.manager, op)(*args)

        self.assertEqual(len(list_collection(db, collection)), 1)

        # Make sure our chunk was not collected.
        self.assertEqual(self.manager.count(), 1)
Example #17
0
    def check_syncs_normal_chunks(self, op, collection, *args):
        c = Chunk(data="<div/>", is_normal=True)
        c.save()
        self.make_reachable(c)
        # If it does not have metadata yet, that's fine.
        try:
            c.chunkmetadata.delete()
        except ChunkMetadata.DoesNotExist:
            pass

        # We have to delete the collection because merely saving the
        # chunk causes it to be synced, but this is not what we are
        # testing here. We want to make sure that calling
        # sync_with_exist will perform the sync.
        db = ExistDB()
        db.removeCollection(collection, True)
        self.assertEqual(len(list_collection(db, collection)), 0)

        getattr(self.manager, op)(*args)

        self.assertEqual(len(list_collection(db, collection)), 1)

        # Make sure our chunk was not collected.
        self.assertEqual(self.manager.count(), 1)