def __call__(self, command, _options): assert_running() db = get_admin_db() for collection in [settings.EXISTDB_ROOT_COLLECTION, get_collection_path("chunks"), get_collection_path("display")]: db.createCollection(collection) db.server.setPermissions(collection, command.server_user, command.btw_group, 0770)
def test_exist_path(self): """ ``exist_path`` returns good values. """ c = Chunk(data="<div/>", is_normal=True) c.save() self.assertEqual(c.exist_path("chunks"), "/".join([get_collection_path("chunks"), c.c_hash])) self.assertEqual(c.exist_path("display"), "/".join([get_collection_path("display"), c.c_hash]))
def test_exist_path(self): """ ``exist_path`` returns good values. """ c = Chunk(data="<div/>", is_normal=True) c.save() self.assertEqual(c.exist_path("chunks"), "/".join([get_collection_path("chunks"), c.c_hash])) self.assertEqual(c.exist_path("display"), "/".join([get_collection_path("display"), c.c_hash]))
def sync_with_exist(self): self.collect() db = ExistDB() present = set() for chunk in self.filter(is_normal=True): chunk.sync_with_exist(db) present.add(chunk.c_hash) self._remove_absent(db, present, get_collection_path("chunks"))
def sync_with_exist(self): self.collect() db = ExistDB() present = set() for chunk in self.all_syncable_chunks(): chunk.sync_with_exist(db) present.add(chunk.c_hash) self._remove_absent(db, present, get_collection_path("chunks"))
def __call__(self, command, _options): """ Load the utilities into the database. This is necessary for BTW to run. """ util_path = get_collection_path("util") db = get_admin_db() if not db.hasCollection(util_path): command.create_collections([util_path]) db.query(xquery.format( "xmldb:store({db}, 'empty.xml', <doc/>)", db=util_path))
def __call__(self, command, _options): """ Load initial data into a new database. This is necessary for BTW to run. """ assert_running() from django.utils import translation translation.activate('en-us') db = ExistDB() chunk_collection_path = get_collection_path("chunks") if db.hasCollection(chunk_collection_path): db.removeCollection(chunk_collection_path) Chunk.objects.sync_with_exist() display_path = get_collection_path("display") if db.hasCollection(display_path): db.removeCollection(display_path) Chunk.objects.prepare("xml", True)
def prepare(self, kind, synchronous): if kind != "xml": raise ValueError("the manager only supports preparing XML data; " "future versions may support other kinds") self.collect() db = ExistDB() present = set() for chunk in self.filter(is_normal=True): chunk.prepare("xml", synchronous) present.add(chunk.c_hash) self._remove_absent(db, present, get_collection_path("display"))
def __call__(self, command, _options): """ Load initial data into a new database. This is necessary for BTW to run. """ assert_running() from django.utils import translation translation.activate('en-us') db = ExistDB() chunk_collection_path = get_collection_path("chunks") if db.hasCollection(chunk_collection_path): db.removeCollection(chunk_collection_path) Chunk.objects.sync_with_exist() display_path = get_collection_path("display") if db.hasCollection(display_path): db.removeCollection(display_path) Chunk.objects.prepare("xml", include_unpublished=False)
def prepare(self, kind, include_unpublished): if kind != "xml": raise ValueError("the manager only supports preparing XML data; " "future versions may support other kinds") self.collect() db = ExistDB() present = set() chunks = self.all_syncable_chunks() if not include_unpublished: chunks = chunks.filter(changerecord__published=True) for chunk in chunks: chunk.prepare("xml", True) present.add(chunk.c_hash) self._remove_absent(db, present, get_collection_path("display"))
def test_no_exist_document(self): """ When the exist document is missing, raise an error. We want an error because it indicates something really broken about our internal state. We should never have metadata without a corresponding XML file. """ cr = ChangeRecord.objects.get(pk=1) chunk = cr.c_hash self.assertIsNotNone(cache.get(chunk.display_key("xml"))) cache.clear() db = ExistDB() db.removeCollection(get_collection_path("display"), True) with self.assertRaises(ExistDBException): tasks.fetch_xml(chunk.c_hash)
def hashes_with_semantic_field(self, sf): """ Returns a set of chunk *hashes* that contain the semantic field requested. """ db = ExistDB() chunks = set() for query_chunk in query_iterator(db, xquery.format( """\ for $m in collection({db})//btw:sf[@ref = {path}] return util:document-name($m)""", db=get_collection_path("display"), path=sf)): for result in query_chunk.values: chunks.add(result) return chunks
def hashes_with_semantic_field(self, sf): """ Returns a set of chunk *hashes* that contain the semantic field requested. """ db = ExistDB() chunks = set() for query_chunk in query_iterator( db, xquery.format("""\ for $m in collection({db})//btw:sf[@ref = {path}] return util:document-name($m)""", db=get_collection_path("display"), path=sf)): for result in query_chunk.values: chunks.add(result) return chunks
def __call__(self, command, _options): assert_running() db = get_admin_db() collection = get_collection_path(None) db.removeCollectionIndex(collection)
def __call__(self, command, _options): assert_running() db = get_admin_db() collection = get_collection_path(None) db.loadCollectionIndex(collection, open(command.chunk_index, 'r')) db.reindexCollection(collection)
def __call__(self, command, _options): command.create_collections([settings.EXISTDB_ROOT_COLLECTION, get_collection_path("chunks"), get_collection_path("display")]) Loadutil()(command, _options)
def filter_queryset(self, qs): # pylint: disable=too-many-branches search_value = self.request.GET.get('search[value]', None) lemmata_only = self.request.GET.get('lemmata_only', "false") == \ "true" if usermod.can_author(self.request.user): publication_status = self.request.GET.get('publication_status', "published") search_all = self.request.GET.get('search_all', "false") == "true" if not search_all: # Remove deleted entries from the set. active = qs.filter(entry__in=Entry.objects.active_entries()) if publication_status == "published": active = active.filter(entry__latest_published=F('pk')) elif publication_status == "unpublished": active = active.filter(entry__latest=F('pk')) \ .exclude(entry__latest_published=F('pk')) elif publication_status == "both": active = active.filter(entry__latest=F('pk')) else: raise ValueError("unknown value for publication_status: " + publication_status) else: if publication_status == "published": active = qs.filter(published=True) elif publication_status == "unpublished": active = qs.filter(published=False) elif publication_status == "both": active = qs else: raise ValueError("unknown value for publication_status: " + publication_status) else: # If the user cannot author, then our queryset is already # reduced to what the user can see: the latest version of # published articles. active = qs if search_value: db = ExistDB() chunks = [] if lemmata_only: scope = "//btw:lemma" # When we do a lemma search, hits are not useful. hit = "" else: scope = "//btw:entry" hit = "{kwic:summarize($m, <config width='80'/>)}" for query_chunk in query_iterator( db, xquery.format( """\ import module namespace kwic="http://exist-db.org/xquery/kwic"; for $m in collection({db}){scope}[ft:query(., {search_text})] order by ft:score($m) descending return <result><doc>{doc}</doc><hit>{hit}</hit></result>""", db=get_collection_path("display"), scope=xquery.Verbatim(scope), doc=xquery.Verbatim("{util:document-name($m)}"), hit=xquery.Verbatim(hit), search_text=search_value)): for result in query_chunk.results: chunk = result[0].text # Content of <doc>. self.chunk_to_hits[chunk] = result[1] chunks.append(chunk) # We need to get the changerecords that pertain to these chunks. qs = active.filter(c_hash__in=set(chunks)) else: qs = active return qs
class ChunkTransactionTestCase(util.DisableMigrationsTransactionMixin, TransactionTestCase): chunk_collection_path = get_collection_path("chunks") display_collection_path = get_collection_path("display") prepare_kinds = Chunk.key_kinds def setUp(self): self.foo = foo = user_model.objects.create(username="******", password="******") scribe = Group.objects.get(name='scribe') foo.groups.add(scribe) cache.clear() db = ExistDB() db.removeCollection(self.chunk_collection_path, True) db.removeCollection(self.display_collection_path, True) return super(ChunkTransactionTestCase, self).setUp() def test_when_chunk_becomes_hidden_cached_data_is_cleared(self): """ When a ``Chunk`` becomes hidden, then its cached data is deleted. """ c = Chunk(data="<div/>", is_normal=True) c.save() keys = [c.display_key(kind) for kind in self.prepare_kinds] e = Entry() e.update(self.foo, "q", c, "foo", ChangeRecord.CREATE, ChangeRecord.MANUAL) db = ExistDB() self.assertIsNotNone(cache.get(c.display_key("xml"))) self.assertEqual(len(list_collection(db, self.chunk_collection_path)), 1) self.assertEqual( len(list_collection(db, self.display_collection_path)), 1) e.latest.hidden = True e.latest.save() for key in keys: self.assertIsNone(cache.get(key)) self.assertEqual(len(list_collection(db, self.chunk_collection_path)), 0) self.assertEqual( len(list_collection(db, self.display_collection_path)), 0) def test_when_chunk_becomes_shown_cached_data_is_created(self): """ When a ``Chunk`` becomes shown, then its cached data is created. """ c = Chunk(data="<div/>", is_normal=True) c.save() keys = [c.display_key(kind) for kind in self.prepare_kinds] e = Entry() e.update(self.foo, "q", c, "foo", ChangeRecord.CREATE, ChangeRecord.MANUAL) db = ExistDB() self.assertIsNotNone(cache.get(c.display_key("xml"))) self.assertEqual(len(list_collection(db, self.chunk_collection_path)), 1) self.assertEqual( len(list_collection(db, self.display_collection_path)), 1) e.latest.hidden = True e.latest.save() for key in keys: self.assertIsNone(cache.get(key)) self.assertEqual(len(list_collection(db, self.chunk_collection_path)), 0) self.assertEqual( len(list_collection(db, self.display_collection_path)), 0) e.update(self.foo, "q", c, "foo", ChangeRecord.CREATE, ChangeRecord.MANUAL) self.assertIsNotNone(cache.get(c.display_key("xml"))) self.assertEqual(len(list_collection(db, self.chunk_collection_path)), 1) self.assertEqual( len(list_collection(db, self.display_collection_path)), 1)
def __call__(self, command, _options): assert_running() db = get_admin_db() collection = get_collection_path(None) db.loadCollectionIndex(collection, open(command.chunk_index, 'r')) db.reindexCollection(collection)
class ChunkManagerSimpleTestCase(util.DisableMigrationsMixin, TestCase): chunk_collection_path = get_collection_path("chunks") display_collection_path = get_collection_path("display") @classmethod def setUpTestData(cls): super(ChunkManagerSimpleTestCase, cls).setUpTestData() cls.foo = user_model.objects.create(username="******", password="******") def setUp(self): self.manager = Chunk.objects return super(ChunkManagerSimpleTestCase, self).setUp() def make_reachable(self, chunk): # Make the chunk reachable e = Entry() e.update(self.foo, "q", chunk, "foo", ChangeRecord.CREATE, ChangeRecord.MANUAL) return e def list_chunk_collection(self): db = ExistDB() return list_collection(db, self.chunk_collection_path) def list_display_collection(self): db = ExistDB() return list_collection(db, self.chunk_collection_path) def check_collects(self, op, *args): self.assertEqual(self.manager.count(), 0) c = Chunk(data="", is_normal=False) c.save() self.assertEqual(self.manager.count(), 1) getattr(self.manager, op)(*args) self.assertEqual(self.manager.count(), 0) def test_collect_collects_unreachable(self): """ ``collect`` collects unreachable chunks. """ self.check_collects("collect") def test_collect_does_not_collect_reachable(self): """ Does not collect reachable chunks. """ self.assertEqual(self.manager.count(), 0) c = Chunk(data="", is_normal=False) c.save() self.make_reachable(c) self.assertEqual(self.manager.count(), 1) self.manager.collect() # Not collected! self.assertEqual(self.manager.count(), 1) def test_sync_collects(self): """ ``sync_with_exist`` causes a collection of unreachable chunks. """ self.check_collects("sync_with_exist") def check_skip_abnormal_chunks(self, op, collection, *args): c = Chunk(data="", is_normal=False) c.save() self.make_reachable(c) db = ExistDB() self.assertEqual(len(list_collection(db, collection)), 0) getattr(self.manager, op)(*args) self.assertEqual(len(list_collection(db, collection)), 0) # Make sure our chunk was not collected. self.assertEqual(self.manager.count(), 1) def check_syncs_normal_chunks(self, op, collection, *args): c = Chunk(data="<div/>", is_normal=True) c.save() self.make_reachable(c) # If it does not have metadata yet, that's fine. try: c.chunkmetadata.delete() except ChunkMetadata.DoesNotExist: pass # We have to delete the collection because merely saving the # chunk causes it to be synced, but this is not what we are # testing here. We want to make sure that calling # sync_with_exist will perform the sync. db = ExistDB() db.removeCollection(collection, True) self.assertEqual(len(list_collection(db, collection)), 0) getattr(self.manager, op)(*args) self.assertEqual(len(list_collection(db, collection)), 1) # Make sure our chunk was not collected. self.assertEqual(self.manager.count(), 1) def test_sync_skips_abnormal_chunks(self): """ ``sync_with_exist`` does not sync abnormal chunks. """ self.check_skip_abnormal_chunks("sync_with_exist", self.chunk_collection_path) def test_sync_syncs_normal_chunks(self): """ ``sync_with_exist`` syncs normal chunks. """ self.check_syncs_normal_chunks("sync_with_exist", self.chunk_collection_path) def check_deletes_documents(self, op, collection, *args): c = Chunk(data="<div/>", is_normal=True) c.save() entry = self.make_reachable(c) # If it does not have metadata yet, that's fine. try: c.chunkmetadata.delete() except ChunkMetadata.DoesNotExist: pass # We have to delete the collection because merely saving the # chunk causes it to be synced, but this is not what we are # testing here. We want to make sure that calling # op will perform the sync. db = ExistDB() db.removeCollection(collection, True) self.assertEqual(len(list_collection(db, collection)), 0) op = getattr(self.manager, op) op(*args) self.assertEqual(len(list_collection(db, collection)), 1) # Make sure our chunk was not collected. self.assertEqual(self.manager.count(), 1) # Now we delete the chunk in SQL because we do not want the # ``delete`` method to be called, as it would take care of # removing the document itself. (And yes, we do interpolate # the table name. This is safe as ``Entry._meta.db_table`` is # a value under our control.) with connection.cursor() as cursor: cr = entry.latest cursor.execute( "DELETE FROM {} WHERE id = %s".format(entry._meta.db_table), [entry.pk]) # We have to do this ourselves because Django's cascading # delete is implemented at the ORM level, not the database # level. cursor.execute( "DELETE FROM {} WHERE id = %s".format(cr._meta.db_table), [cr.pk]) # Check that no collection or syncing has occurred. self.assertEqual(self.manager.count(), 1) self.assertEqual(len(list_collection(db, collection)), 1) op(*args) # Make sure our chunk was collected. self.assertEqual(self.manager.count(), 0) self.assertEqual(len(list_collection(db, collection)), 0) def test_sync_deletes_exist_documents(self): """ ``sync_with_exist`` deletes those eXist documents that belong to chunks that no longer exist. """ self.check_deletes_documents("sync_with_exist", self.chunk_collection_path) def test_prepare_collects(self): """ ``prepare`` causes a collection of unreachable chunks. """ self.check_collects("prepare", "xml", True) def test_prepare_skips_abnormal_chunks(self): """ ``prepare`` does not sync abnormal chunks. """ self.check_skip_abnormal_chunks("prepare", self.display_collection_path, "xml", True) def test_prepare_syncs_normal_chunks(self): """ ``prepare`` syncs normal chunks. """ self.check_syncs_normal_chunks("prepare", self.display_collection_path, "xml", True) def test_prepare_deletes_exist_documents(self): """ ``prepare`` deletes those eXist documents that belong to chunks that no longer exist. """ self.check_deletes_documents("prepare", self.display_collection_path, "xml", True)
class ChunkTestCase(util.DisableMigrationsMixin, TestCase): chunk_collection_path = get_collection_path("chunks") display_collection_path = get_collection_path("display") prepare_kinds = Chunk.key_kinds @classmethod def setUpTestData(cls): super(ChunkTestCase, cls).setUpTestData() cls.foo = foo = user_model.objects.create(username="******", password="******") scribe = Group.objects.get(name='scribe') cls.foo.groups.add(scribe) def setUp(self): cache.clear() return super(ChunkTestCase, self).setUp() def assertLogRegexp(self, handler, stream, regexp): handler.flush() self.assertRegex(stream.getvalue(), regexp) def test_abnormal_is_invalid(self): """ Checks that an abnormal chunk is invalid, and that its validity is saved after being computed. """ c = Chunk(data="", is_normal=False) c.save() self.assertIsNone(c._valid) self.assertFalse(c.valid) self.assertFalse( Chunk.objects.get(pk=c.pk)._valid, "_valid was saved.") def test_valid(self): """ Checks that an normal chunk can be valid, and that its validity is saved after being computed. """ c = Chunk(data=valid_editable.decode('utf-8'), schema_version=schema_version) c.save() self.assertIsNone(c._valid) self.assertTrue(c.valid) self.assertTrue(Chunk.objects.get(pk=c.pk)._valid, "_valid was saved.") def test_valid(self): """ Checks that an normal chunk can be valid, and that its validity is saved after being computed. """ c = Chunk(data=valid_editable.decode('utf-8'), schema_version=schema_version) c.save() self.assertIsNone(c._valid) self.assertTrue(c.valid) self.assertTrue(Chunk.objects.get(pk=c.pk)._valid, "_valid was saved.") def test_invalid(self): """ Checks that data that is invalid is recognized as invalid, and the the validity is saved after being computed. """ # This data is just flat out invalid... data = """ <btw:entry xmlns="http://www.tei-c.org/ns/1.0" version="0.10"\ xmlns:btw="http://mangalamresearch.org/ns/btw-storage"> <btw:lemma></btw:lemma> </btw:entry> """ c = Chunk(data=data, schema_version=schema_version) c.save() self.assertIsNone(c._valid) self.assertFalse(c.valid) self.assertFalse( Chunk.objects.get(pk=c.pk)._valid, "_valid was saved.") def test_invalid_schematron(self): """ Checks that data that is invalid only due to the schematron check is recognized as invalid, and the the validity is saved after being computed. """ tree = lxml.etree.fromstring(valid_editable) sfs = tree.xpath( "//btw:example/btw:semantic-fields | " "//btw:example-explained/btw:semantic-fields", namespaces=xml.default_namespace_mapping) for el in sfs: el.getparent().remove(el) data = lxml.etree.tostring(tree, xml_declaration=True, encoding='utf-8').decode('utf-8') self.assertTrue( util.validate_with_rng(xml.schema_for_version(schema_version), data), "the data should validate") self.assertFalse( util.schematron(xml.schematron_for_version(schema_version), data), "the data should not pass the schematron check") c = Chunk(data=data, schema_version=schema_version) c.save() self.assertIsNone(c._valid) self.assertFalse(c.valid) self.assertFalse( Chunk.objects.get(pk=c.pk)._valid, "_valid was saved.") def test_published_false(self): """ ``published`` is false for chunks that have not been published. """ c = Chunk(data="<div/>", is_normal=True) c.save() self.assertFalse(c.published) def test_published_true(self): """ ``published`` is true for chunks that have been published. """ c = Chunk(data=valid_editable.decode('utf-8'), schema_version=schema_version) c.save() e = Entry() e.update(self.foo, "q", c, "foo", ChangeRecord.CREATE, ChangeRecord.MANUAL) e.latest.publish(self.foo) self.assertTrue(c.published) def test_chunks_start_hidden(self): """ ``hidden`` is ``True`` for new chunks. """ c = Chunk(data="<div/>", is_normal=True) c.save() self.assertTrue(c.hidden) def test_chunks_with_all_hidden_records_are_hidden(self): """ ``hidden`` is ``True`` if all ``ChangeRecord``s are hidden. """ c = Chunk(data="<div/>", is_normal=True) c.save() e = Entry() e.update(self.foo, "q", c, "foo", ChangeRecord.CREATE, ChangeRecord.MANUAL) e.latest.hidden = True e.latest.save() e = Entry() e.update(self.foo, "q", c, "foo2", ChangeRecord.CREATE, ChangeRecord.MANUAL) e.latest.hidden = True e.latest.save() self.assertTrue(c.hidden) def test_chunks_with_one_shown_record_are_shown(self): """ ``hidden`` is ``False`` if one ``ChangeRecord`` is shown. """ c = Chunk(data="<div/>", is_normal=True) c.save() e = Entry() e.update(self.foo, "q", c, "foo", ChangeRecord.CREATE, ChangeRecord.MANUAL) e.latest.hidden = True e.latest.save() e = Entry() e.update(self.foo, "q", c, "foo2", ChangeRecord.CREATE, ChangeRecord.MANUAL) self.assertFalse(c.hidden) def test_exist_path(self): """ ``exist_path`` returns good values. """ c = Chunk(data="<div/>", is_normal=True) c.save() self.assertEqual(c.exist_path("chunks"), "/".join([get_collection_path("chunks"), c.c_hash])) self.assertEqual(c.exist_path("display"), "/".join([get_collection_path("display"), c.c_hash])) def test_exist_path_raises(self): """ ``exist_path`` raises an error if the kind is wrong. """ c = Chunk(data="<div/>", is_normal=True) c.save() with self.assertRaisesRegex(ValueError, "unknown value"): c.exist_path("invalid") def test_display_key(self): """ ``display_key`` returns good values. """ c = Chunk(data="<div/>", is_normal=True) c.save() for kind in self.prepare_kinds: self.assertEqual(c.display_key(kind), "{}_{}".format(c.c_hash, kind).encode("utf8")) def test_display_key_raises(self): """ ``display_key`` raises an error if the kind is wrong. """ c = Chunk(data="<div/>", is_normal=True) c.save() with self.assertRaisesRegex(ValueError, "unknown display key kind"): c.display_key("invalid") def test_get_cached_value_starts_task(self): """ Check that ``get_cached_value`` starts an actual task if the value is missing, and returns ``None``. """ c = Chunk(data="<div/>", is_normal=True) c.save() for kind in self.prepare_kinds: cache.clear() with util.WithStringIO(models.logger) as (stream, handler): self.assertIsNone(c.get_cached_value(kind)) self.assertLogRegexp( handler, stream, "^{0} is missing from article_display, launching task$". format(c.display_key(kind))) def test_get_cached_value_knows_about_tasks(self): """ Check that ``get_cached_value`` will log if a task is already computing the value and will return ``None``. """ c = Chunk(data="<div/>", is_normal=True) c.save() for kind in self.prepare_kinds: cache.clear() cache.set(c.display_key(kind), {"task": "foo"}) with util.WithStringIO(models.logger) as (stream, handler): self.assertIsNone(c.get_cached_value(kind)) self.assertLogRegexp( handler, stream, "^{0} is being computed by task foo$".format( c.display_key(kind))) def test_get_cached_value_returns_available_data(self): """ Check that ``get_cached_value`` will log if a task is already computing the value and will return ``None``. """ c = Chunk(data="<div/>", is_normal=True) c.save() c._create_cached_data() # We have to launch the bibl data preparation ourselves. c.prepare("bibl", True) for kind in self.prepare_kinds: expected = cache.get(c.display_key(kind)) self.assertIsNotNone(expected) self.assertEqual(c.get_cached_value(kind), expected) def check_skip_abnormal_chunks(self, op, collection, *args): c = Chunk(data="", is_normal=False) c.save() # We have to delete the collection because merely saving the # chunk causes it to be synced, but this is not what we are # testing here. We want to make sure that calling # sync_with_exist will perform the sync. db = ExistDB() db.removeCollection(collection, True) self.assertEqual(len(list_collection(db, collection)), 0) getattr(c, op)(*args) self.assertEqual(len(list_collection(db, collection)), 0) def test_sync_skips_abnormal_chunks(self): """ ``sync_with_exist`` skips abnormal chunks. """ self.check_skip_abnormal_chunks("sync_with_exist", self.chunk_collection_path) def check_sync_normal_chunks(self, op, collection, *args): c = Chunk(data="<div/>", is_normal=True) c.save() # We have to delete the collection because merely saving the # chunk causes it to be synced, but this is not what we are # testing here. We want to make sure that calling # sync_with_exist will perform the sync. db = ExistDB() db.removeCollection(collection, True) self.assertEqual(len(list_collection(db, collection)), 0) ret = getattr(c, op)(*args) self.assertEqual(len(list_collection(db, collection)), 1) return ret def test_sync_syncs_normal_chunks(self): """ ``sync_with_exist`` syncs normal chunks. """ self.check_sync_normal_chunks("sync_with_exist", self.chunk_collection_path) def test_sync_handles_overwrites(self): """ ``sync_with_exist`` will not overwrite documents already in eXist. """ db = ExistDB() db.removeCollection(self.chunk_collection_path, True) c = Chunk(data="<div/>", is_normal=True) c.save() c.sync_with_exist() self.assertEqual(len(list_collection(db, self.chunk_collection_path)), 1) with mock.patch('lexicography.models.ExistDB.load') as load_mock: c.sync_with_exist() self.assertEqual(load_mock.call_count, 0, "load should not have been called!") self.assertEqual(len(list_collection(db, self.chunk_collection_path)), 1) def test_prepare_xml_skips_abnormal_chunks(self): """ ``prepare`` with the "xml" kind skips abnormal chunks. """ self.check_skip_abnormal_chunks("prepare", self.display_collection_path, "xml", True) def test_prepare_xml_syncs_normal_chunks(self): """ ``prepare`` with the "xml" kind syncs normal chunks. """ self.check_sync_normal_chunks("prepare", self.display_collection_path, "xml", True) def test_prepare_can_run_asynchronously(self): """ ``prepare`` can run asynchronously """ c = Chunk(data="<doc/>", is_normal=True) c.save() for kind in self.prepare_kinds: ret = c.prepare("xml") # When run asynchronously, we get an AsyncResult on which we # can call ``get``. ret.get() def check_remove_data_from_exist_and_cache(self, op): """ Check that invoking ``op`` will remove the data from the eXist database and the cache. """ db = ExistDB() c = Chunk(data="<div/>", is_normal=True) c.clean() method = op if isinstance(op, Callable) else getattr(c, op) cache.delete(c.c_hash) db.removeCollection(self.chunk_collection_path, True) db.removeCollection(self.display_collection_path, True) self.assertEqual(len(list_collection(db, self.chunk_collection_path)), 0) self.assertEqual( len(list_collection(db, self.display_collection_path)), 0) for kind in self.prepare_kinds: self.assertIsNone(cache.get(c.display_key(kind))) c.save() c._create_cached_data() keys = [c.display_key(kind) for kind in self.prepare_kinds] # Only the "xml" data is created on save. self.assertIsNotNone(cache.get(c.display_key("xml"))) self.assertEqual(len(list_collection(db, self.chunk_collection_path)), 1) self.assertEqual( len(list_collection(db, self.display_collection_path)), 1) method() self.assertEqual(len(list_collection(db, self.chunk_collection_path)), 0) self.assertEqual( len(list_collection(db, self.display_collection_path)), 0) for key in keys: self.assertIsNone(cache.get(key)) def test_delete_removes_data_from_exist_and_cache(self): """ Deleting a chunk removes its associated data from eXist and from the cache. """ self.check_remove_data_from_exist_and_cache("delete") def test_delete_cached_data_removes_data_from_exist_and_cache(self): """ _delete_cached_data removes data from eXist and the cache. """ self.check_remove_data_from_exist_and_cache("_delete_cached_data") def check_abnormal_remove_data_from_exist_and_cache(self, op): db = ExistDB() c = Chunk(data="<div/>", is_normal=False) c.clean() cache.delete(c.c_hash) method = getattr(c, op) db.removeCollection(self.chunk_collection_path, True) db.removeCollection(self.display_collection_path, True) c.save() keys = [c.display_key(kind) for kind in self.prepare_kinds] for kind in self.prepare_kinds: self.assertIsNone(cache.get(c.display_key(kind))) self.assertEqual(len(list_collection(db, self.chunk_collection_path)), 0) self.assertEqual( len(list_collection(db, self.display_collection_path)), 0) with mock.patch('lexicography.models.ExistDB.removeDocument') as \ remove_mock: method() self.assertEqual(remove_mock.call_count, 0) self.assertEqual(len(list_collection(db, self.chunk_collection_path)), 0) self.assertEqual( len(list_collection(db, self.display_collection_path)), 0) for key in keys: self.assertIsNone(cache.get(key)) def test_delete_abnormal_does_not_touch_exist_or_cache(self): """ Deleting an abnormal chunk does not touch the cache or the database. """ self.check_abnormal_remove_data_from_exist_and_cache("delete") def test_delete_cached_data_abnormal_does_not_touch_exist_or_cache(self): """ ``_delete_cached_data`` on an abnormal chunk does not touch the cache or the database. """ self.check_abnormal_remove_data_from_exist_and_cache( "_delete_cached_data")
def __call__(self, command, _options): assert_running() db = get_admin_db() collection = get_collection_path(None) db.removeCollectionIndex(collection)
def filter_queryset(self, qs): # pylint: disable=too-many-branches search_value = self.request.GET.get('search[value]', None) if search_value is not None: search_value = search_value.encode("utf-8") lemmata_only = self.request.GET.get('lemmata_only', "false") == \ "true" if usermod.can_author(self.request.user): publication_status = self.request.GET.get('publication_status', "published") search_all = self.request.GET.get('search_all', "false") == "true" if not search_all: # Remove deleted entries from the set. active = qs.filter(entry__in=Entry.objects.active_entries()) if publication_status == "published": active = active.filter(entry__latest_published=F('pk')) elif publication_status == "unpublished": active = active.filter(entry__latest=F('pk')) \ .exclude(entry__latest_published=F('pk')) elif publication_status == "both": active = active.filter(entry__latest=F('pk')) else: raise ValueError("unknown value for publication_status: " + publication_status) else: if publication_status == "published": active = qs.filter(published=True) elif publication_status == "unpublished": active = qs.filter(published=False) elif publication_status == "both": active = qs else: raise ValueError("unknown value for publication_status: " + publication_status) else: # If the user cannot author, then our queryset is already # reduced to what the user can see: the latest version of # published articles. active = qs if search_value: db = ExistDB() chunks = [] if lemmata_only: scope = "//btw:lemma" # When we do a lemma search, hits are not useful. hit = "" else: scope = "//btw:entry" hit = "{kwic:summarize($m, <config width='80'/>)}" for query_chunk in query_iterator(db, xquery.format( """\ import module namespace kwic="http://exist-db.org/xquery/kwic"; for $m in collection({db}){scope}[ft:query(., {search_text})] order by ft:score($m) descending return <result><doc>{doc}</doc><hit>{hit}</hit></result>""", db=get_collection_path("display"), scope=xquery.Verbatim(scope), doc=xquery.Verbatim("{util:document-name($m)}"), hit=xquery.Verbatim(hit), search_text=search_value)): for result in query_chunk.results: chunk = result[0].text # Content of <doc>. self.chunk_to_hits[chunk] = result[1] chunks.append(chunk) # We need to get the changerecords that pertain to these chunks. qs = active.filter(c_hash__in=set(chunks)) else: qs = active return qs