def setup_catalog(context):
portal = context.getSite()
catalog_name = 'marginalia_catalog'
try:
catalog = cmfutils.getToolByName(portal, catalog_name)
except AttributeError:
# register catalog
catalog = ZCatalog(catalog_name, u'Marginalia catalog', None, portal)
portal._setObject(catalog_name, catalog)
# add indexes and columns
plaintext_extra = SimpleRecord(lexicon_id='plaintext_lexicon',
index_type='Okapi BM25 Rank')
indexes = catalog.indexes()
columns = catalog.schema()
# install lexicon
_id = 'plaintext_lexicon'
if not hasattr(catalog, _id):
lexicon = PLexicon(_id, '', Splitter(), CaseNormalizer(),
StopWordRemover())
catalog._setObject(_id, lexicon)
for indexName, indexType, extra in (('edit_type', 'FieldIndex',
None), ('note', 'ZCTextIndex',
plaintext_extra),
('link_title', 'FieldIndex',
None)):
if indexName not in indexes:
catalog.addIndex(indexName, indexType, extra=extra)
def setUp(self):
from Products.ZCatalog.ZCatalog import ZCatalog
catalog = ZCatalog('catalog')
catalog.addIndex('id', 'FieldIndex')
root = Folder('')
root.getPhysicalRoot = lambda: root
self.root = root
self.root.catalog = catalog
def setUp(self):
cleanup.CleanUp.setUp(self)
self.zcat = ZCatalog('catalog')
self.zcat.long_query_time = 0.0
self._add_indexes()
for i in range(9):
obj = Dummy(i)
self.zcat.catalog_object(obj, str(i))
def afterSetUp(self):
self.app.real_cat = ZCatalog('real_cat')
self.app.real_cat.addIndex('id', 'FieldIndex')
self.app.real_cat.addIndex('title', 'FieldIndex')
self.app.real_cat.addIndex('meta_type', 'FieldIndex')
self.app.queue_cat = QueueCatalog(3) # 3 buckets
self.app.queue_cat.id = 'queue_cat'
self.app.queue_cat.manage_edit(location='/real_cat',
immediate_indexes=['id', 'title'])
def setUp(self):
cleanup.CleanUp.setUp(self)
from Products.ZCatalog.ZCatalog import ZCatalog
self.zcat = ZCatalog('catalog')
self.zcat.long_query_time = 0.0
self._add_indexes()
for i in range(9):
obj = dummy(i)
self.zcat.catalog_object(obj, str(i))
def setUp(self):
app = Zope.app()
self.app = app
app.real_cat = ZCatalog('real_cat')
app.real_cat.addIndex('id', 'FieldIndex')
app.real_cat.addIndex('meta_type', 'FieldIndex')
app.queue_cat = QueueCatalog(3) # 3 buckets
app.queue_cat.id = 'queue_cat'
app.queue_cat.manage_edit(location='/real_cat',
immediate_indexes=['id'])
def setUp(self):
from Products.ZCatalog.ZCatalog import ZCatalog
PlacelessSetup.setUp(self)
zcml.load_config('meta.zcml', Products.Five)
zcml.load_config('configure.zcml', Products.GenericSetup.PluginIndexes)
zcml.load_config('configure.zcml', Products.GenericSetup.ZCatalog)
zcml.load_config('configure.zcml', Products.GenericSetup.ZCTextIndex)
self._obj = ZCatalog('foo_catalog')
self._XML = _CATALOG_XML % ('', '')
def _make_catalog(self):
zcat = ZCatalog('catalog')
zcat._catalog.addIndex('big', BooleanIndex('big'))
zcat._catalog.addIndex('date', DateRangeIndex('date', 'start', 'end'))
zcat._catalog.addIndex('num', FieldIndex('num'))
zcat._catalog.addIndex('numbers', KeywordIndex('numbers'))
zcat._catalog.addIndex('path', PathIndex('getPhysicalPath'))
zcat._catalog.addIndex('uuid', UUIDIndex('num'))
for i in range(9):
obj = Dummy(i)
zcat.catalog_object(obj, str(i))
return zcat
def __init__(self):
# Create the ZCatalog instance
self.catalog = ZCatalog('catalog')
self.catalog.addIndex('requester', 'FieldIndex')
self.catalog.addIndex('status', 'FieldIndex')
self.catalog.addIndex('user', 'FieldIndex')
self.catalog.addColumn('requester')
self.catalog.addColumn('roles')
self.catalog.addColumn('status')
self.catalog.addColumn('user')
self._p_changed=1
def setUp(cls):
cls.app = makerequest(Zope2.app())
cls.app._setObject('Catalog', ZCatalog('Catalog'))
cls.app.Catalog.addIndex('meta_type', FieldIndex('meta_type'))
cls.app.Catalog.addColumn('id')
cls.app.Catalog.addColumn('title')
cls.app._setObject('Database', Folder('Database'))
# newly added objects have ._p_jar == None, initialize it
transaction.savepoint()
cache_size = cls.app._p_jar.db().getCacheSize()
cls._fill_catalog(cls.app, cls.app.Catalog, cache_size * 3)
# attach new persistent objects to ZODB connection
transaction.savepoint()
def test_getCatalogPlan_full(self):
zcat = ZCatalog('catalog')
plan = self._makeOne(zcat._catalog, query={'index1': 1, 'index2': 2})
plan.start()
plan.start_split('index1')
time.sleep(0.001)
plan.stop_split('index1')
plan.start_split('index2')
time.sleep(0.001)
plan.stop_split('index2')
plan.stop()
plan_str = zcat.getCatalogPlan()
self.assertTrue('queryplan = {' in plan_str)
self.assertTrue('index1' in plan_str)
def _create_catalog(self):
"""Creates the ZCatalog instance for searching links"""
# self.catalog = ZCatalog('catalog').__of__(self)
self.catalog = ZCatalog('catalog')
self.catalog.addIndex('source', 'FieldIndex')
self.catalog.addIndex('strength', 'FieldIndex')
self.catalog.addColumn('target')
self.catalog.addColumn('category')
self.catalog.addColumn('strength')
self.catalog.addColumn('title')
self._p_changed=1
def test_fixOkapiIndexes(self):
catalog = ZCatalog('catalog')
catalog.lexicon = PLexicon('lexicon')
catalog.addIndex(
'test',
ZCTextIndex('test',
index_factory=OkapiIndex,
caller=catalog,
lexicon_id='lexicon'))
catalog.Indexes['test'].index._totaldoclen = -1000
from plone.app.upgrade.v41.final import fixOkapiIndexes
fixOkapiIndexes(catalog)
self.assertEqual(0, catalog.Indexes['test'].index._totaldoclen())
def setUp(self):
from Products.ZCatalog.ZCatalog import ZCatalog
self._catalog = ZCatalog('Catalog')
self._catalog.resolve_path = self._resolve_num
self._catalog.addIndex('title', 'KeywordIndex')
self._catalog.addColumn('title')
self.upper = 10
self.d = {}
for x in range(0, self.upper):
# make uid a string of the number
ob = zdummy(x)
self.d[str(x)] = ob
self._catalog.catalog_object(ob, str(x))
def __init__(self, id='Help', title=''):
self.id=id
self.title=title
c=self.catalog=ZCatalog('catalog')
# clear catalog
for index in c.indexes():
c.delIndex(index)
for col in c.schema():
c.delColumn(col)
c.addIndex('SearchableText', 'TextIndex')
c.addIndex('categories', 'KeywordIndex')
c.addIndex('permissions', 'KeywordIndex')
c.addColumn('categories')
c.addColumn('permissions')
c.addColumn('title_or_id')
c.addColumn('url')
c.addColumn('id')
def test_getCatalogPlan_full(self):
from Products.ZCatalog.ZCatalog import ZCatalog
zcat = ZCatalog('catalog')
plan = self._makeOne(zcat._catalog, query={'index1': 1, 'index2': 2})
plan.start()
plan.start_split('index1')
time.sleep(0.1111)
plan.stop_split('index1')
plan.start_split('index2')
time.sleep(0.2222)
plan.stop_split('index2')
plan.stop()
plan_str = zcat.getCatalogPlan()
self.assertTrue('queryplan = {' in plan_str)
self.assertTrue('index1' in plan_str)
# test rounding worked
self.assertTrue('(0.11, 1, False),' in plan_str)
self.assertTrue('(0.22, 1, False),' in plan_str)
def after_manage_edit(self, form):
"add the catalog, add the appropriate indexes to it, and cause it to reindex if we modify it"
folder = self.getCompoundDocContainer().restrictedTraverse(
self.orderLocation, None)
if folder is not None and not hasattr(folder, self.orderCatalog):
folder._setObject(self.orderCatalog, ZCatalog(self.orderCatalog))
catalog = getattr(folder, self.orderCatalog)
catalog.addIndex('customerLogin', 'FieldIndex')
catalog.addIndex('id', 'FieldIndex')
catalog.addIndex('bobobase_modification_time', 'FieldIndex')
path = '/'.join(folder.getPhysicalPath())
catalog.ZopeFindAndApply(folder,
obj_metatypes=['CompoundDoc'],
search_sub=1,
apply_func=catalog.catalog_object,
apply_path=path)
self.addToCatalog(folder)
transaction.get().commit()
def setUp(self):
self.cat = ZCatalog("catalog")
self.cat.addIndex('text',"TextIndex")
self.cat.addColumn('text')
self.cat.addIndex('kw','KeywordIndex')
self.cat.addColumn('kw')
t1 = TO('the quick brown fox jumps over the lazy dog',['quick','fox'])
t2 = TO('i am the nice alien from the future',['alien','future'])
t3 = TO('i am a brown fox dancing with a future alien',['zerstört','könnten'])
t4 = TO('i am a brown ' + unicode('fox') + ' dancing with a future alien',[])
t5 = TO("""
Die USA und Großbritannien können nach der Zerstörung der
afghanischen Luftabwehr nun rund um die Uhr Angriffe fliegen. Das gab
Verteidigungsminister Donald Rumsfeld bekannt. Bei den dreitägigen Angriffen
seien auch bis auf einen alle Flugplätze der Taliban zerstört worden. Rumsfeld
erklärte weiter, er könne die Berichte nicht bestätigen, wonach bei den
amerikanischen Angriffen vier afghanische Mitarbeiter einer von den UN
finanzierten Hilfsorganisation getötet wurden. Diese könnten auch durch
Gegenfeuer der Taliban getötet worden sein.
""",[unicode('dreitägigen','latin1'),'zerstört'])
self.cat.catalog_object(t1,"o1")
self.cat.catalog_object(t2,"o2")
self.cat.catalog_object(t3,"o3")
self.cat.catalog_object(t4,"o4")
self.cat.catalog_object(t5,"o5")
self.tests = [('quick',('o1',)),
('fox',('o1','o3','o4')),
('afghanischen', ('o5',)),
('dreitägigen',('o5',))
]
self.kw_tests = [ ('quick',('o1',) ),
('zerstört',('o3','o5')),
('dreitägigen',('o5',))
]
def __init__(self, id='Help', title=''):
self.id = id
self.title = title
c = self.catalog = ZCatalog('catalog')
l = PLexicon('lexicon', '', HTMLWordSplitter(), CaseNormalizer(),
StopWordRemover())
c._setObject('lexicon', l)
i = ZCTextIndex('SearchableText',
caller=c,
index_factory=OkapiIndex,
lexicon_id=l.id)
# not using c.addIndex because it depends on Product initialization
c._catalog.addIndex('SearchableText', i)
c._catalog.addIndex('categories', KeywordIndex('categories'))
c._catalog.addIndex('permissions', KeywordIndex('permissions'))
c.addColumn('categories')
c.addColumn('permissions')
c.addColumn('title_or_id')
c.addColumn('url')
c.addColumn('id')
def _make_catalog(self):
from Products.PluginIndexes.BooleanIndex.BooleanIndex import \
BooleanIndex
from Products.PluginIndexes.DateRangeIndex.DateRangeIndex import \
DateRangeIndex
from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
from Products.PluginIndexes.KeywordIndex.KeywordIndex import \
KeywordIndex
from Products.PluginIndexes.PathIndex.PathIndex import PathIndex
from Products.PluginIndexes.UUIDIndex.UUIDIndex import UUIDIndex
from Products.ZCatalog.ZCatalog import ZCatalog
zcat = ZCatalog('catalog')
zcat._catalog.addIndex('big', BooleanIndex('big'))
zcat._catalog.addIndex('date', DateRangeIndex('date', 'start', 'end'))
zcat._catalog.addIndex('num', FieldIndex('num'))
zcat._catalog.addIndex('numbers', KeywordIndex('numbers'))
zcat._catalog.addIndex('path', PathIndex('getPhysicalPath'))
zcat._catalog.addIndex('uuid', UUIDIndex('num'))
for i in range(9):
obj = dummy(i)
zcat.catalog_object(obj, str(i))
return zcat
def setUp(self):
app = Application()
catalog = ZCatalog('Catalog')
app._setObject('Catalog', catalog)
self.catalog = catalog = app._getOb('Catalog')
install_products(app, 'ManagableIndex')
# field
self.fi = self._createIndex('id', FieldIndex)
# keyword
self.ki = self._createIndex('kw', KeywordIndex)
# range
self.ri = self._createIndex(
'ri',
RangeIndex,
dict(CombineType='aggregate',
ValueProviders=[
dict(id='rlow', type='AttributeLookup'),
dict(id='rhigh', type='AttributeLookup'),
]),
)
# word
lexicon = Lexicon(Splitter())
app._setObject('lexicon', lexicon)
self.wi = self._createIndex('wi', WordIndex, dict(Lexicon='lexicon'))
# simple text
self.sti = self._createIndex('sti', SimpleTextIndex,
dict(Lexicon='lexicon'))
# path
self.pi = self._createIndex('pi', PathIndex)
# create objects
self.obj1 = obj1 = _Object()
obj1.kw = (1, 2)
obj1.fkw = _Caller(lambda obj: obj.kw)
obj1.fid = _Caller(lambda obj: obj.id)
self.obj2 = obj2 = _Object().__of__(obj1)
obj2.id = 'id'
def setUp(self):
from Products.ZCatalog.ZCatalog import ZCatalog
self._obj = ZCatalog('foo_catalog')
self._BODY = _CATALOG_BODY % ('', '', '')
def _makeOne(self):
from Products.ZCatalog.ZCatalog import ZCatalog
return ZCatalog('Catalog')
def test_get_id_persistent(self):
zcat = ZCatalog('catalog')
plan = self._makeOne(zcat._catalog)
self.assertEquals(plan.get_id(), ('catalog', ))
def test_getCatalogPlan_empty(self):
from Products.ZCatalog.ZCatalog import ZCatalog
zcat = ZCatalog('catalog')
self._makeOne(zcat._catalog)
plan_str = zcat.getCatalogPlan()
self.assertTrue('queryplan = {' in plan_str)
def test_get_id_persistent(self):
from Products.ZCatalog.ZCatalog import ZCatalog
zcat = ZCatalog('catalog')
plan = self._makeOne(zcat._catalog)
self.assertEquals(plan.get_id(), ('catalog', ))
def test_getCatalogPlan_empty(self):
zcat = ZCatalog('catalog')
self._makeOne(zcat._catalog)
plan_str = zcat.getCatalogPlan()
self.assertTrue('queryplan = {' in plan_str)
def afterSetUp(self):
self.folder._setObject('catalog', ZCatalog('catalog'))
def _create_catalog(self):
"""Creates the ZCatalog instance for versioned objects"""
self.catalog = ZCatalog('catalog')
lexicon = PLexicon('lexicon', '', Splitter(), CaseNormalizer(),
StopWordAndSingleCharRemover())
self.catalog._setObject('lexicon', lexicon)
ZCText_extras = Empty()
ZCText_extras.doc_attr = 'abstract'
ZCText_extras.index_type = 'Okapi BM25 Rank'
ZCText_extras.lexicon_id = 'lexicon'
self.catalog.addIndex('abstract', 'ZCTextIndex', ZCText_extras)
ZCText_extras.doc_attr = 'Title'
self.catalog.addIndex('Title', 'ZCTextIndex', ZCText_extras)
ZCText_extras.doc_attr = 'institution'
self.catalog.addIndex('institution', 'ZCTextIndex', ZCText_extras)
ZCText_extras.doc_attr = 'keywords'
self.catalog.addIndex('keywordstext', 'ZCTextIndex', ZCText_extras)
self.catalog.addIndex('atomicInstitution', 'FieldIndex',
{'indexed_attrs': 'institution'})
self.catalog.addIndex('authors', 'KeywordIndex')
self.catalog.addIndex('parentAuthors', 'KeywordIndex')
self.catalog.addIndex('maintainers', 'KeywordIndex')
self.catalog.addIndex('language', 'KeywordIndex')
self.catalog.addIndex('modified', 'DateIndex')
self.catalog.addIndex('revised', 'DateIndex')
self.catalog.addIndex('objectId', 'FieldIndex')
self.catalog.addIndex('portal_type', 'FieldIndex')
self.catalog.addIndex('containedModuleIds', 'KeywordIndex')
self.catalog.addIndex('subject', 'KeywordIndex')
extra = Empty()
extra.indexed_attrs = 'keywords'
self.catalog.addIndex('keywordscase', 'KeywordIndex', extra)
ki = KeywordIndex('keywords')
self.catalog._catalog.addIndex('keywords', ki)
ki._updateProperty('PrenormalizeTerm', 'python: value.lower()')
ki._updateProperty('TermType', 'string')
ki.keywords._updateProperty('Normalizer',
'python: [k.lower() for k in value]')
ki = KeywordIndex('baselanguage')
self.catalog._catalog.addIndex('baselanguage', ki)
ki._updateProperty(
'PrenormalizeTerm',
"python: value[:(value.find('-') > 0 ) and value.find('-') or len(value)]"
)
ki.baselanguage._updateProperty('Name', 'language')
ki.baselanguage._updateProperty(
'Normalizer',
"python: [value[:(value.find('-') > 0 ) and value.find('-') or len(value)]]"
)
fi = FieldIndex('sortTitle')
self.catalog._catalog.addIndex('sortTitle', fi)
fi._updateProperty('PrenormalizeTerm', 'python: value.lower()')
fi._updateProperty('TermType', 'string')
fi.sortTitle._updateProperty('Name', 'Title')
fi.sortTitle._updateProperty('Normalizer',
'python: here.stripArticles(value)')
fi = FieldIndex('parent')
self.catalog._catalog.addIndex('parent', fi)
fi.parent._updateProperty('Name', 'getParent')
fi.parent._updateProperty('Normalizer', 'python:value.objectId')
ki = KeywordIndex('translators')
self.catalog._catalog.addIndex('translators', ki)
ki._delObject('translators')
ee = ExpressionEvaluator()
ee.id = 'translators'
ki._setObject(ee.id, ee)
ki.translators._updateProperty(
'Expression', "python: lambda o: o.roles['translators']")
ki = KeywordIndex('editors')
self.catalog._catalog.addIndex('editors', ki)
ki._delObject('editors')
ee = ExpressionEvaluator()
ee.id = 'editors'
ki._setObject(ee.id, ee)
ki.editors._updateProperty('Expression',
"python: lambda o: o.roles['editors']")
self._set_metadata()
self._p_changed = 1
def setUp(self):
from Products.ZCatalog.ZCatalog import ZCatalog
BodyAdapterTestCase.setUp(self)
self._obj = ZCatalog('foo_catalog')
self._BODY = _CATALOG_BODY % ('', '', '')
