def testRange(self):
"""Test a range search"""
index = FieldIndex( 'foo' )
for i in range(100):
index.index_object(i, Dummy(i%10))
record = { 'foo' : { 'query' : [-99, 3]
, 'range' : 'min:max'
}
}
r=index._apply_index( record )
assert tuple(r[1])==('foo',), r[1]
r=list(r[0].keys())
expect=[
0, 1, 2, 3, 10, 11, 12, 13, 20, 21, 22, 23, 30, 31, 32, 33,
40, 41, 42, 43, 50, 51, 52, 53, 60, 61, 62, 63, 70, 71, 72, 73,
80, 81, 82, 83, 90, 91, 92, 93
]
assert r==expect, r
#
# Make sure that range tests with incompatible paramters
# don't return empty sets.
#
record[ 'foo' ][ 'operator' ] = 'and'
r2, ignore = index._apply_index( record )
r2 = list( r2.keys() )
assert r2 == r
def setUp(self):
self._vocabulary = Vocabulary.Vocabulary('Vocabulary',
'Vocabulary',
globbing=1)
col1 = FieldIndex('col1')
col2 = TextIndex('col2')
col3 = KeywordIndex('col3')
self._catalog = Catalog()
self._catalog.addIndex('col1', col1)
self._catalog.addIndex('col2', col2)
self._catalog.addIndex('col3', col3)
self._catalog.addColumn('col1')
self._catalog.addColumn('col2')
self._catalog.addColumn('col3')
att1 = FieldIndex('att1')
att2 = TextIndex('att2')
att3 = KeywordIndex('att3')
num = FieldIndex('num')
self._catalog.addIndex('att1', att1)
self._catalog.addIndex('att2', att2)
self._catalog.addIndex('att3', att3)
self._catalog.addIndex('num', num)
self._catalog.addColumn('att1')
self._catalog.addColumn('att2')
self._catalog.addColumn('att3')
self._catalog.addColumn('num')
for x in range(0, self.upper):
self._catalog.catalogObject(dummy(self.nums[x]), ` x `)
self._catalog.aq_parent = dummy('foo') # fake out acquisition
def setUp( self ):
"""
"""
self._index = FieldIndex( 'foo' )
self._marker = []
self._values = [ ( 0, Dummy( 'a' ) )
, ( 1, Dummy( 'ab' ) )
, ( 2, Dummy( 'abc' ) )
, ( 3, Dummy( 'abca' ) )
, ( 4, Dummy( 'abcd' ) )
, ( 5, Dummy( 'abce' ) )
, ( 6, Dummy( 'abce' ) )
, ( 7, Dummy( 0 ) ) # Collector #1959
, ( 8, Dummy(None) )]
self._forward = {}
self._backward = {}
for k, v in self._values:
self._backward[k] = v
keys = self._forward.get( v, [] )
self._forward[v] = keys
self._noop_req = { 'bar': 123 }
self._request = { 'foo': 'abce' }
self._min_req = { 'foo': {'query': 'abc'
, 'range': 'min'}
}
self._max_req = { 'foo': {'query': 'abc'
, 'range': 'max' }
}
self._range_req = { 'foo': {'query': ( 'abc', 'abcd' )
, 'range': 'min:max' }
}
self._zero_req = { 'foo': 0 }
self._none_req = { 'foo': None }
def setUp(self):
self._indexes = [
FieldIndex('review_state'),
FieldIndex('portal_type'),
BooleanIndex('is_default_page'),
KeywordIndex('subject', extra={'indexed_attrs':
'keyword,subject'}),
CompositeIndex('comp01',
extra=[{
'id': 'portal_type',
'meta_type': 'FieldIndex',
'attributes': ''
}, {
'id': 'review_state',
'meta_type': 'FieldIndex',
'attributes': ''
}, {
'id': 'is_default_page',
'meta_type': 'BooleanIndex',
'attributes': ''
}, {
'id': 'subject',
'meta_type': 'KeywordIndex',
'attributes': 'keyword,subject'
}])
]
def _add_indexes(self):
num = FieldIndex('num')
self.zcat._catalog.addIndex('num', num)
big = FieldIndex('big')
self.zcat._catalog.addIndex('big', big)
numbers = KeywordIndex('numbers')
self.zcat._catalog.addIndex('numbers', numbers)
def setUp(self):
self._catalog = self._makeOne()
self._catalog.lexicon = PLexicon('lexicon')
col1 = FieldIndex('col1')
col2 = ZCTextIndex('col2', caller=self._catalog,
index_factory=OkapiIndex, lexicon_id='lexicon')
col3 = KeywordIndex('col3')
self._catalog.addIndex('col1', col1)
self._catalog.addIndex('col2', col2)
self._catalog.addIndex('col3', col3)
self._catalog.addColumn('col1')
self._catalog.addColumn('col2')
self._catalog.addColumn('col3')
att1 = FieldIndex('att1')
att2 = ZCTextIndex('att2', caller=self._catalog,
index_factory=OkapiIndex, lexicon_id='lexicon')
att3 = KeywordIndex('att3')
num = FieldIndex('num')
self._catalog.addIndex('att1', att1)
self._catalog.addIndex('att2', att2)
self._catalog.addIndex('att3', att3)
self._catalog.addIndex('num', num)
self._catalog.addColumn('att1')
self._catalog.addColumn('att2')
self._catalog.addColumn('att3')
self._catalog.addColumn('num')
for x in range(0, self.upper):
self._catalog.catalogObject(dummy(self.nums[x]), repr(x))
self._catalog = self._catalog.__of__(dummy('foo'))
def _make_one(self, extra=None):
from Products.ZCatalog.Catalog import Catalog
catalog = Catalog()
catalog.lexicon = PLexicon('lexicon')
att1 = FieldIndex('att1')
att2 = ZCTextIndex('att2',
caller=catalog,
index_factory=OkapiIndex,
lexicon_id='lexicon')
catalog.addIndex('att2', att2)
num = FieldIndex('num')
catalog.addIndex('att1', att1)
catalog.addIndex('num', num)
catalog.addColumn('num')
foo = MultiFieldIndex('foo')
catalog.addIndex('foo', foo)
if extra is not None:
extra(catalog)
for x in range(0, self.upper):
catalog.catalogObject(Dummy(self.nums[x]), repr(x))
return catalog.__of__(Dummy('foo'))
def _add_indexes(self):
from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
from Products.PluginIndexes.KeywordIndex.KeywordIndex import \
KeywordIndex
num = FieldIndex('num')
self.zcat._catalog.addIndex('num', num)
big = FieldIndex('big')
self.zcat._catalog.addIndex('big', big)
numbers = KeywordIndex('numbers')
self.zcat._catalog.addIndex('numbers', numbers)
def test_FieldIndex(self):
from xml.dom.minidom import parseString
from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
from Products.GenericSetup.testing import DummySetupEnviron
from Products.GenericSetup.PluginIndexes.exportimport \
import PluggableIndexNodeAdapter
environ = DummySetupEnviron()
def _no_clear(*a):
raise AssertionError("Don't clear me!")
index = FieldIndex('foo_field')
index.indexed_attrs = ['bar']
index.clear = _no_clear
adapted = PluggableIndexNodeAdapter(index, environ)
adapted.node = parseString(_FIELD_XML).documentElement # no raise
def get_catalog(self, createIfNotExists=True):
# Create catalog.
zmsroot = self.getRootElement()
home = zmsroot.getHome()
catalog = getattr(home,self.catalog_id,None)
if catalog is None:
catalog = ZCatalog.ZCatalog(id=self.catalog_id, title=self.meta_id, container=home)
home._setObject(catalog.id, catalog)
catalog = getattr(home,self.catalog_id,None)
# Index names.
index_names = self.get_index_names(True)
# Add indices.
for index_name in index_names:
if index_name not in catalog.indexes():
index_type = FieldIndex(index_name)
catalog.manage_addIndex(index_name,index_type)
index_name = 'path'
if index_name not in catalog.indexes():
index_type = PathIndex(index_name)
catalog.manage_addIndex(index_name,index_type)
# Add columns
for index_name in index_names + ['getPath']:
if index_name not in catalog.schema():
catalog.manage_addColumn(index_name)
return catalog
def test_FieldIndex(self):
from xml.dom.minidom import parseString
from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
from Products.GenericSetup.testing import DummySetupEnviron
from Products.GenericSetup.PluginIndexes.exportimport \
import PluggableIndexNodeAdapter
environ = DummySetupEnviron()
def _no_clear(*a):
raise AssertionError("Don't clear me!")
index = FieldIndex('foo_field')
index.indexed_attrs = ['bar']
index.clear = _no_clear
adapted = PluggableIndexNodeAdapter(index, environ)
adapted.node = parseString(_FIELD_XML).documentElement # no raise
def setUp( self ):
"""
"""
self._index = FieldIndex( 'foo' )
self._marker = []
self._values = [ ( 0, Dummy( 'a' ) )
, ( 1, Dummy( 'ab' ) )
, ( 2, Dummy( 'abc' ) )
, ( 3, Dummy( 'abca' ) )
, ( 4, Dummy( 'abcd' ) )
, ( 5, Dummy( 'abce' ) )
, ( 6, Dummy( 'abce' ) )
, ( 7, Dummy( 0 ) ) # Collector #1959
, ( 8, Dummy(None) )]
self._forward = {}
self._backward = {}
for k, v in self._values:
self._backward[k] = v
keys = self._forward.get( v, [] )
self._forward[v] = keys
self._noop_req = { 'bar': 123 }
self._request = { 'foo': 'abce' }
self._min_req = { 'foo': {'query': 'abc'
, 'range': 'min'}
}
self._max_req = { 'foo': {'query': 'abc'
, 'range': 'max' }
}
self._range_req = { 'foo': {'query': ( 'abc', 'abcd' )
, 'range': 'min:max' }
}
self._zero_req = { 'foo': 0 }
self._none_req = { 'foo': None }
def testEmptyMappingReturnsAll(self):
col1 = FieldIndex('col1')
self._catalog.addIndex('col1', col1)
for x in range(0, 10):
self._catalog.catalogObject(dummy(x), repr(x))
self.assertEqual(len(self._catalog), 10)
length = len(self._catalog({}))
self.assertEqual(length, 10)
def setUp(self):
vocabulary = Vocabulary.Vocabulary('Vocabulary',
'Vocabulary',
globbing=1)
self.catalogs = []
for i in range(3):
cat = Catalog()
cat.addIndex('num', FieldIndex('num'))
cat.addIndex('big', FieldIndex('big'))
cat.addIndex('title', TextIndex('title'))
cat.vocabulary = vocabulary
cat.aq_parent = zdummy(16336)
for i in range(10):
obj = zdummy(i)
obj.big = i > 5
cat.catalogObject(obj, str(i))
self.catalogs.append(cat)
def setUp(self):
self.catalogs = []
for i in range(3):
cat = self._makeOne()
cat.lexicon = PLexicon('lexicon')
cat.addIndex('num', FieldIndex('num'))
cat.addIndex('big', FieldIndex('big'))
cat.addIndex('number', FieldIndex('number'))
i = ZCTextIndex('title', caller=cat, index_factory=OkapiIndex,
lexicon_id='lexicon')
cat.addIndex('title', i)
cat = cat.__of__(zdummy(16336))
for i in range(10):
obj = zdummy(i)
obj.big = i > 5
obj.number = True
cat.catalogObject(obj, str(i))
self.catalogs.append(cat)
def setUp(self):
import Products.GenericSetup.PluginIndexes
from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
NodeAdapterTestCase.setUp(self)
zcml.load_config('configure.zcml', Products.GenericSetup.PluginIndexes)
self._obj = FieldIndex('foo_field')
self._XML = _FIELD_XML
def setUp(self):
from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
PlacelessSetup.setUp(self)
zcml.load_config('meta.zcml', Products.Five)
zcml.load_config('configure.zcml', Products.GenericSetup.PluginIndexes)
self._obj = FieldIndex('foo_field')
self._XML = _FIELD_XML
def setUp(self):
self._index = FieldIndex('foo')
self._marker = []
self._values = [(0, Dummy('a')),
(1, Dummy('ab')),
(2, Dummy('abc')),
(3, Dummy('abca')),
(4, Dummy('abcd')),
(5, Dummy('abce')),
(6, Dummy('abce')),
(7, Dummy(0)), # Collector #1959
(8, Dummy(None))]
self._forward = {}
self._backward = {}
for k, v in self._values:
self._backward[k] = v
keys = self._forward.get(v, [])
self._forward[v] = keys
self._noop_req = {'bar': 123}
self._request = {'foo': 'abce'}
self._min_req = {'foo':
{'query': 'abc', 'range': 'min'}}
self._min_req_n = {'foo':
{'query': 'abc', 'range': 'min', 'not': 'abca'}}
self._max_req = {'foo':
{'query': 'abc', 'range': 'max'}}
self._max_req_n = {'foo':
{'query': 'abc', 'range': 'max', 'not': ['a', 'b', None, 0]}}
self._range_req = {'foo':
{'query': ('abc', 'abcd'), 'range': 'min:max'}}
self._range_ren = {'foo':
{'query': ('abc', 'abcd'), 'range': 'min:max', 'not': 'abcd'}}
self._range_non = {'foo':
{'query': ('a', 'aa'), 'range': 'min:max', 'not': 'a'}}
self._zero_req = {'foo': 0 }
self._none_req = {'foo': None }
self._not_1 = {'foo': {'query': 'a', 'not': 'a'}}
self._not_2 = {'foo': {'query': ['a', 'ab'], 'not': 'a'}}
self._not_3 = {'foo': {'not': 'a'}}
self._not_4 = {'foo': {'not': [0, None]}}
self._not_5 = {'foo': {'not': ['a', 'b']}}
self._not_6 = {'foo': 'a', 'bar': {'query': 123, 'not': 1}}
def setUp(self):
self._catalog = self._makeOne()
index = FieldIndex('number')
self._catalog.addIndex('number', index)
self._catalog.addColumn('number')
for i in range(5000):
obj = objRS(random.randrange(0, 20000))
self._catalog.catalogObject(obj, i)
self._catalog = self._catalog.__of__(objRS(200))
def _make_many(self):
from Products.ZCatalog.Catalog import mergeResults
catalogs = []
for i in range(3):
cat = self._make_one()
cat.lexicon = PLexicon('lexicon')
cat.addIndex('num', FieldIndex('num'))
cat.addIndex('big', FieldIndex('big'))
cat.addIndex('number', FieldIndex('number'))
i = ZCTextIndex('title', caller=cat, index_factory=OkapiIndex,
lexicon_id='lexicon')
cat.addIndex('title', i)
cat = cat.__of__(ZDummy(16336))
for i in range(10):
obj = ZDummy(i)
obj.big = i > 5
obj.number = True
cat.catalogObject(obj, str(i))
catalogs.append(cat)
return catalogs, mergeResults
def _make_catalog(self):
zcat = ZCatalog('catalog')
zcat._catalog.addIndex('big', BooleanIndex('big'))
zcat._catalog.addIndex('date', DateRangeIndex('date', 'start', 'end'))
zcat._catalog.addIndex('num', FieldIndex('num'))
zcat._catalog.addIndex('numbers', KeywordIndex('numbers'))
zcat._catalog.addIndex('path', PathIndex('getPhysicalPath'))
zcat._catalog.addIndex('uuid', UUIDIndex('num'))
for i in range(9):
obj = Dummy(i)
zcat.catalog_object(obj, str(i))
return zcat
def testRange(self):
"""Test a range search"""
index = FieldIndex( 'foo' )
for i in range(100):
index.index_object(i, Dummy(i%10))
record = { 'foo' : { 'query' : [-99, 3]
, 'range' : 'min:max'
}
}
r=index._apply_index( record )
assert tuple(r[1])==('foo',), r[1]
r=list(r[0].keys())
expect=[
0, 1, 2, 3, 10, 11, 12, 13, 20, 21, 22, 23, 30, 31, 32, 33,
40, 41, 42, 43, 50, 51, 52, 53, 60, 61, 62, 63, 70, 71, 72, 73,
80, 81, 82, 83, 90, 91, 92, 93
]
assert r==expect, r
#
# Make sure that range tests with incompatible paramters
# don't return empty sets.
#
record[ 'foo' ][ 'operator' ] = 'and'
r2, ignore = index._apply_index( record )
r2 = list( r2.keys() )
assert r2 == r
def setUp(cls):
cls.app = makerequest(Zope2.app())
cls.app._setObject('Catalog', ZCatalog('Catalog'))
cls.app.Catalog.addIndex('meta_type', FieldIndex('meta_type'))
cls.app.Catalog.addColumn('id')
cls.app.Catalog.addColumn('title')
cls.app._setObject('Database', Folder('Database'))
# newly added objects have ._p_jar == None, initialize it
transaction.savepoint()
cache_size = cls.app._p_jar.db().getCacheSize()
cls._fill_catalog(cls.app, cls.app.Catalog, cache_size * 3)
# attach new persistent objects to ZODB connection
transaction.savepoint()
def test_catalogtool_index_patch(self):
mo = _MockObject(name='Unadapted', **{'testing_purposes': True})
fi1 = FieldIndex('testing_purposes')
fi2 = FieldIndex('name')
cat_tool = CatalogTool.CatalogTool('testcatalog', 'testcatalog')
# clear existent indexes
for name in cat_tool._catalog.indexes.keys():
cat_tool.delindex(name)
cat_tool._catalog.addIndex('testing_purposes', fi1)
cat_tool._catalog.addIndex('name', fi2)
cataloged = cat_tool._catalog.catalogObject(mo, 'mo_ui')
self.assertEqual(len(fi1.items()), 1)
self.assertEqual(len(fi2.items()), 1)
self.assertEqual((fi1.items()[0][0], fi2.items()[0][0]),
('Adapted_value', 'Unadapted'))
def setUp(self):
self._vocabulary = Vocabulary.Vocabulary('Vocabulary',
'Vocabulary',
globbing=1)
self._catalog = Catalog()
index = FieldIndex('number')
self._catalog.addIndex('number', index)
self._catalog.addColumn('number')
for i in range(5000):
obj = objRS(random.randrange(0, 20000))
self._catalog.catalogObject(obj, i)
self._catalog.aq_parent = objRS(200)
def installCatalogIndexes(context):
""" Catalog indexes
"""
if context.readDataFile('Products.EEAEnquiry.txt') is None:
return
portal = context.getSite()
catalog = getToolByName(portal, 'portal_enquiry_catalog')
try:
catalog._catalog.getIndex('portal_type')
catalog._catalog.getIndex('Title')
except KeyError:
logger.info("Adding indexes to catalog")
FieldIndex("portal_type")
class Empty(object):
""" Empty """
pass
elem = []
wordSplitter = Empty()
wordSplitter.group = 'Word Splitter'
wordSplitter.name = 'HTML aware splitter'
caseNormalizer = Empty()
caseNormalizer.group = 'Case Normalizer'
caseNormalizer.name = 'Case Normalizer'
stopWords = Empty()
stopWords.group = 'Stop Words'
stopWords.name = 'Remove listed and single char words'
elem.append(wordSplitter)
elem.append(caseNormalizer)
elem.append(stopWords)
catalog.manage_addProduct['ZCTextIndex'].manage_addLexicon(
'plone_lexicon', 'Default Lexicon', elem)
title_extras = Empty()
title_extras.doc_attr = 'Title'
title_extras.index_type = 'Okapi BM25 Rank'
title_extras.lexicon_id = 'plone_lexicon'
catalog.addIndex('portal_type', 'FieldIndex')
catalog.addIndex('Title', "ZCTextIndex", title_extras)
catalog.addColumn('portal_type')
catalog.addColumn('Title')
def _make_one(self):
from Products.ZCatalog.Catalog import Catalog
catalog = Catalog()
catalog.lexicon = PLexicon('lexicon')
att1 = FieldIndex('att1')
att2 = ZCTextIndex('att2', caller=catalog,
index_factory=OkapiIndex, lexicon_id='lexicon')
att3 = KeywordIndex('att3')
catalog.addIndex('att1', att1)
catalog.addIndex('att2', att2)
catalog.addIndex('att3', att3)
for x in range(0, self.upper):
catalog.catalogObject(Dummy(x), repr(x))
return catalog.__of__(Dummy('foo'))
def setUp(self):
self._catalog = self._makeOne()
self._catalog.lexicon = PLexicon('lexicon')
idx = ZCTextIndex('title', caller=self._catalog,
index_factory=OkapiIndex, lexicon_id='lexicon')
self._catalog.addIndex('title', idx)
self._catalog.addIndex('true', FieldIndex('true'))
self._catalog.addColumn('title')
cat = self._get_catalog()
for i in (1, 2, 3, 10, 11, 110, 111):
obj = zdummy(i)
obj.true = True
if i == 110:
obj.true = False
cat.catalogObject(obj, str(i))
def _make_one(self):
from Products.ZCatalog.Catalog import Catalog
catalog = Catalog()
catalog.lexicon = PLexicon('lexicon')
idx = ZCTextIndex('title', caller=catalog,
index_factory=OkapiIndex, lexicon_id='lexicon')
catalog.addIndex('title', idx)
catalog.addIndex('true', FieldIndex('true'))
catalog.addColumn('title')
for i in (1, 2, 3, 10, 11, 110, 111):
obj = ZDummy(i)
obj.true = True
if i == 110:
obj.true = False
catalog.catalogObject(obj, str(i))
return catalog.__of__(ZDummy(1))
def test_range_search(self):
catalog = self._make_one()
index = FieldIndex('number')
catalog.addIndex('number', index)
catalog.addColumn('number')
for i in range(50):
obj = objRS(random.randrange(0, 200))
catalog.catalogObject(obj, i)
catalog = catalog.__of__(objRS(20))
for i in range(10):
m = random.randrange(0, 200)
n = m + 10
for r in catalog(number={'query': (m, n), 'range': 'min:max'}):
size = r.number
self.assert_(m <= size and size <= n,
"%d vs [%d,%d]" % (r.number, m, n))
def setUp(self):
self._index = FieldIndex('foo')
self._marker = []
self._values = [(0, Dummy('a')),
(1, Dummy('ab')),
(2, Dummy('abc')),
(3, Dummy('abca')),
(4, Dummy('abcd')),
(5, Dummy('abce')),
(6, Dummy('abce')),
(7, Dummy(0)), # Collector #1959
(8, Dummy(None))]
self._forward = {}
self._backward = {}
for k, v in self._values:
self._backward[k] = v
keys = self._forward.get(v, [])
self._forward[v] = keys
self._noop_req = {'bar': 123}
self._request = {'foo': 'abce'}
self._min_req = {'foo':
{'query': 'abc', 'range': 'min'}}
self._min_req_n = {'foo':
{'query': 'abc', 'range': 'min', 'not': 'abca'}}
self._max_req = {'foo':
{'query': 'abc', 'range': 'max'}}
self._max_req_n = {'foo':
{'query': 'abc', 'range': 'max', 'not': ['a', 'b', None, 0]}}
self._range_req = {'foo':
{'query': ('abc', 'abcd'), 'range': 'min:max'}}
self._range_ren = {'foo':
{'query': ('abc', 'abcd'), 'range': 'min:max', 'not': 'abcd'}}
self._range_non = {'foo':
{'query': ('a', 'aa'), 'range': 'min:max', 'not': 'a'}}
self._zero_req = {'foo': 0 }
self._none_req = {'foo': None }
self._not_1 = {'foo': {'query': 'a', 'not': 'a'}}
self._not_2 = {'foo': {'query': ['a', 'ab'], 'not': 'a'}}
self._not_3 = {'foo': {'not': 'a'}}
self._not_4 = {'foo': {'not': [0, None]}}
self._not_5 = {'foo': {'not': ['a', 'b']}}
self._not_6 = {'foo': 'a', 'bar': {'query': 123, 'not': 1}}
def test_catalogtool_index_patch(self):
mo = _MockObject(name='Unadapted', **{'testing_purposes': True})
fi1 = FieldIndex('testing_purposes')
fi2 = FieldIndex('name')
cat_tool = CatalogTool.CatalogTool('testcatalog', 'testcatalog')
# clear existent indexes
for name in cat_tool._catalog.indexes.keys():
cat_tool.delindex(name)
cat_tool._catalog.addIndex('testing_purposes', fi1)
cat_tool._catalog.addIndex('name', fi2)
cataloged = cat_tool._catalog.catalogObject(mo, 'mo_ui')
self.assertEqual(len(fi1.items()), 1)
self.assertEqual(len(fi2.items()), 1)
self.assertEqual((fi1.items()[0][0], fi2.items()[0][0]),
('Adapted_value', 'Unadapted'))
def _make_catalog(self):
from Products.PluginIndexes.BooleanIndex.BooleanIndex import \
BooleanIndex
from Products.PluginIndexes.DateRangeIndex.DateRangeIndex import \
DateRangeIndex
from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
from Products.PluginIndexes.KeywordIndex.KeywordIndex import \
KeywordIndex
from Products.PluginIndexes.PathIndex.PathIndex import PathIndex
from Products.PluginIndexes.UUIDIndex.UUIDIndex import UUIDIndex
from Products.ZCatalog.ZCatalog import ZCatalog
zcat = ZCatalog('catalog')
zcat._catalog.addIndex('big', BooleanIndex('big'))
zcat._catalog.addIndex('date', DateRangeIndex('date', 'start', 'end'))
zcat._catalog.addIndex('num', FieldIndex('num'))
zcat._catalog.addIndex('numbers', KeywordIndex('numbers'))
zcat._catalog.addIndex('path', PathIndex('getPhysicalPath'))
zcat._catalog.addIndex('uuid', UUIDIndex('num'))
for i in range(9):
obj = dummy(i)
zcat.catalog_object(obj, str(i))
return zcat
def setUp(self):
self._index = FieldIndex("foo")
self._marker = []
self._values = [
(0, Dummy("a")),
(1, Dummy("ab")),
(2, Dummy("abc")),
(3, Dummy("abca")),
(4, Dummy("abcd")),
(5, Dummy("abce")),
(6, Dummy("abce")),
(7, Dummy(0)), # Collector #1959
(8, Dummy(None)),
]
self._forward = {}
self._backward = {}
for k, v in self._values:
self._backward[k] = v
keys = self._forward.get(v, [])
self._forward[v] = keys
self._noop_req = {"bar": 123}
self._request = {"foo": "abce"}
self._min_req = {"foo": {"query": "abc", "range": "min"}}
self._min_req_n = {"foo": {"query": "abc", "range": "min", "not": "abca"}}
self._max_req = {"foo": {"query": "abc", "range": "max"}}
self._max_req_n = {"foo": {"query": "abc", "range": "max", "not": ["a", "b", None, 0]}}
self._range_req = {"foo": {"query": ("abc", "abcd"), "range": "min:max"}}
self._range_ren = {"foo": {"query": ("abc", "abcd"), "range": "min:max", "not": "abcd"}}
self._range_non = {"foo": {"query": ("a", "aa"), "range": "min:max", "not": "a"}}
self._zero_req = {"foo": 0}
self._none_req = {"foo": None}
self._not_1 = {"foo": {"query": "a", "not": "a"}}
self._not_2 = {"foo": {"query": ["a", "ab"], "not": "a"}}
self._not_3 = {"foo": {"not": "a"}}
self._not_4 = {"foo": {"not": [0, None]}}
self._not_5 = {"foo": {"not": ["a", "b"]}}
self._not_6 = {"foo": "a", "bar": {"query": 123, "not": 1}}
class FieldIndexTests(unittest.TestCase):
"""Test FieldIndex objects.
"""
def setUp( self ):
"""
"""
self._index = FieldIndex( 'foo' )
self._marker = []
self._values = [ ( 0, Dummy( 'a' ) )
, ( 1, Dummy( 'ab' ) )
, ( 2, Dummy( 'abc' ) )
, ( 3, Dummy( 'abca' ) )
, ( 4, Dummy( 'abcd' ) )
, ( 5, Dummy( 'abce' ) )
, ( 6, Dummy( 'abce' ) )
, ( 7, Dummy( 0 ) ) # Collector #1959
, ( 8, Dummy(None) )]
self._forward = {}
self._backward = {}
for k, v in self._values:
self._backward[k] = v
keys = self._forward.get( v, [] )
self._forward[v] = keys
self._noop_req = { 'bar': 123 }
self._request = { 'foo': 'abce' }
self._min_req = { 'foo': {'query': 'abc'
, 'range': 'min'}
}
self._max_req = { 'foo': {'query': 'abc'
, 'range': 'max' }
}
self._range_req = { 'foo': {'query': ( 'abc', 'abcd' )
, 'range': 'min:max' }
}
self._zero_req = { 'foo': 0 }
self._none_req = { 'foo': None }
def tearDown( self ):
"""
"""
def _populateIndex( self ):
for k, v in self._values:
self._index.index_object( k, v )
def _checkApply( self, req, expectedValues ):
result, used = self._index._apply_index( req )
if hasattr(result, 'keys'):
result = result.keys()
assert used == ( 'foo', )
assert len( result ) == len( expectedValues ), \
'%s | %s' % ( map( None, result ), expectedValues )
for k, v in expectedValues:
assert k in result
def test_z3interfaces(self):
from Products.PluginIndexes.interfaces import IPluggableIndex
from Products.PluginIndexes.interfaces import ISortIndex
from Products.PluginIndexes.interfaces import IUniqueValueIndex
from zope.interface.verify import verifyClass
verifyClass(IPluggableIndex, FieldIndex)
verifyClass(ISortIndex, FieldIndex)
verifyClass(IUniqueValueIndex, FieldIndex)
def testEmpty( self ):
"Test an empty FieldIndex."
assert len( self._index ) == 0
assert len( self._index.referencedObjects() ) == 0
self.assertEqual(self._index.numObjects(), 0)
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker )
self._index.unindex_object( 1234 ) # nothrow
assert self._index.hasUniqueValuesFor( 'foo' )
assert not self._index.hasUniqueValuesFor( 'bar' )
assert len( self._index.uniqueValues( 'foo' ) ) == 0
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._request, [] )
self._checkApply( self._min_req, [] )
self._checkApply( self._max_req, [] )
self._checkApply( self._range_req, [] )
def testPopulated( self ):
""" Test a populated FieldIndex """
self._populateIndex()
values = self._values
assert len( self._index ) == len( values )-1 #'abce' is duplicate
assert len( self._index.referencedObjects() ) == len( values )
self.assertEqual(self._index.indexSize(), len( values )-1)
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker )
self._index.unindex_object( 1234 ) # nothrow
for k, v in values:
assert self._index.getEntryForObject( k ) == v.foo()
assert len( self._index.uniqueValues( 'foo' ) ) == len( values )-1
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._request, values[ -4:-2 ] )
self._checkApply( self._min_req, values[ 2:-2 ] )
self._checkApply( self._max_req, values[ :3 ] + values[ -2: ] )
self._checkApply( self._range_req, values[ 2:5 ] )
def testZero( self ):
""" Make sure 0 gets indexed """
self._populateIndex()
values = self._values
self._checkApply( self._zero_req, values[ -2:-1 ] )
assert 0 in self._index.uniqueValues( 'foo' )
def testNone(self):
""" make sure None gets indexed """
self._populateIndex()
values = self._values
self._checkApply(self._none_req, values[-1:])
assert None in self._index.uniqueValues('foo')
def testReindex( self ):
self._populateIndex()
result, used = self._index._apply_index( {'foo':'abc'} )
assert list(result)==[2]
assert self._index.keyForDocument(2)=='abc'
d = Dummy('world')
self._index.index_object(2,d)
result, used = self._index._apply_index( {'foo':'world'} )
assert list(result)==[2]
assert self._index.keyForDocument(2)=='world'
del d._foo
self._index.index_object(2,d)
result, used = self._index._apply_index( {'foo':'world'} )
assert list(result)==[]
try:
should_not_be = self._index.keyForDocument(2)
except KeyError:
# As expected, we deleted that attribute.
pass
else:
# before Collector #291 this would be 'world'
raise ValueError(repr(should_not_be))
def testRange(self):
"""Test a range search"""
index = FieldIndex( 'foo' )
for i in range(100):
index.index_object(i, Dummy(i%10))
record = { 'foo' : { 'query' : [-99, 3]
, 'range' : 'min:max'
}
}
r=index._apply_index( record )
assert tuple(r[1])==('foo',), r[1]
r=list(r[0].keys())
expect=[
0, 1, 2, 3, 10, 11, 12, 13, 20, 21, 22, 23, 30, 31, 32, 33,
40, 41, 42, 43, 50, 51, 52, 53, 60, 61, 62, 63, 70, 71, 72, 73,
80, 81, 82, 83, 90, 91, 92, 93
]
assert r==expect, r
#
# Make sure that range tests with incompatible paramters
# don't return empty sets.
#
record[ 'foo' ][ 'operator' ] = 'and'
r2, ignore = index._apply_index( record )
r2 = list( r2.keys() )
assert r2 == r
def testDelFieldIndex(self):
idx = FieldIndex('id')
self._catalog.addIndex('id', idx)
self._catalog.delIndex('id')
self.assert_('id' not in self._catalog.indexes,
'del index failed')
def testAddFieldIndex(self):
idx = FieldIndex('id')
self._catalog.addIndex('id', idx)
self.assert_(isinstance(self._catalog.indexes['id'],
type(FieldIndex('id'))),
'add field index failed')
class FieldIndexTests(unittest.TestCase):
"""Test FieldIndex objects.
"""
def setUp(self):
self._index = FieldIndex("foo")
self._marker = []
self._values = [
(0, Dummy("a")),
(1, Dummy("ab")),
(2, Dummy("abc")),
(3, Dummy("abca")),
(4, Dummy("abcd")),
(5, Dummy("abce")),
(6, Dummy("abce")),
(7, Dummy(0)), # Collector #1959
(8, Dummy(None)),
]
self._forward = {}
self._backward = {}
for k, v in self._values:
self._backward[k] = v
keys = self._forward.get(v, [])
self._forward[v] = keys
self._noop_req = {"bar": 123}
self._request = {"foo": "abce"}
self._min_req = {"foo": {"query": "abc", "range": "min"}}
self._min_req_n = {"foo": {"query": "abc", "range": "min", "not": "abca"}}
self._max_req = {"foo": {"query": "abc", "range": "max"}}
self._max_req_n = {"foo": {"query": "abc", "range": "max", "not": ["a", "b", None, 0]}}
self._range_req = {"foo": {"query": ("abc", "abcd"), "range": "min:max"}}
self._range_ren = {"foo": {"query": ("abc", "abcd"), "range": "min:max", "not": "abcd"}}
self._range_non = {"foo": {"query": ("a", "aa"), "range": "min:max", "not": "a"}}
self._zero_req = {"foo": 0}
self._none_req = {"foo": None}
self._not_1 = {"foo": {"query": "a", "not": "a"}}
self._not_2 = {"foo": {"query": ["a", "ab"], "not": "a"}}
self._not_3 = {"foo": {"not": "a"}}
self._not_4 = {"foo": {"not": [0, None]}}
self._not_5 = {"foo": {"not": ["a", "b"]}}
self._not_6 = {"foo": "a", "bar": {"query": 123, "not": 1}}
def _populateIndex(self):
for k, v in self._values:
self._index.index_object(k, v)
def _checkApply(self, req, expectedValues):
result, used = self._index._apply_index(req)
if hasattr(result, "keys"):
result = result.keys()
assert used == ("foo",)
assert len(result) == len(expectedValues), "%s | %s" % (map(None, result), expectedValues)
for k, v in expectedValues:
assert k in result
def test_interfaces(self):
from Products.PluginIndexes.interfaces import IPluggableIndex
from Products.PluginIndexes.interfaces import ISortIndex
from Products.PluginIndexes.interfaces import IUniqueValueIndex
from zope.interface.verify import verifyClass
verifyClass(IPluggableIndex, FieldIndex)
verifyClass(ISortIndex, FieldIndex)
verifyClass(IUniqueValueIndex, FieldIndex)
def testEmpty(self):
"Test an empty FieldIndex."
assert len(self._index) == 0
assert len(self._index.referencedObjects()) == 0
self.assertEqual(self._index.numObjects(), 0)
assert self._index.getEntryForObject(1234) is None
assert self._index.getEntryForObject(1234, self._marker) is self._marker
self._index.unindex_object(1234) # nothrow
assert self._index.hasUniqueValuesFor("foo")
assert not self._index.hasUniqueValuesFor("bar")
assert len(list(self._index.uniqueValues("foo"))) == 0
assert self._index._apply_index(self._noop_req) is None
self._checkApply(self._request, [])
self._checkApply(self._min_req, [])
self._checkApply(self._min_req_n, [])
self._checkApply(self._max_req, [])
self._checkApply(self._max_req_n, [])
self._checkApply(self._range_req, [])
self._checkApply(self._range_ren, [])
self._checkApply(self._range_non, [])
def testPopulated(self):
""" Test a populated FieldIndex """
self._populateIndex()
values = self._values
assert len(self._index) == len(values) - 1 # 'abce' is duplicate
assert len(self._index.referencedObjects()) == len(values)
self.assertEqual(self._index.indexSize(), len(values) - 1)
assert self._index.getEntryForObject(1234) is None
assert self._index.getEntryForObject(1234, self._marker) is self._marker
self._index.unindex_object(1234) # nothrow
for k, v in values:
assert self._index.getEntryForObject(k) == v.foo()
assert len(list(self._index.uniqueValues("foo"))) == len(values) - 1
assert self._index._apply_index(self._noop_req) is None
self._checkApply(self._request, values[-4:-2])
self._checkApply(self._min_req, values[2:-2])
self._checkApply(self._min_req_n, values[2:3] + values[4:-2])
self._checkApply(self._max_req, values[:3] + values[-2:])
self._checkApply(self._max_req_n, values[1:3])
self._checkApply(self._range_req, values[2:5])
self._checkApply(self._range_ren, values[2:4])
self._checkApply(self._range_non, [])
self._checkApply(self._not_1, [])
self._checkApply(self._not_2, values[1:2])
self._checkApply(self._not_3, values[1:])
self._checkApply(self._not_4, values[:7])
self._checkApply(self._not_5, values[1:])
self._checkApply(self._not_6, values[0:1])
def testZero(self):
""" Make sure 0 gets indexed """
self._populateIndex()
values = self._values
self._checkApply(self._zero_req, values[-2:-1])
assert 0 in self._index.uniqueValues("foo")
def testNone(self):
""" make sure None gets indexed """
self._populateIndex()
values = self._values
self._checkApply(self._none_req, values[-1:])
assert None in self._index.uniqueValues("foo")
def testReindex(self):
self._populateIndex()
result, used = self._index._apply_index({"foo": "abc"})
assert list(result) == [2]
assert self._index.keyForDocument(2) == "abc"
d = Dummy("world")
self._index.index_object(2, d)
result, used = self._index._apply_index({"foo": "world"})
assert list(result) == [2]
assert self._index.keyForDocument(2) == "world"
del d._foo
self._index.index_object(2, d)
result, used = self._index._apply_index({"foo": "world"})
assert list(result) == []
try:
should_not_be = self._index.keyForDocument(2)
except KeyError:
# As expected, we deleted that attribute.
pass
else:
# before Collector #291 this would be 'world'
raise ValueError(repr(should_not_be))
def testRange(self):
"""Test a range search"""
index = FieldIndex("foo")
for i in range(100):
index.index_object(i, Dummy(i % 10))
record = {"foo": {"query": [-99, 3], "range": "min:max"}}
r = index._apply_index(record)
assert tuple(r[1]) == ("foo",), r[1]
r = list(r[0].keys())
expect = [
0,
1,
2,
3,
10,
11,
12,
13,
20,
21,
22,
23,
30,
31,
32,
33,
40,
41,
42,
43,
50,
51,
52,
53,
60,
61,
62,
63,
70,
71,
72,
73,
80,
81,
82,
83,
90,
91,
92,
93,
]
assert r == expect, r
# Make sure that range tests with incompatible paramters
# don't return empty sets.
record["foo"]["operator"] = "and"
r2, ignore = index._apply_index(record)
r2 = list(r2.keys())
assert r2 == r
def testRange(self):
"""Test a range search"""
index = FieldIndex("foo")
for i in range(100):
index.index_object(i, Dummy(i % 10))
record = {"foo": {"query": [-99, 3], "range": "min:max"}}
r = index._apply_index(record)
assert tuple(r[1]) == ("foo",), r[1]
r = list(r[0].keys())
expect = [
0,
1,
2,
3,
10,
11,
12,
13,
20,
21,
22,
23,
30,
31,
32,
33,
40,
41,
42,
43,
50,
51,
52,
53,
60,
61,
62,
63,
70,
71,
72,
73,
80,
81,
82,
83,
90,
91,
92,
93,
]
assert r == expect, r
# Make sure that range tests with incompatible paramters
# don't return empty sets.
record["foo"]["operator"] = "and"
r2, ignore = index._apply_index(record)
r2 = list(r2.keys())
assert r2 == r
