def test_contains(self):
mapper = UUIDMapper()
uid, docid = uuid.uuid4(), 12345
mapper.add(uid, docid)
assert uid in mapper # normalized UUID->str
assert str(uid) in mapper
assert docid in mapper
def test_contains(self):
mapper = UUIDMapper()
uid, docid = uuid.uuid4(), 12345
mapper.add(uid, docid)
assert uid in mapper # normalized UUID->str
assert str(uid) in mapper
assert docid in mapper
def __init__(self, context, schema=None):
self._context_uid = IUUID(context)
if schema is None:
schema = getattr(context, 'schema', None)
if schema is None:
raise ValueError('Context does not provide schema')
self.indexer = Indexer()
self.uidmap = UUIDMapper()
self.bind(schema)
def __init__(self, context, schema=None):
self._context_uid = IUUID(context)
if schema is None:
schema = getattr(context, 'schema', None)
if schema is None:
raise ValueError('Context does not provide schema')
self.indexer = Indexer()
self.uidmap = UUIDMapper()
self.bind(schema)
def test_add_remove(self):
"""Test add/remove and containment/get/length"""
mapper = UUIDMapper()
# add with pre-calculated docid
uid, docid = uuid.uuid4(), 12345
rv = mapper.add(uid, docid)
assert uid in mapper
assert rv == (str(uid), docid)
assert mapper.get(uid) == docid
assert len(mapper) == 1
# Cannot add duplicate:
self.assertRaises(KeyError, mapper.add, uid)
self.assertRaises(KeyError, mapper.add, str(uid))
# add with generation of docid
uid2 = uuid.uuid4()
rv = mapper.add(uid2)
assert len(mapper) == 2
docid2 = rv[1]
assert rv == (str(uid2), docid2)
assert uid2 in mapper
assert docid2 in mapper
assert mapper.get(uid2) == docid2
assert mapper.get(docid2) == str(uid2)
# remove one by UUID
mapper.remove(uid2)
assert uid2 not in mapper
assert docid2 not in mapper
assert len(mapper) == 1
mapper.remove(docid)
assert docid not in mapper
assert uid not in mapper
assert str(uid) not in mapper
assert mapper.get(uid, None) is None
assert len(mapper) == 0
# re-add okay:
mapper.add(uid, docid)
assert uid in mapper and docid in mapper
assert len(mapper) == 1
def test_enumeration(self):
"""test enumeration and iteration"""
_uids = []
mapper = UUIDMapper()
# add ten random pairs:
for i in range(10):
uid = uuid.uuid4()
_uids.append(str(uid))
mapper.add(uid)
assert len(mapper) == 10 == len(mapper.keys())
assert len(mapper.items()) == len(mapper.values()) == 10
assert len(list(mapper.iteritems())) == 10
for uid in _uids:
assert uid in mapper
assert uid in mapper.keys()
assert uid in mapper.iterkeys()
assert mapper.get(uid) in mapper.values()
assert mapper.get(uid) in mapper.itervalues()
assert (uid, mapper.get(uid)) in mapper.items()
assert (uid, mapper.get(uid)) in mapper.iteritems()
class SimpleCatalog(Persistent):
"""
Simple catalog for items sharing a common single search schema,
and for which items are resolved from a single container which
is a content item.
Items are externally referenced and results are keyed by UUID.
"""
implements(ISimpleCatalog)
def __init__(self, context, schema=None):
self._context_uid = IUUID(context)
if schema is None:
schema = getattr(context, 'schema', None)
if schema is None:
raise ValueError('Context does not provide schema')
self.indexer = Indexer()
self.uidmap = UUIDMapper()
self.bind(schema)
## ILocation implementation:
__name__ = 'simple_catalog'
@property
def __parent__(self):
return self.resolver.context # based on UID of container content
## ISearchContext properties:
@property
def resolver(self):
if not getattr(self, '_v_resolver', None):
self._v_resolver = ContentContainerUIDResolver(self._context_uid)
return self._v_resolver
## ISimpleCatalog indexing methods:
def bind(self, schema):
if hasattr(self, '_v_schema'):
delattr(self, '_v_schema')
self._schema = identify_interface(schema)
self.make_indexes()
def _search_schema(self):
if not getattr(self, '_v_schema', None):
self._v_schema = resolve(self._schema)
return self._v_schema
search_schema = property(_search_schema, bind)
def indexes(self):
"""index names per schema"""
return ISchemaIndexes(self.search_schema, ())
def make_indexes(self):
names = self.indexes()
for name in names:
idx_type = name.split('_')[0]
fieldname = name[(len(idx_type) + 1):]
field = self.search_schema[fieldname]
## need a persistent callable discriminator to support value
## normalization, it is the only way to have a callable
## discriminator that is anonymous (not importable) that
## works around limitations in ZODB/pickle.
discriminator = fieldname
if idx_type != 'text':
discriminator = ValueDiscriminator(field)
self.indexer[name] = IDXCLS.get(idx_type)(discriminator)
def index(self, obj):
uid = IUUID(obj)
uid, docid = self.uidmap.add(uid)
self.indexer.index_doc(docid, obj)
def unindex(self, obj):
if isinstance(obj, str):
uid = obj
else:
uid = IUUID(obj)
if uid not in self.uidmap:
raise KeyError(uid)
docid = self.uidmap.docid_for(uid)
self.indexer.unindex_doc(docid)
self.uidmap.remove(uid)
def reindex(self, obj=None):
if obj is None:
for uid, docid in self.uidmap.iteritems():
self.reindex(obj=uid)
else:
if isinstance(obj, str):
uid = obj
obj = self.get(uid)
if obj is None:
self.unindex(uid) # stale entry, now gone
return
else:
uid = IUUID(obj)
docid = self.uidmap.docid_for(uid)
self.indexer.redindex_doc(docid, obj)
## ISearchContext base mapping methods:
def __len__(self):
return len(self.uidmap)
def get(self, key, default=None):
uid = key
if isinstance(key, int) or isinstance(key, long):
uid = self.uidmap.equivalent(key)
v = self.resolver(uid)
if v is None:
return default
return v
def __getitem__(self, key):
v = self.get(key, None)
if v is None:
raise KeyError(key)
return v
def __contains__(self, spec):
uid = spec
if not isinstance(spec, str):
uid = IUUID(spec, None)
if uid is None:
uid = normalize_uuid(spec)
if uid is None:
return False
return uid in self.uidmap
def iterkeys(self):
return self.uidmap.iterkeys() # UIDs, not docids
__iter__ = iterkeys
def itervalues(self):
return (self.get(uid) for uid in self.iterkeys())
def iteritems(self):
return ((uid, self.get(uid)) for uid in self.iterkeys())
def keys(self):
return list(self.iterkeys())
def values(self):
return list(self.itervalues())
def items(self):
return list(self.iteritems())
## IItemCollection
def byuid(self):
return self
def byname(self):
return self # technically, we don't map local ids
## ISimpleCatalog query methods:
def _query_from_mapping(self, qdict):
"""
return a query.Query object given mapping of keys/values.
Value normalization is not in scope (should happen to
resulting query).
"""
r = []
for idxname, value in qdict.items():
if isinstance(value, tuple) and len(value) > 1:
if issubclass(value[0], query.Query):
comparator = value[0]
r.append(comparator(idxname, value[1]))
continue
if idxname.startswith('text'):
r.append(query.Contains(idxname, value))
elif idxname.startswith('keyword'):
r.append(query.Any(idxname, value))
else:
r.append(query.Eq(idxname, value))
if len(r) == 1:
return r[0]
return query.And(*r)
def _make_result(self, result):
"""
Given a result as tuple of length, integer docids,
construct a search result keyed by UUID.
"""
size, docids = result # unpack, but we do not care about size
t = tuple((docid, self.uidmap.uuid_for(docid))
for docid in docids) # iterate into pairs of docid, uid
result = SearchResult.fromtuples(t, resolver=self.resolver)
result.__parent__ = self
result.__name__ = 'result'
return result
def query(self, *args, **kwargs):
qdict = None
if not args and kwargs:
qdict = kwargs
elif args and hasattr(args[0], 'iteritems'):
## looks like mapping/dict
qdict = dict(args[0].items())
elif not args:
raise ValueError('Empty query')
else:
_query = args[0]
if not isinstance(_query, query.Query):
raise ValueError('Invalid query')
if qdict:
_query = self._query_from_mapping(qdict)
if kwargs.get('return_query_result_count', False):
return self.indexer.query(_query)[0]
normalize_query(_query) # normalize values recursively in-place
return self._make_result(self.indexer.query(_query))
def rcount(self, *args, **kwargs):
kwargs['return_query_result_count'] = True
return self.query(*args, **kwargs)
__call__ = query
def test_add_remove(self):
"""Test add/remove and containment/get/length"""
mapper = UUIDMapper()
# add with pre-calculated docid
uid, docid = uuid.uuid4(), 12345
rv = mapper.add(uid, docid)
assert uid in mapper
assert rv == (str(uid), docid)
assert mapper.get(uid) == docid
assert len(mapper) == 1
# Cannot add duplicate:
self.assertRaises(KeyError, mapper.add, uid)
self.assertRaises(KeyError, mapper.add, str(uid))
# add with generation of docid
uid2 = uuid.uuid4()
rv = mapper.add(uid2)
assert len(mapper) == 2
docid2 = rv[1]
assert rv == (str(uid2), docid2)
assert uid2 in mapper
assert docid2 in mapper
assert mapper.get(uid2) == docid2
assert mapper.get(docid2) == str(uid2)
# remove one by UUID
mapper.remove(uid2)
assert uid2 not in mapper
assert docid2 not in mapper
assert len(mapper) == 1
mapper.remove(docid)
assert docid not in mapper
assert uid not in mapper
assert str(uid) not in mapper
assert mapper.get(uid, None) is None
assert len(mapper) == 0
# re-add okay:
mapper.add(uid, docid)
assert uid in mapper and docid in mapper
assert len(mapper) == 1
def test_btrees(self):
mapper = UUIDMapper()
from BTrees.LOBTree import LOBTree
from BTrees.OLBTree import OLBTree
self.assertIsInstance(mapper.uuid_to_docid, OLBTree)
self.assertIsInstance(mapper.docid_to_uuid, LOBTree)
def test_enumeration(self):
"""test enumeration and iteration"""
_uids = []
mapper = UUIDMapper()
# add ten random pairs:
for i in range(10):
uid = uuid.uuid4()
_uids.append(str(uid))
mapper.add(uid)
assert len(mapper) == 10 == len(mapper.keys())
assert len(mapper.items()) == len(mapper.values()) == 10
assert len(list(mapper.iteritems())) == 10
for uid in _uids:
assert uid in mapper
assert uid in mapper.keys()
assert uid in mapper.iterkeys()
assert mapper.get(uid) in mapper.values()
assert mapper.get(uid) in mapper.itervalues()
assert (uid, mapper.get(uid)) in mapper.items()
assert (uid, mapper.get(uid)) in mapper.iteritems()
class SimpleCatalog(Persistent):
"""
Simple catalog for items sharing a common single search schema,
and for which items are resolved from a single container which
is a content item.
Items are externally referenced and results are keyed by UUID.
"""
implements(ISimpleCatalog)
def __init__(self, context, schema=None):
self._context_uid = IUUID(context)
if schema is None:
schema = getattr(context, 'schema', None)
if schema is None:
raise ValueError('Context does not provide schema')
self.indexer = Indexer()
self.uidmap = UUIDMapper()
self.bind(schema)
## ILocation implementation:
__name__ = 'simple_catalog'
@property
def __parent__(self):
return self.resolver.context # based on UID of container content
## ISearchContext properties:
@property
def resolver(self):
if not getattr(self, '_v_resolver', None):
self._v_resolver = ContentContainerUIDResolver(self._context_uid)
return self._v_resolver
## ISimpleCatalog indexing methods:
def bind(self, schema):
if hasattr(self, '_v_schema'):
delattr(self, '_v_schema')
self._schema = identify_interface(schema)
self.make_indexes()
def _search_schema(self):
if not getattr(self, '_v_schema', None):
self._v_schema = resolve(self._schema)
return self._v_schema
search_schema = property(_search_schema, bind)
def indexes(self):
"""index names per schema"""
return ISchemaIndexes(self.search_schema, ())
def make_indexes(self):
names = self.indexes()
for name in names:
idx_type = name.split('_')[0]
fieldname = name[(len(idx_type) + 1):]
field = self.search_schema[fieldname]
## need a persistent callable discriminator to support value
## normalization, it is the only way to have a callable
## discriminator that is anonymous (not importable) that
## works around limitations in ZODB/pickle.
discriminator = fieldname
if idx_type != 'text':
discriminator = ValueDiscriminator(field)
self.indexer[name] = IDXCLS.get(idx_type)(discriminator)
def index(self, obj):
uid = IUUID(obj)
uid, docid = self.uidmap.add(uid)
self.indexer.index_doc(docid, obj)
def unindex(self, obj):
if isinstance(obj, str):
uid = obj
else:
uid = IUUID(obj)
if uid not in self.uidmap:
raise KeyError(uid)
docid = self.uidmap.docid_for(uid)
self.indexer.unindex_doc(docid)
self.uidmap.remove(uid)
def reindex(self, obj=None):
if obj is None:
for uid, docid in self.uidmap.iteritems():
self.reindex(obj=uid)
else:
if isinstance(obj, str):
uid = obj
obj = self.get(uid)
if obj is None:
self.unindex(uid) # stale entry, now gone
return
else:
uid = IUUID(obj)
docid = self.uidmap.docid_for(uid)
self.indexer.redindex_doc(docid, obj)
## ISearchContext base mapping methods:
def __len__(self):
return len(self.uidmap)
def get(self, key, default=None):
uid = key
if isinstance(key, int) or isinstance(key, long):
uid = self.uidmap.equivalent(key)
v = self.resolver(uid)
if v is None:
return default
return v
def __getitem__(self, key):
v = self.get(key, None)
if v is None:
raise KeyError(key)
return v
def __contains__(self, spec):
uid = spec
if not isinstance(spec, str):
uid = IUUID(spec, None)
if uid is None:
uid = normalize_uuid(spec)
if uid is None:
return False
return uid in self.uidmap
def iterkeys(self):
return self.uidmap.iterkeys() # UIDs, not docids
__iter__ = iterkeys
def itervalues(self):
return (self.get(uid) for uid in self.iterkeys())
def iteritems(self):
return ((uid, self.get(uid)) for uid in self.iterkeys())
def keys(self):
return list(self.iterkeys())
def values(self):
return list(self.itervalues())
def items(self):
return list(self.iteritems())
## IItemCollection
def byuid(self):
return self
def byname(self):
return self # technically, we don't map local ids
## ISimpleCatalog query methods:
def _query_from_mapping(self, qdict):
"""
return a query.Query object given mapping of keys/values.
Value normalization is not in scope (should happen to
resulting query).
"""
r = []
for idxname, value in qdict.items():
if isinstance(value, tuple) and len(value) > 1:
if issubclass(value[0], query.Query):
comparator = value[0]
r.append(comparator(idxname, value[1]))
continue
if idxname.startswith('text'):
r.append(query.Contains(idxname, value))
elif idxname.startswith('keyword'):
r.append(query.Any(idxname, value))
else:
r.append(query.Eq(idxname, value))
if len(r) == 1:
return r[0]
return query.And(*r)
def _make_result(self, result):
"""
Given a result as tuple of length, integer docids,
construct a search result keyed by UUID.
"""
size, docids = result # unpack, but we do not care about size
t = tuple(
(docid, self.uidmap.uuid_for(docid)) for docid in docids
) # iterate into pairs of docid, uid
result = SearchResult.fromtuples(t, resolver=self.resolver)
result.__parent__ = self
result.__name__ = 'result'
return result
def query(self, *args, **kwargs):
qdict = None
if not args and kwargs:
qdict = kwargs
elif args and hasattr(args[0], 'iteritems'):
## looks like mapping/dict
qdict = dict(args[0].items())
elif not args:
raise ValueError('Empty query')
else:
_query = args[0]
if not isinstance(_query, query.Query):
raise ValueError('Invalid query')
if qdict:
_query = self._query_from_mapping(qdict)
if kwargs.get('return_query_result_count', False):
return self.indexer.query(_query)[0]
normalize_query(_query) # normalize values recursively in-place
return self._make_result(self.indexer.query(_query))
def rcount(self, *args, **kwargs):
kwargs['return_query_result_count'] = True
return self.query(*args, **kwargs)
__call__ = query