def test_contains(self): mapper = UUIDMapper() uid, docid = uuid.uuid4(), 12345 mapper.add(uid, docid) assert uid in mapper # normalized UUID->str assert str(uid) in mapper assert docid in mapper
def test_contains(self): mapper = UUIDMapper() uid, docid = uuid.uuid4(), 12345 mapper.add(uid, docid) assert uid in mapper # normalized UUID->str assert str(uid) in mapper assert docid in mapper
def test_enumeration(self): """test enumeration and iteration""" _uids = [] mapper = UUIDMapper() # add ten random pairs: for i in range(10): uid = uuid.uuid4() _uids.append(str(uid)) mapper.add(uid) assert len(mapper) == 10 == len(mapper.keys()) assert len(mapper.items()) == len(mapper.values()) == 10 assert len(list(mapper.iteritems())) == 10 for uid in _uids: assert uid in mapper assert uid in mapper.keys() assert uid in mapper.iterkeys() assert mapper.get(uid) in mapper.values() assert mapper.get(uid) in mapper.itervalues() assert (uid, mapper.get(uid)) in mapper.items() assert (uid, mapper.get(uid)) in mapper.iteritems()
def test_enumeration(self): """test enumeration and iteration""" _uids = [] mapper = UUIDMapper() # add ten random pairs: for i in range(10): uid = uuid.uuid4() _uids.append(str(uid)) mapper.add(uid) assert len(mapper) == 10 == len(mapper.keys()) assert len(mapper.items()) == len(mapper.values()) == 10 assert len(list(mapper.iteritems())) == 10 for uid in _uids: assert uid in mapper assert uid in mapper.keys() assert uid in mapper.iterkeys() assert mapper.get(uid) in mapper.values() assert mapper.get(uid) in mapper.itervalues() assert (uid, mapper.get(uid)) in mapper.items() assert (uid, mapper.get(uid)) in mapper.iteritems()
def test_add_remove(self): """Test add/remove and containment/get/length""" mapper = UUIDMapper() # add with pre-calculated docid uid, docid = uuid.uuid4(), 12345 rv = mapper.add(uid, docid) assert uid in mapper assert rv == (str(uid), docid) assert mapper.get(uid) == docid assert len(mapper) == 1 # Cannot add duplicate: self.assertRaises(KeyError, mapper.add, uid) self.assertRaises(KeyError, mapper.add, str(uid)) # add with generation of docid uid2 = uuid.uuid4() rv = mapper.add(uid2) assert len(mapper) == 2 docid2 = rv[1] assert rv == (str(uid2), docid2) assert uid2 in mapper assert docid2 in mapper assert mapper.get(uid2) == docid2 assert mapper.get(docid2) == str(uid2) # remove one by UUID mapper.remove(uid2) assert uid2 not in mapper assert docid2 not in mapper assert len(mapper) == 1 mapper.remove(docid) assert docid not in mapper assert uid not in mapper assert str(uid) not in mapper assert mapper.get(uid, None) is None assert len(mapper) == 0 # re-add okay: mapper.add(uid, docid) assert uid in mapper and docid in mapper assert len(mapper) == 1
def test_add_remove(self): """Test add/remove and containment/get/length""" mapper = UUIDMapper() # add with pre-calculated docid uid, docid = uuid.uuid4(), 12345 rv = mapper.add(uid, docid) assert uid in mapper assert rv == (str(uid), docid) assert mapper.get(uid) == docid assert len(mapper) == 1 # Cannot add duplicate: self.assertRaises(KeyError, mapper.add, uid) self.assertRaises(KeyError, mapper.add, str(uid)) # add with generation of docid uid2 = uuid.uuid4() rv = mapper.add(uid2) assert len(mapper) == 2 docid2 = rv[1] assert rv == (str(uid2), docid2) assert uid2 in mapper assert docid2 in mapper assert mapper.get(uid2) == docid2 assert mapper.get(docid2) == str(uid2) # remove one by UUID mapper.remove(uid2) assert uid2 not in mapper assert docid2 not in mapper assert len(mapper) == 1 mapper.remove(docid) assert docid not in mapper assert uid not in mapper assert str(uid) not in mapper assert mapper.get(uid, None) is None assert len(mapper) == 0 # re-add okay: mapper.add(uid, docid) assert uid in mapper and docid in mapper assert len(mapper) == 1
class SimpleCatalog(Persistent): """ Simple catalog for items sharing a common single search schema, and for which items are resolved from a single container which is a content item. Items are externally referenced and results are keyed by UUID. """ implements(ISimpleCatalog) def __init__(self, context, schema=None): self._context_uid = IUUID(context) if schema is None: schema = getattr(context, 'schema', None) if schema is None: raise ValueError('Context does not provide schema') self.indexer = Indexer() self.uidmap = UUIDMapper() self.bind(schema) ## ILocation implementation: __name__ = 'simple_catalog' @property def __parent__(self): return self.resolver.context # based on UID of container content ## ISearchContext properties: @property def resolver(self): if not getattr(self, '_v_resolver', None): self._v_resolver = ContentContainerUIDResolver(self._context_uid) return self._v_resolver ## ISimpleCatalog indexing methods: def bind(self, schema): if hasattr(self, '_v_schema'): delattr(self, '_v_schema') self._schema = identify_interface(schema) self.make_indexes() def _search_schema(self): if not getattr(self, '_v_schema', None): self._v_schema = resolve(self._schema) return self._v_schema search_schema = property(_search_schema, bind) def indexes(self): """index names per schema""" return ISchemaIndexes(self.search_schema, ()) def make_indexes(self): names = self.indexes() for name in names: idx_type = name.split('_')[0] fieldname = name[(len(idx_type) + 1):] field = self.search_schema[fieldname] ## need a persistent callable discriminator to support value ## normalization, it is the only way to have a callable ## discriminator that is anonymous (not importable) that ## works around limitations in ZODB/pickle. discriminator = fieldname if idx_type != 'text': discriminator = ValueDiscriminator(field) self.indexer[name] = IDXCLS.get(idx_type)(discriminator) def index(self, obj): uid = IUUID(obj) uid, docid = self.uidmap.add(uid) self.indexer.index_doc(docid, obj) def unindex(self, obj): if isinstance(obj, str): uid = obj else: uid = IUUID(obj) if uid not in self.uidmap: raise KeyError(uid) docid = self.uidmap.docid_for(uid) self.indexer.unindex_doc(docid) self.uidmap.remove(uid) def reindex(self, obj=None): if obj is None: for uid, docid in self.uidmap.iteritems(): self.reindex(obj=uid) else: if isinstance(obj, str): uid = obj obj = self.get(uid) if obj is None: self.unindex(uid) # stale entry, now gone return else: uid = IUUID(obj) docid = self.uidmap.docid_for(uid) self.indexer.redindex_doc(docid, obj) ## ISearchContext base mapping methods: def __len__(self): return len(self.uidmap) def get(self, key, default=None): uid = key if isinstance(key, int) or isinstance(key, long): uid = self.uidmap.equivalent(key) v = self.resolver(uid) if v is None: return default return v def __getitem__(self, key): v = self.get(key, None) if v is None: raise KeyError(key) return v def __contains__(self, spec): uid = spec if not isinstance(spec, str): uid = IUUID(spec, None) if uid is None: uid = normalize_uuid(spec) if uid is None: return False return uid in self.uidmap def iterkeys(self): return self.uidmap.iterkeys() # UIDs, not docids __iter__ = iterkeys def itervalues(self): return (self.get(uid) for uid in self.iterkeys()) def iteritems(self): return ((uid, self.get(uid)) for uid in self.iterkeys()) def keys(self): return list(self.iterkeys()) def values(self): return list(self.itervalues()) def items(self): return list(self.iteritems()) ## IItemCollection def byuid(self): return self def byname(self): return self # technically, we don't map local ids ## ISimpleCatalog query methods: def _query_from_mapping(self, qdict): """ return a query.Query object given mapping of keys/values. Value normalization is not in scope (should happen to resulting query). """ r = [] for idxname, value in qdict.items(): if isinstance(value, tuple) and len(value) > 1: if issubclass(value[0], query.Query): comparator = value[0] r.append(comparator(idxname, value[1])) continue if idxname.startswith('text'): r.append(query.Contains(idxname, value)) elif idxname.startswith('keyword'): r.append(query.Any(idxname, value)) else: r.append(query.Eq(idxname, value)) if len(r) == 1: return r[0] return query.And(*r) def _make_result(self, result): """ Given a result as tuple of length, integer docids, construct a search result keyed by UUID. """ size, docids = result # unpack, but we do not care about size t = tuple((docid, self.uidmap.uuid_for(docid)) for docid in docids) # iterate into pairs of docid, uid result = SearchResult.fromtuples(t, resolver=self.resolver) result.__parent__ = self result.__name__ = 'result' return result def query(self, *args, **kwargs): qdict = None if not args and kwargs: qdict = kwargs elif args and hasattr(args[0], 'iteritems'): ## looks like mapping/dict qdict = dict(args[0].items()) elif not args: raise ValueError('Empty query') else: _query = args[0] if not isinstance(_query, query.Query): raise ValueError('Invalid query') if qdict: _query = self._query_from_mapping(qdict) if kwargs.get('return_query_result_count', False): return self.indexer.query(_query)[0] normalize_query(_query) # normalize values recursively in-place return self._make_result(self.indexer.query(_query)) def rcount(self, *args, **kwargs): kwargs['return_query_result_count'] = True return self.query(*args, **kwargs) __call__ = query
class SimpleCatalog(Persistent): """ Simple catalog for items sharing a common single search schema, and for which items are resolved from a single container which is a content item. Items are externally referenced and results are keyed by UUID. """ implements(ISimpleCatalog) def __init__(self, context, schema=None): self._context_uid = IUUID(context) if schema is None: schema = getattr(context, 'schema', None) if schema is None: raise ValueError('Context does not provide schema') self.indexer = Indexer() self.uidmap = UUIDMapper() self.bind(schema) ## ILocation implementation: __name__ = 'simple_catalog' @property def __parent__(self): return self.resolver.context # based on UID of container content ## ISearchContext properties: @property def resolver(self): if not getattr(self, '_v_resolver', None): self._v_resolver = ContentContainerUIDResolver(self._context_uid) return self._v_resolver ## ISimpleCatalog indexing methods: def bind(self, schema): if hasattr(self, '_v_schema'): delattr(self, '_v_schema') self._schema = identify_interface(schema) self.make_indexes() def _search_schema(self): if not getattr(self, '_v_schema', None): self._v_schema = resolve(self._schema) return self._v_schema search_schema = property(_search_schema, bind) def indexes(self): """index names per schema""" return ISchemaIndexes(self.search_schema, ()) def make_indexes(self): names = self.indexes() for name in names: idx_type = name.split('_')[0] fieldname = name[(len(idx_type) + 1):] field = self.search_schema[fieldname] ## need a persistent callable discriminator to support value ## normalization, it is the only way to have a callable ## discriminator that is anonymous (not importable) that ## works around limitations in ZODB/pickle. discriminator = fieldname if idx_type != 'text': discriminator = ValueDiscriminator(field) self.indexer[name] = IDXCLS.get(idx_type)(discriminator) def index(self, obj): uid = IUUID(obj) uid, docid = self.uidmap.add(uid) self.indexer.index_doc(docid, obj) def unindex(self, obj): if isinstance(obj, str): uid = obj else: uid = IUUID(obj) if uid not in self.uidmap: raise KeyError(uid) docid = self.uidmap.docid_for(uid) self.indexer.unindex_doc(docid) self.uidmap.remove(uid) def reindex(self, obj=None): if obj is None: for uid, docid in self.uidmap.iteritems(): self.reindex(obj=uid) else: if isinstance(obj, str): uid = obj obj = self.get(uid) if obj is None: self.unindex(uid) # stale entry, now gone return else: uid = IUUID(obj) docid = self.uidmap.docid_for(uid) self.indexer.redindex_doc(docid, obj) ## ISearchContext base mapping methods: def __len__(self): return len(self.uidmap) def get(self, key, default=None): uid = key if isinstance(key, int) or isinstance(key, long): uid = self.uidmap.equivalent(key) v = self.resolver(uid) if v is None: return default return v def __getitem__(self, key): v = self.get(key, None) if v is None: raise KeyError(key) return v def __contains__(self, spec): uid = spec if not isinstance(spec, str): uid = IUUID(spec, None) if uid is None: uid = normalize_uuid(spec) if uid is None: return False return uid in self.uidmap def iterkeys(self): return self.uidmap.iterkeys() # UIDs, not docids __iter__ = iterkeys def itervalues(self): return (self.get(uid) for uid in self.iterkeys()) def iteritems(self): return ((uid, self.get(uid)) for uid in self.iterkeys()) def keys(self): return list(self.iterkeys()) def values(self): return list(self.itervalues()) def items(self): return list(self.iteritems()) ## IItemCollection def byuid(self): return self def byname(self): return self # technically, we don't map local ids ## ISimpleCatalog query methods: def _query_from_mapping(self, qdict): """ return a query.Query object given mapping of keys/values. Value normalization is not in scope (should happen to resulting query). """ r = [] for idxname, value in qdict.items(): if isinstance(value, tuple) and len(value) > 1: if issubclass(value[0], query.Query): comparator = value[0] r.append(comparator(idxname, value[1])) continue if idxname.startswith('text'): r.append(query.Contains(idxname, value)) elif idxname.startswith('keyword'): r.append(query.Any(idxname, value)) else: r.append(query.Eq(idxname, value)) if len(r) == 1: return r[0] return query.And(*r) def _make_result(self, result): """ Given a result as tuple of length, integer docids, construct a search result keyed by UUID. """ size, docids = result # unpack, but we do not care about size t = tuple( (docid, self.uidmap.uuid_for(docid)) for docid in docids ) # iterate into pairs of docid, uid result = SearchResult.fromtuples(t, resolver=self.resolver) result.__parent__ = self result.__name__ = 'result' return result def query(self, *args, **kwargs): qdict = None if not args and kwargs: qdict = kwargs elif args and hasattr(args[0], 'iteritems'): ## looks like mapping/dict qdict = dict(args[0].items()) elif not args: raise ValueError('Empty query') else: _query = args[0] if not isinstance(_query, query.Query): raise ValueError('Invalid query') if qdict: _query = self._query_from_mapping(qdict) if kwargs.get('return_query_result_count', False): return self.indexer.query(_query)[0] normalize_query(_query) # normalize values recursively in-place return self._make_result(self.indexer.query(_query)) def rcount(self, *args, **kwargs): kwargs['return_query_result_count'] = True return self.query(*args, **kwargs) __call__ = query