Esempio n. 1
0
 def test_contains(self):
     mapper = UUIDMapper()
     uid, docid = uuid.uuid4(), 12345
     mapper.add(uid, docid)
     assert uid in mapper        # normalized UUID->str
     assert str(uid) in mapper
     assert docid in mapper
Esempio n. 2
0
 def test_contains(self):
     mapper = UUIDMapper()
     uid, docid = uuid.uuid4(), 12345
     mapper.add(uid, docid)
     assert uid in mapper  # normalized UUID->str
     assert str(uid) in mapper
     assert docid in mapper
Esempio n. 3
0
 def __init__(self, context, schema=None):
     self._context_uid = IUUID(context)
     if schema is None:
         schema = getattr(context, 'schema', None)
         if schema is None:
             raise ValueError('Context does not provide schema')
     self.indexer = Indexer()
     self.uidmap = UUIDMapper()
     self.bind(schema)
Esempio n. 4
0
 def __init__(self, context, schema=None):
     self._context_uid = IUUID(context)
     if schema is None:
         schema = getattr(context, 'schema', None)
         if schema is None:
             raise ValueError('Context does not provide schema')
     self.indexer = Indexer()
     self.uidmap = UUIDMapper()
     self.bind(schema)
Esempio n. 5
0
 def test_add_remove(self):
     """Test add/remove and containment/get/length"""
     mapper = UUIDMapper()
     # add with pre-calculated docid
     uid, docid = uuid.uuid4(), 12345
     rv = mapper.add(uid, docid)
     assert uid in mapper
     assert rv == (str(uid), docid)
     assert mapper.get(uid) == docid
     assert len(mapper) == 1
     # Cannot add duplicate:
     self.assertRaises(KeyError, mapper.add, uid)
     self.assertRaises(KeyError, mapper.add, str(uid))
     # add with generation of docid
     uid2 = uuid.uuid4()
     rv = mapper.add(uid2)
     assert len(mapper) == 2
     docid2 = rv[1]
     assert rv == (str(uid2), docid2)
     assert uid2 in mapper
     assert docid2 in mapper
     assert mapper.get(uid2) == docid2
     assert mapper.get(docid2) == str(uid2)
     # remove one by UUID
     mapper.remove(uid2)
     assert uid2 not in mapper
     assert docid2 not in mapper
     assert len(mapper) == 1
     mapper.remove(docid)
     assert docid not in mapper
     assert uid not in mapper
     assert str(uid) not in mapper
     assert mapper.get(uid, None) is None
     assert len(mapper) == 0
     # re-add okay:
     mapper.add(uid, docid)
     assert uid in mapper and docid in mapper
     assert len(mapper) == 1
Esempio n. 6
0
 def test_enumeration(self):
     """test enumeration and iteration"""
     _uids = []
     mapper = UUIDMapper()
     # add ten random pairs:
     for i in range(10):
         uid = uuid.uuid4()
         _uids.append(str(uid))
         mapper.add(uid)
     assert len(mapper) == 10 == len(mapper.keys())
     assert len(mapper.items()) == len(mapper.values()) == 10
     assert len(list(mapper.iteritems())) == 10
     for uid in _uids:
         assert uid in mapper
         assert uid in mapper.keys()
         assert uid in mapper.iterkeys()
         assert mapper.get(uid) in mapper.values()
         assert mapper.get(uid) in mapper.itervalues()
         assert (uid, mapper.get(uid)) in mapper.items()
         assert (uid, mapper.get(uid)) in mapper.iteritems()
Esempio n. 7
0
class SimpleCatalog(Persistent):
    """
    Simple catalog for items sharing a common single search schema,
    and for which items are resolved from a single container which
    is a content item.
    
    Items are externally referenced and results are keyed by UUID.
    """

    implements(ISimpleCatalog)

    def __init__(self, context, schema=None):
        self._context_uid = IUUID(context)
        if schema is None:
            schema = getattr(context, 'schema', None)
            if schema is None:
                raise ValueError('Context does not provide schema')
        self.indexer = Indexer()
        self.uidmap = UUIDMapper()
        self.bind(schema)

    ## ILocation implementation:

    __name__ = 'simple_catalog'

    @property
    def __parent__(self):
        return self.resolver.context  # based on UID of container content

    ## ISearchContext properties:

    @property
    def resolver(self):
        if not getattr(self, '_v_resolver', None):
            self._v_resolver = ContentContainerUIDResolver(self._context_uid)
        return self._v_resolver

    ## ISimpleCatalog indexing methods:

    def bind(self, schema):
        if hasattr(self, '_v_schema'):
            delattr(self, '_v_schema')
        self._schema = identify_interface(schema)
        self.make_indexes()

    def _search_schema(self):
        if not getattr(self, '_v_schema', None):
            self._v_schema = resolve(self._schema)
        return self._v_schema

    search_schema = property(_search_schema, bind)

    def indexes(self):
        """index names per schema"""
        return ISchemaIndexes(self.search_schema, ())

    def make_indexes(self):
        names = self.indexes()
        for name in names:
            idx_type = name.split('_')[0]
            fieldname = name[(len(idx_type) + 1):]
            field = self.search_schema[fieldname]
            ## need a persistent callable discriminator to support value
            ## normalization, it is the only way to have a callable
            ## discriminator that is anonymous (not importable) that
            ## works around limitations in ZODB/pickle.
            discriminator = fieldname
            if idx_type != 'text':
                discriminator = ValueDiscriminator(field)
            self.indexer[name] = IDXCLS.get(idx_type)(discriminator)

    def index(self, obj):
        uid = IUUID(obj)
        uid, docid = self.uidmap.add(uid)
        self.indexer.index_doc(docid, obj)

    def unindex(self, obj):
        if isinstance(obj, str):
            uid = obj
        else:
            uid = IUUID(obj)
        if uid not in self.uidmap:
            raise KeyError(uid)
        docid = self.uidmap.docid_for(uid)
        self.indexer.unindex_doc(docid)
        self.uidmap.remove(uid)

    def reindex(self, obj=None):
        if obj is None:
            for uid, docid in self.uidmap.iteritems():
                self.reindex(obj=uid)
        else:
            if isinstance(obj, str):
                uid = obj
                obj = self.get(uid)
                if obj is None:
                    self.unindex(uid)  # stale entry, now gone
                    return
            else:
                uid = IUUID(obj)
            docid = self.uidmap.docid_for(uid)
            self.indexer.redindex_doc(docid, obj)

    ## ISearchContext base mapping methods:

    def __len__(self):
        return len(self.uidmap)

    def get(self, key, default=None):
        uid = key
        if isinstance(key, int) or isinstance(key, long):
            uid = self.uidmap.equivalent(key)
        v = self.resolver(uid)
        if v is None:
            return default
        return v

    def __getitem__(self, key):
        v = self.get(key, None)
        if v is None:
            raise KeyError(key)
        return v

    def __contains__(self, spec):
        uid = spec
        if not isinstance(spec, str):
            uid = IUUID(spec, None)
            if uid is None:
                uid = normalize_uuid(spec)
                if uid is None:
                    return False
        return uid in self.uidmap

    def iterkeys(self):
        return self.uidmap.iterkeys()  # UIDs, not docids

    __iter__ = iterkeys

    def itervalues(self):
        return (self.get(uid) for uid in self.iterkeys())

    def iteritems(self):
        return ((uid, self.get(uid)) for uid in self.iterkeys())

    def keys(self):
        return list(self.iterkeys())

    def values(self):
        return list(self.itervalues())

    def items(self):
        return list(self.iteritems())

    ## IItemCollection
    def byuid(self):
        return self

    def byname(self):
        return self  # technically, we don't map local ids

    ## ISimpleCatalog query methods:

    def _query_from_mapping(self, qdict):
        """
        return a query.Query object given mapping of keys/values.
        Value normalization is not in scope (should happen to
        resulting query).
        """
        r = []
        for idxname, value in qdict.items():
            if isinstance(value, tuple) and len(value) > 1:
                if issubclass(value[0], query.Query):
                    comparator = value[0]
                    r.append(comparator(idxname, value[1]))
                    continue
            if idxname.startswith('text'):
                r.append(query.Contains(idxname, value))
            elif idxname.startswith('keyword'):
                r.append(query.Any(idxname, value))
            else:
                r.append(query.Eq(idxname, value))
        if len(r) == 1:
            return r[0]
        return query.And(*r)

    def _make_result(self, result):
        """
        Given a result as tuple of length, integer docids,
        construct a search result keyed by UUID.
        """
        size, docids = result  # unpack, but we do not care about size
        t = tuple((docid, self.uidmap.uuid_for(docid))
                  for docid in docids)  # iterate into pairs of docid, uid
        result = SearchResult.fromtuples(t, resolver=self.resolver)
        result.__parent__ = self
        result.__name__ = 'result'
        return result

    def query(self, *args, **kwargs):
        qdict = None
        if not args and kwargs:
            qdict = kwargs
        elif args and hasattr(args[0], 'iteritems'):
            ## looks like mapping/dict
            qdict = dict(args[0].items())
        elif not args:
            raise ValueError('Empty query')
        else:
            _query = args[0]
            if not isinstance(_query, query.Query):
                raise ValueError('Invalid query')
        if qdict:
            _query = self._query_from_mapping(qdict)
        if kwargs.get('return_query_result_count', False):
            return self.indexer.query(_query)[0]
        normalize_query(_query)  # normalize values recursively in-place
        return self._make_result(self.indexer.query(_query))

    def rcount(self, *args, **kwargs):
        kwargs['return_query_result_count'] = True
        return self.query(*args, **kwargs)

    __call__ = query
Esempio n. 8
0
 def test_add_remove(self):
     """Test add/remove and containment/get/length"""
     mapper = UUIDMapper()
     # add with pre-calculated docid
     uid, docid = uuid.uuid4(), 12345
     rv = mapper.add(uid, docid)
     assert uid in mapper
     assert rv == (str(uid), docid)
     assert mapper.get(uid) == docid
     assert len(mapper) == 1
     # Cannot add duplicate:
     self.assertRaises(KeyError, mapper.add, uid)
     self.assertRaises(KeyError, mapper.add, str(uid))
     # add with generation of docid
     uid2 = uuid.uuid4()
     rv = mapper.add(uid2)
     assert len(mapper) == 2
     docid2 = rv[1]
     assert rv == (str(uid2), docid2)
     assert uid2 in mapper
     assert docid2 in mapper
     assert mapper.get(uid2) == docid2
     assert mapper.get(docid2) == str(uid2)
     # remove one by UUID
     mapper.remove(uid2)
     assert uid2 not in mapper
     assert docid2 not in mapper
     assert len(mapper) == 1
     mapper.remove(docid)
     assert docid not in mapper
     assert uid not in mapper
     assert str(uid) not in mapper
     assert mapper.get(uid, None) is None
     assert len(mapper) == 0
     # re-add okay:
     mapper.add(uid, docid)
     assert uid in mapper and docid in mapper
     assert len(mapper) == 1
Esempio n. 9
0
 def test_btrees(self):
     mapper = UUIDMapper()
     from BTrees.LOBTree import LOBTree
     from BTrees.OLBTree import OLBTree
     self.assertIsInstance(mapper.uuid_to_docid, OLBTree)
     self.assertIsInstance(mapper.docid_to_uuid, LOBTree)
Esempio n. 10
0
 def test_enumeration(self):
     """test enumeration and iteration"""
     _uids = []
     mapper = UUIDMapper()
     # add ten random pairs:
     for i in range(10):
         uid = uuid.uuid4()
         _uids.append(str(uid))
         mapper.add(uid)
     assert len(mapper) == 10 == len(mapper.keys())
     assert len(mapper.items()) == len(mapper.values()) == 10
     assert len(list(mapper.iteritems())) == 10
     for uid in _uids:
         assert uid in mapper
         assert uid in mapper.keys()
         assert uid in mapper.iterkeys()
         assert mapper.get(uid) in mapper.values()
         assert mapper.get(uid) in mapper.itervalues()
         assert (uid, mapper.get(uid)) in mapper.items()
         assert (uid, mapper.get(uid)) in mapper.iteritems()
Esempio n. 11
0
class SimpleCatalog(Persistent):
    """
    Simple catalog for items sharing a common single search schema,
    and for which items are resolved from a single container which
    is a content item.
    
    Items are externally referenced and results are keyed by UUID.
    """
    
    implements(ISimpleCatalog)
    
    def __init__(self, context, schema=None):
        self._context_uid = IUUID(context)
        if schema is None:
            schema = getattr(context, 'schema', None)
            if schema is None:
                raise ValueError('Context does not provide schema')
        self.indexer = Indexer()
        self.uidmap = UUIDMapper()
        self.bind(schema)
    
    ## ILocation implementation:
    
    __name__ = 'simple_catalog'

    @property
    def __parent__(self):
        return self.resolver.context  # based on UID of container content
    
    ## ISearchContext properties:

    @property
    def resolver(self):
        if not getattr(self, '_v_resolver', None):
            self._v_resolver = ContentContainerUIDResolver(self._context_uid)
        return self._v_resolver
    
    ## ISimpleCatalog indexing methods:

    def bind(self, schema):
        if hasattr(self, '_v_schema'):
            delattr(self, '_v_schema')
        self._schema = identify_interface(schema)
        self.make_indexes()
    
    def _search_schema(self):
        if not getattr(self, '_v_schema', None):
            self._v_schema = resolve(self._schema)
        return self._v_schema
    
    search_schema = property(_search_schema, bind)
    
    def indexes(self):
        """index names per schema"""
        return ISchemaIndexes(self.search_schema, ())
    
    def make_indexes(self):
        names = self.indexes()
        for name in names:
            idx_type = name.split('_')[0]
            fieldname = name[(len(idx_type) + 1):]
            field = self.search_schema[fieldname]
            ## need a persistent callable discriminator to support value
            ## normalization, it is the only way to have a callable
            ## discriminator that is anonymous (not importable) that
            ## works around limitations in ZODB/pickle.
            discriminator = fieldname
            if idx_type != 'text':
                discriminator = ValueDiscriminator(field)
            self.indexer[name] = IDXCLS.get(idx_type)(discriminator)
    
    def index(self, obj):
        uid = IUUID(obj)
        uid, docid = self.uidmap.add(uid)
        self.indexer.index_doc(docid, obj)
    
    def unindex(self, obj):
        if isinstance(obj, str):
            uid = obj
        else:
            uid = IUUID(obj)
        if uid not in self.uidmap:
            raise KeyError(uid)
        docid = self.uidmap.docid_for(uid)
        self.indexer.unindex_doc(docid)
        self.uidmap.remove(uid)
    
    def reindex(self, obj=None):
        if obj is None:
            for uid, docid in self.uidmap.iteritems():
                self.reindex(obj=uid)
        else:
            if isinstance(obj, str):
                uid = obj
                obj = self.get(uid)
                if obj is None:
                    self.unindex(uid)  # stale entry, now gone
                    return
            else:
                uid = IUUID(obj)
            docid = self.uidmap.docid_for(uid)
            self.indexer.redindex_doc(docid, obj)
   
    ## ISearchContext base mapping methods:
    
    def __len__(self):
        return len(self.uidmap)
    
    def get(self, key, default=None):
        uid = key
        if isinstance(key, int) or isinstance(key, long):
            uid = self.uidmap.equivalent(key)
        v = self.resolver(uid)
        if v is None:
            return default
        return v
    
    def __getitem__(self, key):
        v = self.get(key, None)
        if v is None:
            raise KeyError(key)
        return v
    
    def __contains__(self, spec):
        uid = spec
        if not isinstance(spec, str):
            uid = IUUID(spec, None)
            if uid is None:
                uid = normalize_uuid(spec)
                if uid is None:
                    return False
        return uid in self.uidmap
    
    def iterkeys(self):
        return self.uidmap.iterkeys()  # UIDs, not docids
   
    __iter__ = iterkeys

    def itervalues(self):
        return (self.get(uid) for uid in self.iterkeys())
    
    def iteritems(self):
        return ((uid, self.get(uid)) for uid in self.iterkeys())
    
    def keys(self):
        return list(self.iterkeys())
    
    def values(self):
        return list(self.itervalues())

    def items(self):
        return list(self.iteritems())
    
    ## IItemCollection
    def byuid(self):
        return self
    
    def byname(self):
        return self  # technically, we don't map local ids
    
    ## ISimpleCatalog query methods:
    
    def _query_from_mapping(self, qdict):
        """
        return a query.Query object given mapping of keys/values.
        Value normalization is not in scope (should happen to
        resulting query).
        """
        r = []
        for idxname, value in qdict.items():
            if isinstance(value, tuple) and len(value) > 1:
                if issubclass(value[0], query.Query):
                    comparator = value[0]
                    r.append(comparator(idxname, value[1]))
                    continue
            if idxname.startswith('text'):
                r.append(query.Contains(idxname, value))
            elif idxname.startswith('keyword'):
                r.append(query.Any(idxname, value))
            else:
                r.append(query.Eq(idxname, value))
        if len(r) == 1:
            return r[0]
        return query.And(*r)
    
    def _make_result(self, result):
        """
        Given a result as tuple of length, integer docids,
        construct a search result keyed by UUID.
        """
        size, docids = result  # unpack, but we do not care about size
        t = tuple(
            (docid, self.uidmap.uuid_for(docid)) for docid in docids
            )  # iterate into pairs of docid, uid
        result = SearchResult.fromtuples(t, resolver=self.resolver)
        result.__parent__ = self
        result.__name__ = 'result'
        return result
    
    def query(self, *args, **kwargs):
        qdict = None
        if not args and kwargs:
            qdict = kwargs
        elif args and hasattr(args[0], 'iteritems'):
            ## looks like mapping/dict
            qdict = dict(args[0].items())
        elif not args:
            raise ValueError('Empty query')
        else:
            _query = args[0]
            if not isinstance(_query, query.Query):
                raise ValueError('Invalid query')
        if qdict:
            _query = self._query_from_mapping(qdict)
        if kwargs.get('return_query_result_count', False):
            return self.indexer.query(_query)[0]
        normalize_query(_query)  # normalize values recursively in-place
        return self._make_result(self.indexer.query(_query))
    
    def rcount(self, *args, **kwargs):
        kwargs['return_query_result_count'] = True
        return self.query(*args, **kwargs)
    
    __call__ = query