Exemplo n.º 1
0
    def __call__(self, container, name, args):
        copts, cargs = self.parser.parse_args(args)
        data = os.path.join(container.opts.data, name)

        # First, pack the ZODB
        storage = FileStorage.FileStorage("%s/var/Data.fs" % data)
        db = DB(storage)
        db.pack()

        # Can't pack an Rtree's storage in-place, so we move it away and 
        # recreate from the contents of the ZODB
        rtree = None
        rtree_filename = '%s/var/vrt1' % data
 
        try:
            shutil.move(rtree_filename + ".dat", rtree_filename + ".bkup.dat")
            shutil.move(rtree_filename + ".idx", rtree_filename + ".bkup.idx")
        
            conn = db.open()
            root = conn.root()
            keys = root['index'].keys

            bkup = Rtree('%s/var/vrt1.bkup' % data)
            pagesize = bkup.properties.pagesize

            if len(keys) == 0:
                fwd = Rtree(
                        '%s/var/vrt1' % data,
                        # Passing in copied properties doesn't work,
                        # leading to errors involving page ids
                        # properties=new_properties, 
                        pagesize=pagesize
                        )
            else:
                gen = ((intid, bbox, None) for intid, (uid, bbox) \
                      in keys.items())
                fwd = Rtree(
                        '%s/var/vrt1' % data,
                        gen, 
                        # Passing in copied properties doesn't work,
                        # leading to errors involving page ids
                        # properties=new_properties,
                        pagesize=pagesize
                        )
            
            conn.close()
            db.close()
            storage.close()
        except:
            # Restore backups
            shutil.copy(rtree_filename + ".bkup.dat", rtree_filename + ".dat")
            shutil.copy(rtree_filename + ".bkup.idx", rtree_filename + ".idx")
            raise
        finally:
            if fwd is not None:
                fwd.close()
Exemplo n.º 2
0
class SpatialIndex(Persistent):

    def __init__(self, *args):
        self.rtree_args = args
        self.rtree = Rtree(*args)
        self.backward = IOBTree()

    def index_doc(self, docid, value):
        if docid in self.backward:
            self.unindex_doc(docid)
        self.backward[docid] = value
        self.rtree.add(docid, value, obj=docid)

    def unindex_doc(self, docid):
        value = self.backward.get(docid)
        if value is None:
            return
        self.rtree.delete(docid, value)
        del self.backward[docid]

    def apply(self, value):
        return [x.object for x in self.rtree.intersection(value, objects=True)]

    def clear(self):
        self.backward.clear()
        props = self.rtree.properties
        if props.storage == RT_Disk:
            self.rtree.close()
            fname = props.filename
            try:
                os.unlink('%s.%s' % (fname, props.dat_extension))
            except OSError:
                pass
            try:
                os.unlink('%s.%s' % (fname, props.idx_extension))
            except OSError:
                pass
        self.rtree = Rtree(*self.rtree_args)

    def count(self, value):
        return self.rtree.count(value)
Exemplo n.º 3
0
class IntRtreeIndex(BaseIndex):
    """Avoids the slower Rtree query object=True interface
    """
    _v_nextuid = None
    family = BTrees.family32

    def clear(self):
        self.fwd = Rtree()
        self.bwd = self.family.OO.BTree()
        self.keys = self.family.IO.BTree()
        self.intids = self.family.OI.BTree()
        self.ids = self.family.OO.BTree()
    def __init__(self):
        self.clear()
    def key(self, item):
        try:
            return item['id'], tuple(self.bbox(item))
        except:
            return tuple(item.items())
    def fid(self, item):
        return item['id']
    def intid(self, item):
        # Get and track next available key using zope.intid algorithm
        # Item might be already registered
        uid = self.intids.get(self.key(item))
        if uid is not None:
            return uid
        # But if not registered
        nextuid = getattr(self, '_v_nextuid', None)
        while True:
            if nextuid is None:
                nextuid = random.randrange(0, self.family.maxint)
            uid = nextuid
            if uid not in self.keys:
                nextuid += 1
                if nextuid > self.family.maxint:
                    nextuid = None
                self._v_nextuid = nextuid
                return uid
            nextuid = None
    def intersection(self, bbox):
        """Return an iterator over Items that intersect with the bbox"""
        for hit in self.fwd.intersection(bbox, objects=False):
            yield self.bwd[int(hit)]
    def nearest(self, bbox, limit=1):
        """Return an iterator over the nearest N=limit Items to the bbox"""
        for hit in self.fwd.nearest(bbox, num_results=limit, objects=False):
            yield self.bwd[int(hit)]
    def item(self, fid, bbox):
        return self.bwd[self.intids[(fid, bbox)]]
    def items(self, fid):
        return [self.bwd[intid] for intid in self.ids[fid]]
    def index_item(self, itemid, bbox, item):
        """Add an Item to the index"""
        if itemid in self.bwd:
            self.unindex_item(itemid, bbox)
        # Store an id for the item if it has None
        try:
            item.update(id=item.get('id') or str(uuid.uuid4()))
            key = self.key(item)
            sid = self.fid(item)
            
            # Map keys <-> intids
            intid = self.intid(item)
            self.keys[intid] = key
            self.intids[key] = intid
            
            if sid not in self.ids:
                self.ids[sid] = IISet([])
            self.ids[sid].add(intid)

            self.bwd[intid] = item
            self.fwd.add(intid, bbox)
        except:
            import pdb; pdb.set_trace()
            raise
    def unindex_item(self, itemid, bbox):
        """Remove an Item from the index"""
        intid = int(itemid)
        key = self.keys.get(intid)
        if key is None:
            return
        self.ids[key[0]].remove(intid)
        del self.keys[intid]
        del self.intids[key]
        del self.bwd[intid]
        self.fwd.delete(intid, bbox)
    def batch(self, changeset):
        BaseIndex.batch(self, changeset)
    def commit(self):
        transaction.commit()
        rtree_storage = self.fwd.properties.filename
        self.fwd.close()
        self.fwd = Rtree(rtree_storage)
    def close(self):
        self.fwd.close()