def __call__(self, container, name, args):
copts, cargs = self.parser.parse_args(args)
data = os.path.join(container.opts.data, name)
# First, pack the ZODB
storage = FileStorage.FileStorage("%s/var/Data.fs" % data)
db = DB(storage)
db.pack()
# Can't pack an Rtree's storage in-place, so we move it away and
# recreate from the contents of the ZODB
rtree = None
rtree_filename = '%s/var/vrt1' % data
try:
shutil.move(rtree_filename + ".dat", rtree_filename + ".bkup.dat")
shutil.move(rtree_filename + ".idx", rtree_filename + ".bkup.idx")
conn = db.open()
root = conn.root()
keys = root['index'].keys
bkup = Rtree('%s/var/vrt1.bkup' % data)
pagesize = bkup.properties.pagesize
if len(keys) == 0:
fwd = Rtree(
'%s/var/vrt1' % data,
# Passing in copied properties doesn't work,
# leading to errors involving page ids
# properties=new_properties,
pagesize=pagesize
)
else:
gen = ((intid, bbox, None) for intid, (uid, bbox) \
in keys.items())
fwd = Rtree(
'%s/var/vrt1' % data,
gen,
# Passing in copied properties doesn't work,
# leading to errors involving page ids
# properties=new_properties,
pagesize=pagesize
)
conn.close()
db.close()
storage.close()
except:
# Restore backups
shutil.copy(rtree_filename + ".bkup.dat", rtree_filename + ".dat")
shutil.copy(rtree_filename + ".bkup.idx", rtree_filename + ".idx")
raise
finally:
if fwd is not None:
fwd.close()
class SpatialIndex(Persistent):
def __init__(self, *args):
self.rtree_args = args
self.rtree = Rtree(*args)
self.backward = IOBTree()
def index_doc(self, docid, value):
if docid in self.backward:
self.unindex_doc(docid)
self.backward[docid] = value
self.rtree.add(docid, value, obj=docid)
def unindex_doc(self, docid):
value = self.backward.get(docid)
if value is None:
return
self.rtree.delete(docid, value)
del self.backward[docid]
def apply(self, value):
return [x.object for x in self.rtree.intersection(value, objects=True)]
def clear(self):
self.backward.clear()
props = self.rtree.properties
if props.storage == RT_Disk:
self.rtree.close()
fname = props.filename
try:
os.unlink('%s.%s' % (fname, props.dat_extension))
except OSError:
pass
try:
os.unlink('%s.%s' % (fname, props.idx_extension))
except OSError:
pass
self.rtree = Rtree(*self.rtree_args)
def count(self, value):
return self.rtree.count(value)
class IntRtreeIndex(BaseIndex):
"""Avoids the slower Rtree query object=True interface
"""
_v_nextuid = None
family = BTrees.family32
def clear(self):
self.fwd = Rtree()
self.bwd = self.family.OO.BTree()
self.keys = self.family.IO.BTree()
self.intids = self.family.OI.BTree()
self.ids = self.family.OO.BTree()
def __init__(self):
self.clear()
def key(self, item):
try:
return item['id'], tuple(self.bbox(item))
except:
return tuple(item.items())
def fid(self, item):
return item['id']
def intid(self, item):
# Get and track next available key using zope.intid algorithm
# Item might be already registered
uid = self.intids.get(self.key(item))
if uid is not None:
return uid
# But if not registered
nextuid = getattr(self, '_v_nextuid', None)
while True:
if nextuid is None:
nextuid = random.randrange(0, self.family.maxint)
uid = nextuid
if uid not in self.keys:
nextuid += 1
if nextuid > self.family.maxint:
nextuid = None
self._v_nextuid = nextuid
return uid
nextuid = None
def intersection(self, bbox):
"""Return an iterator over Items that intersect with the bbox"""
for hit in self.fwd.intersection(bbox, objects=False):
yield self.bwd[int(hit)]
def nearest(self, bbox, limit=1):
"""Return an iterator over the nearest N=limit Items to the bbox"""
for hit in self.fwd.nearest(bbox, num_results=limit, objects=False):
yield self.bwd[int(hit)]
def item(self, fid, bbox):
return self.bwd[self.intids[(fid, bbox)]]
def items(self, fid):
return [self.bwd[intid] for intid in self.ids[fid]]
def index_item(self, itemid, bbox, item):
"""Add an Item to the index"""
if itemid in self.bwd:
self.unindex_item(itemid, bbox)
# Store an id for the item if it has None
try:
item.update(id=item.get('id') or str(uuid.uuid4()))
key = self.key(item)
sid = self.fid(item)
# Map keys <-> intids
intid = self.intid(item)
self.keys[intid] = key
self.intids[key] = intid
if sid not in self.ids:
self.ids[sid] = IISet([])
self.ids[sid].add(intid)
self.bwd[intid] = item
self.fwd.add(intid, bbox)
except:
import pdb; pdb.set_trace()
raise
def unindex_item(self, itemid, bbox):
"""Remove an Item from the index"""
intid = int(itemid)
key = self.keys.get(intid)
if key is None:
return
self.ids[key[0]].remove(intid)
del self.keys[intid]
del self.intids[key]
del self.bwd[intid]
self.fwd.delete(intid, bbox)
def batch(self, changeset):
BaseIndex.batch(self, changeset)
def commit(self):
transaction.commit()
rtree_storage = self.fwd.properties.filename
self.fwd.close()
self.fwd = Rtree(rtree_storage)
def close(self):
self.fwd.close()
