def search(self, query, conditions):
"""
Perform a raw mongodb find call.
========== ==================
Parameter Description
========== ==================
query Query hash
conditions Conditions hash
========== ==================
For more information on the query format, consult the mongodb documentation.
``Return``: List of dicts
"""
if GlobalLock.exists("scan_index"):
raise FilterException(C.make_error('INDEXING', "base"))
return self.db.index.find(query, conditions)
def sync_index(self):
# Don't index if someone else is already doing it
if GlobalLock.exists():
return
# Don't run index, if someone else already did until the last
# restart.
cr = PluginRegistry.getInstance("CommandRegistry")
nodes = cr.getNodes()
if len([n for n, v in nodes.items() if 'Indexed' in v and v['Indexed']]):
return
GlobalLock.acquire()
ObjectIndex.first_run = True
try:
self._indexed = True
t0 = time.time()
def resolve_children(dn):
self.log.debug("found object '%s'" % dn)
res = {}
children = self.factory.getObjectChildren(dn)
res = dict(res.items() + children.items())
for chld in children.keys():
res = dict(res.items() + resolve_children(chld).items())
return res
self.log.info("scanning for objects")
res = resolve_children(self.env.base)
res[self.env.base] = 'dummy'
self.log.info("generating object index")
# Find new entries
backend_objects = []
for o in sorted(res.keys(), key=len):
# Get object
try:
obj = ObjectProxy(o)
except ProxyException as e:
self.log.warning("not indexing %s: %s" % (o, str(e)))
continue
except ObjectException as e:
self.log.warning("not indexing %s: %s" % (o, str(e)))
continue
# Check for index entry
indexEntry = self.db.index.find_one({'_uuid': obj.uuid}, {'_last_changed': 1})
# Entry is not in the database
if not indexEntry:
self.insert(obj)
# Entry is in the database
else:
# OK: already there
if obj.modifyTimestamp == indexEntry['_last_changed']:
self.log.debug("found up-to-date object index for %s" % obj.uuid)
else:
self.log.debug("updating object index for %s" % obj.uuid)
self.update(obj)
backend_objects.append(obj.uuid)
del obj
# Remove entries that are in the index, but not in any other backends
for entry in self.db.index.find({'_uuid': {'$exists': True}}, {'_uuid': 1}):
if entry['_uuid'] not in backend_objects:
self.remove_by_uuid(entry['_uuid'])
t1 = time.time()
self.log.info("processed %d objects in %ds" % (len(res), t1 - t0))
except Exception as e:
self.log.critical("building the index failed: %s" % str(e))
import traceback
traceback.print_exc()
finally:
ObjectIndex.first_run = False
# Some object may have queued themselves to be re-indexed, process them now.
self.log.info("need to refresh index for %d objects" % (len(ObjectIndex.to_be_updated)))
for uuid in ObjectIndex.to_be_updated:
entry = self.db.index.find_one({'_uuid': uuid, 'dn': {'$exists': True}}, {'dn': 1})
if entry:
obj = ObjectProxy(entry['dn'])
self.update(obj)
self.log.info("index refresh finished")
zope.event.notify(IndexScanFinished())
GlobalLock.release()
