def search(self, query, conditions): """ Perform a raw mongodb find call. ========== ================== Parameter Description ========== ================== query Query hash conditions Conditions hash ========== ================== For more information on the query format, consult the mongodb documentation. ``Return``: List of dicts """ if GlobalLock.exists("scan_index"): raise FilterException(C.make_error('INDEXING', "base")) return self.db.index.find(query, conditions)
def sync_index(self): # Don't index if someone else is already doing it if GlobalLock.exists(): return # Don't run index, if someone else already did until the last # restart. cr = PluginRegistry.getInstance("CommandRegistry") nodes = cr.getNodes() if len([n for n, v in nodes.items() if 'Indexed' in v and v['Indexed']]): return GlobalLock.acquire() ObjectIndex.first_run = True try: self._indexed = True t0 = time.time() def resolve_children(dn): self.log.debug("found object '%s'" % dn) res = {} children = self.factory.getObjectChildren(dn) res = dict(res.items() + children.items()) for chld in children.keys(): res = dict(res.items() + resolve_children(chld).items()) return res self.log.info("scanning for objects") res = resolve_children(self.env.base) res[self.env.base] = 'dummy' self.log.info("generating object index") # Find new entries backend_objects = [] for o in sorted(res.keys(), key=len): # Get object try: obj = ObjectProxy(o) except ProxyException as e: self.log.warning("not indexing %s: %s" % (o, str(e))) continue except ObjectException as e: self.log.warning("not indexing %s: %s" % (o, str(e))) continue # Check for index entry indexEntry = self.db.index.find_one({'_uuid': obj.uuid}, {'_last_changed': 1}) # Entry is not in the database if not indexEntry: self.insert(obj) # Entry is in the database else: # OK: already there if obj.modifyTimestamp == indexEntry['_last_changed']: self.log.debug("found up-to-date object index for %s" % obj.uuid) else: self.log.debug("updating object index for %s" % obj.uuid) self.update(obj) backend_objects.append(obj.uuid) del obj # Remove entries that are in the index, but not in any other backends for entry in self.db.index.find({'_uuid': {'$exists': True}}, {'_uuid': 1}): if entry['_uuid'] not in backend_objects: self.remove_by_uuid(entry['_uuid']) t1 = time.time() self.log.info("processed %d objects in %ds" % (len(res), t1 - t0)) except Exception as e: self.log.critical("building the index failed: %s" % str(e)) import traceback traceback.print_exc() finally: ObjectIndex.first_run = False # Some object may have queued themselves to be re-indexed, process them now. self.log.info("need to refresh index for %d objects" % (len(ObjectIndex.to_be_updated))) for uuid in ObjectIndex.to_be_updated: entry = self.db.index.find_one({'_uuid': uuid, 'dn': {'$exists': True}}, {'dn': 1}) if entry: obj = ObjectProxy(entry['dn']) self.update(obj) self.log.info("index refresh finished") zope.event.notify(IndexScanFinished()) GlobalLock.release()