def drop_all(): ''' Drop all docs from the solr index ''' connection = index.get_sunburnt_connection() connection.delete_all() connection.commit()
def sunburnt_query(entity_type=None, instance=None, connection=None): ''' return a pre configured sunburnt query object. If *entity_type* is given, return a query object preconfigured to only fetch documents from solr with a matching doc_type. if instance is given, only documents are returned that contain the index key. *entity_type* An indexed model class. Indexed classes are listed in :data:`adhocracy.lib.search.INDEXED_CLASSES`. *instance* A :class:`adhocracy.model.Instance` object *connection* An existing sunburnt connection. Mostly useful in tests. ''' if connection == None: connection = get_sunburnt_connection() q = connection.query() if entity_type: q = q.filter(doc_type=refs.cls_type(entity_type)) if instance and c.instance: q = q.filter(instance=instance.key) return q
def sunburnt_query(entity_type=None, excluded_entity_types=set(), instance=None, connection=None): ''' return a pre configured sunburnt query object. If *entity_type* is given, return a query object preconfigured to only fetch documents from solr with a matching doc_type. if instance is given, only documents are returned that contain the index key. *entity_type* An indexed model class. Indexed classes are listed in :data:`adhocracy.lib.search.INDEXED_CLASSES`. *instance* A :class:`adhocracy.model.Instance` object *connection* An existing sunburnt connection. Mostly useful in tests. ''' if connection is None: connection = get_sunburnt_connection() q = connection.query() if entity_type: q = q.filter(doc_type=refs.cls_type(entity_type)) for t in excluded_entity_types: q = q.filter_exclude(doc_type=refs.cls_type(t)) if instance and c.instance: q = q.filter(instance=instance.key) return q
def drop(entity_type, instance): ''' Drop all documents for *entity_type* from solr. If *instance* is given, only the documents for the given instance are removed. *entity_type* A content model class *instance* An :class:`adhocracy.model.Instance` object ''' if entity_type is None and instance is None: drop_all() return connection = index.get_sunburnt_connection() q = query.sunburnt_query(entity_type, instance, connection=connection) connection.delete(queries=q) connection.commit()
def rebuild(classes, instances=None): ''' (Re)Index all entities of the given *classes*. ''' log = logging.getLogger('index') connection = index.get_sunburnt_connection() start = time.time() batch_start = time.time() done = 0 other_classes = 0 docs = {index.ADD: [], index.SKIP: [], index.DELETE: [], index.IGNORE: []} instance_ids = [i.id for i in instances] if instances else None for cls in classes: if cls not in INDEXED_CLASSES: log.warn('Class "%s" is not an indexable class! skipping.' % cls) continue log.info("Re-indexing %ss..." % cls.__name__) q = model.meta.Session.query(cls) if instances: if isinstance(cls, model.Instance): q = q.filter(cls.id.in_(instance_ids)) elif hasattr(cls, 'instance_id'): q = q.filter(cls.instance_id.in_(instance_ids)) elif hasattr(cls, 'topic_id'): q = q.filter(cls.topic_id.in_(instance_ids)) elif cls is model.User: q = q.filter(model.User.memberships.any( model.Membership.instance_id.in_(instance_ids))) for entity in q: (action, data) = index.get_update_information(entity) docs[action].append(data) done = done + 1 if done % 1000 == 0: docs = commit_docs(docs, connection, log, start, batch_start) batch_start = time.time() log.info("...re-indexed %s %ss" % (done - other_classes, cls.__name__)) other_classes = done commit_docs(docs, connection, log, start, batch_start) now = time.time() log.info('total: %s updates, %0.1f s' % (done, now - start))
def rebuild(classes, instances=None): ''' (Re)Index all entities of the given *classes*. ''' log = logging.getLogger('index') connection = index.get_sunburnt_connection() start = time.time() batch_start = time.time() done = 0 other_classes = 0 docs = {index.ADD: [], index.SKIP: [], index.DELETE: [], index.IGNORE: []} instance_ids = [i.id for i in instances] if instances else None for cls in classes: if cls not in INDEXED_CLASSES: log.warn('Class "%s" is not an indexable class! skipping.' % cls) continue log.info("Re-indexing %ss..." % cls.__name__) q = model.meta.Session.query(cls) if instances: if isinstance(cls, model.Instance): q = q.filter(cls.id.in_(instance_ids)) elif hasattr(cls, 'instance_id'): q = q.filter(cls.instance_id.in_(instance_ids)) elif hasattr(cls, 'topic_id'): q = q.filter(cls.topic_id.in_(instance_ids)) elif cls is model.User: q = q.filter( model.User.memberships.any( model.Membership.instance_id.in_(instance_ids))) for entity in q: (action, data) = index.get_update_information(entity) docs[action].append(data) done = done + 1 if done % 1000 == 0: docs = commit_docs(docs, connection, log, start, batch_start) batch_start = time.time() log.info("...re-indexed %s %ss" % (done - other_classes, cls.__name__)) other_classes = done commit_docs(docs, connection, log, start, batch_start) now = time.time() log.info('total: %s updates, %0.1f s' % (done, now - start))