Example #1
0
def drop_all():
    '''
    Drop all docs from the solr index
    '''
    connection = index.get_sunburnt_connection()
    connection.delete_all()
    connection.commit()
Example #2
0
def drop_all():
    '''
    Drop all docs from the solr index
    '''
    connection = index.get_sunburnt_connection()
    connection.delete_all()
    connection.commit()
Example #3
0
def sunburnt_query(entity_type=None, instance=None, connection=None):
    '''
    return a pre configured sunburnt query object. If *entity_type*
    is given, return a query object preconfigured to only fetch
    documents from solr with a matching doc_type. if instance is
    given, only documents are returned that contain the index
    key.

    *entity_type*
        An indexed model class. Indexed classes are listed
        in :data:`adhocracy.lib.search.INDEXED_CLASSES`.
    *instance*
        A :class:`adhocracy.model.Instance` object
    *connection*
        An existing sunburnt connection. Mostly useful
        in tests.
    '''
    if connection == None:
        connection = get_sunburnt_connection()
    q = connection.query()
    if entity_type:
        q = q.filter(doc_type=refs.cls_type(entity_type))
    if instance and c.instance:
        q = q.filter(instance=instance.key)
    return q
Example #4
0
def sunburnt_query(entity_type=None,
                   excluded_entity_types=set(),
                   instance=None,
                   connection=None):
    '''
    return a pre configured sunburnt query object. If *entity_type*
    is given, return a query object preconfigured to only fetch
    documents from solr with a matching doc_type. if instance is
    given, only documents are returned that contain the index
    key.

    *entity_type*
        An indexed model class. Indexed classes are listed
        in :data:`adhocracy.lib.search.INDEXED_CLASSES`.
    *instance*
        A :class:`adhocracy.model.Instance` object
    *connection*
        An existing sunburnt connection. Mostly useful
        in tests.
    '''
    if connection is None:
        connection = get_sunburnt_connection()
    q = connection.query()
    if entity_type:
        q = q.filter(doc_type=refs.cls_type(entity_type))
    for t in excluded_entity_types:
        q = q.filter_exclude(doc_type=refs.cls_type(t))
    if instance and c.instance:
        q = q.filter(instance=instance.key)
    return q
Example #5
0
def drop(entity_type, instance):
    '''
    Drop all documents for *entity_type* from solr. If *instance* is
    given, only the documents for the given instance are removed.

    *entity_type*
       A content model class
    *instance*
       An :class:`adhocracy.model.Instance` object
    '''
    if entity_type is None and instance is None:
        drop_all()
        return
    connection = index.get_sunburnt_connection()
    q = query.sunburnt_query(entity_type, instance, connection=connection)
    connection.delete(queries=q)
    connection.commit()
Example #6
0
def drop(entity_type, instance):
    '''
    Drop all documents for *entity_type* from solr. If *instance* is
    given, only the documents for the given instance are removed.

    *entity_type*
       A content model class
    *instance*
       An :class:`adhocracy.model.Instance` object
    '''
    if entity_type is None and instance is None:
        drop_all()
        return
    connection = index.get_sunburnt_connection()
    q = query.sunburnt_query(entity_type, instance, connection=connection)
    connection.delete(queries=q)
    connection.commit()
Example #7
0
def rebuild(classes, instances=None):
    '''
    (Re)Index all entities of the given *classes*.
    '''
    log = logging.getLogger('index')
    connection = index.get_sunburnt_connection()
    start = time.time()
    batch_start = time.time()
    done = 0
    other_classes = 0
    docs = {index.ADD: [],
            index.SKIP: [],
            index.DELETE: [],
            index.IGNORE: []}
    instance_ids = [i.id for i in instances] if instances else None

    for cls in classes:
        if cls not in INDEXED_CLASSES:
            log.warn('Class "%s" is not an indexable class! skipping.' %
                     cls)
            continue
        log.info("Re-indexing %ss..." % cls.__name__)
        q = model.meta.Session.query(cls)
        if instances:
            if isinstance(cls, model.Instance):
                q = q.filter(cls.id.in_(instance_ids))
            elif hasattr(cls, 'instance_id'):
                q = q.filter(cls.instance_id.in_(instance_ids))
            elif hasattr(cls, 'topic_id'):
                q = q.filter(cls.topic_id.in_(instance_ids))
            elif cls is model.User:
                q = q.filter(model.User.memberships.any(
                    model.Membership.instance_id.in_(instance_ids)))
        for entity in q:
            (action, data) = index.get_update_information(entity)
            docs[action].append(data)
            done = done + 1
            if done % 1000 == 0:
                docs = commit_docs(docs, connection, log, start, batch_start)
                batch_start = time.time()
        log.info("...re-indexed %s %ss" % (done - other_classes, cls.__name__))
        other_classes = done
    commit_docs(docs, connection, log, start, batch_start)
    now = time.time()
    log.info('total: %s updates, %0.1f s' % (done, now - start))
Example #8
0
def rebuild(classes, instances=None):
    '''
    (Re)Index all entities of the given *classes*.
    '''
    log = logging.getLogger('index')
    connection = index.get_sunburnt_connection()
    start = time.time()
    batch_start = time.time()
    done = 0
    other_classes = 0
    docs = {index.ADD: [], index.SKIP: [], index.DELETE: [], index.IGNORE: []}
    instance_ids = [i.id for i in instances] if instances else None

    for cls in classes:
        if cls not in INDEXED_CLASSES:
            log.warn('Class "%s" is not an indexable class! skipping.' % cls)
            continue
        log.info("Re-indexing %ss..." % cls.__name__)
        q = model.meta.Session.query(cls)
        if instances:
            if isinstance(cls, model.Instance):
                q = q.filter(cls.id.in_(instance_ids))
            elif hasattr(cls, 'instance_id'):
                q = q.filter(cls.instance_id.in_(instance_ids))
            elif hasattr(cls, 'topic_id'):
                q = q.filter(cls.topic_id.in_(instance_ids))
            elif cls is model.User:
                q = q.filter(
                    model.User.memberships.any(
                        model.Membership.instance_id.in_(instance_ids)))
        for entity in q:
            (action, data) = index.get_update_information(entity)
            docs[action].append(data)
            done = done + 1
            if done % 1000 == 0:
                docs = commit_docs(docs, connection, log, start, batch_start)
                batch_start = time.time()
        log.info("...re-indexed %s %ss" % (done - other_classes, cls.__name__))
        other_classes = done
    commit_docs(docs, connection, log, start, batch_start)
    now = time.time()
    log.info('total: %s updates, %0.1f s' % (done, now - start))