Esempio n. 1
0
def appendtextindex(table,
                    index_or_dirname,
                    indexname=None,
                    merge=True,
                    optimize=False):
    """
    Load all rows from `table` into a Whoosh index, adding them to any existing
    data in the index.

    Keyword arguments:

    table
        A table container with the data to be loaded.
    index_or_dirname
        Either an instance of `whoosh.index.Index` or a string containing the
        directory path where the index is to be stored.
    indexname
        String containing the name of the index, if multiple indexes are stored
        in the same directory.
    merge
        Merge small segments during commit?
    optimize
        Merge all segments together?

    """
    import whoosh.index

    # deal with polymorphic argument
    if isinstance(index_or_dirname, string_types):
        dirname = index_or_dirname
        index = whoosh.index.open_dir(dirname,
                                      indexname=indexname,
                                      readonly=False)
        needs_closing = True
    elif isinstance(index_or_dirname, whoosh.index.Index):
        index = index_or_dirname
        needs_closing = False
    else:
        raise ArgumentError('expected string or index, found %r' %
                            index_or_dirname)

    writer = index.writer()
    try:

        for d in dicts(table):
            writer.add_document(**d)
        writer.commit(merge=merge, optimize=optimize)

    except Exception:
        writer.cancel()
        raise

    finally:
        if needs_closing:
            index.close()
Esempio n. 2
0
def appendtextindex(table, index_or_dirname, indexname=None, merge=True,
                    optimize=False):
    """
    Load all rows from `table` into a Whoosh index, adding them to any existing
    data in the index.

    Keyword arguments:

    table
        A table container with the data to be loaded.
    index_or_dirname
        Either an instance of `whoosh.index.Index` or a string containing the
        directory path where the index is to be stored.
    indexname
        String containing the name of the index, if multiple indexes are stored
        in the same directory.
    merge
        Merge small segments during commit?
    optimize
        Merge all segments together?

    """
    import whoosh.index

    # deal with polymorphic argument
    if isinstance(index_or_dirname, string_types):
        dirname = index_or_dirname
        index = whoosh.index.open_dir(dirname, indexname=indexname,
                                      readonly=False)
        needs_closing = True
    elif isinstance(index_or_dirname, whoosh.index.Index):
        index = index_or_dirname
        needs_closing = False
    else:
        raise ArgumentError('expected string or index, found %r'
                            % index_or_dirname)

    writer = index.writer()
    try:

        for d in dicts(table):
            writer.add_document(**d)
        writer.commit(merge=merge, optimize=optimize)

    except Exception:
        writer.cancel()
        raise

    finally:
        if needs_closing:
            index.close()
Esempio n. 3
0
def itertextindex(index_or_dirname, indexname, docnum_field):
    import whoosh.index

    if isinstance(index_or_dirname, string_types):
        dirname = index_or_dirname
        index = whoosh.index.open_dir(dirname,
                                      indexname=indexname,
                                      readonly=True)
        needs_closing = True
    elif isinstance(index_or_dirname, whoosh.index.Index):
        index = index_or_dirname
        needs_closing = False
    else:
        raise ArgumentError('expected string or index, found %r' %
                            index_or_dirname)

    try:

        if docnum_field is None:

            # figure out the field names
            hdr = tuple(index.schema.stored_names())
            yield hdr

            # yield all documents
            astuple = operator.itemgetter(*index.schema.stored_names())
            for _, stored_fields_dict in index.reader().iter_docs():
                yield astuple(stored_fields_dict)

        else:

            # figure out the field names
            hdr = (docnum_field, ) + tuple(index.schema.stored_names())
            yield hdr

            # yield all documents
            astuple = operator.itemgetter(*index.schema.stored_names())
            for docnum, stored_fields_dict in index.reader().iter_docs():
                yield (docnum, ) + astuple(stored_fields_dict)

    except:
        raise

    finally:
        if needs_closing:
            # close the index if we're the ones who opened it
            index.close()
Esempio n. 4
0
def itertextindex(index_or_dirname, indexname, docnum_field):
    import whoosh.index

    if isinstance(index_or_dirname, string_types):
        dirname = index_or_dirname
        index = whoosh.index.open_dir(dirname, indexname=indexname,
                                      readonly=True)
        needs_closing = True
    elif isinstance(index_or_dirname, whoosh.index.Index):
        index = index_or_dirname
        needs_closing = False
    else:
        raise ArgumentError('expected string or index, found %r'
                            % index_or_dirname)

    try:

        if docnum_field is None:

            # figure out the field names
            hdr = tuple(index.schema.stored_names())
            yield hdr

            # yield all documents
            astuple = operator.itemgetter(*index.schema.stored_names())
            for _, stored_fields_dict in index.reader().iter_docs():
                yield astuple(stored_fields_dict)

        else:

            # figure out the field names
            hdr = (docnum_field,) + tuple(index.schema.stored_names())
            yield hdr

            # yield all documents
            astuple = operator.itemgetter(*index.schema.stored_names())
            for docnum, stored_fields_dict in index.reader().iter_docs():
                yield (docnum,) + astuple(stored_fields_dict)

    except:
        raise

    finally:
        if needs_closing:
            # close the index if we're the ones who opened it
            index.close()
Esempio n. 5
0
def itersearchindex(index_or_dirname, query, limit, pagenum, pagelen, indexname,
                    docnum_field, score_field, fieldboosts, search_kwargs):
    import whoosh.index
    import whoosh.query
    import whoosh.qparser

    if not search_kwargs:
        search_kwargs = dict()

    if isinstance(index_or_dirname, string_types):
        dirname = index_or_dirname
        index = whoosh.index.open_dir(dirname,
                                      indexname=indexname,
                                      readonly=True)
        needs_closing = True
    elif isinstance(index_or_dirname, whoosh.index.Index):
        index = index_or_dirname
        needs_closing = False
    else:
        raise ArgumentError('expected string or index, found %r'
                            % index_or_dirname)

    try:

        # figure out header
        hdr = tuple()
        if docnum_field is not None:
            hdr += (docnum_field,)
        if score_field is not None:
            hdr += (score_field,)
        stored_names = tuple(index.schema.stored_names())
        hdr += stored_names
        yield hdr

        # parse the query
        if isinstance(query, string_types):
            # search all fields by default
            parser = whoosh.qparser.MultifieldParser(
                index.schema.names(),
                index.schema,
                fieldboosts=fieldboosts
            )
            query = parser.parse(query)
        elif isinstance(query, whoosh.query.Query):
            pass
        else:
            raise ArgumentError(
                'expected string or whoosh.query.Query, found %r' % query
            )

        # make a function to turn docs into tuples
        astuple = operator.itemgetter(*index.schema.stored_names())

        with index.searcher() as searcher:
            if limit is not None:
                results = searcher.search(query, limit=limit,
                                          **search_kwargs)
            else:
                results = searcher.search_page(query, pagenum,
                                               pagelen=pagelen,
                                               **search_kwargs)

            if docnum_field is None and score_field is None:

                for doc in results:
                    yield astuple(doc)

            else:

                for (docnum, score), doc in izip(results.items(), results):
                    row = tuple()
                    if docnum_field is not None:
                        row += (docnum,)
                    if score_field is not None:
                        row += (score,)
                    row += astuple(doc)
                    yield row

    except:
        raise

    finally:
        if needs_closing:
            # close the index if we're the ones who opened it
            index.close()


# TODO guess schema
Esempio n. 6
0
def totextindex(table, index_or_dirname, schema=None, indexname=None,
                merge=False, optimize=False):
    """
    Load all rows from `table` into a Whoosh index. N.B., this will clear any
    existing data in the index before loading. E.g.::

        >>> import petl as etl
        >>> import datetime
        >>> import os
        >>> # here is the table we want to load into an index
        ... table = (('f0', 'f1', 'f2', 'f3', 'f4'),
        ...          ('AAA', 12, 4.3, True, datetime.datetime.now()),
        ...          ('BBB', 6, 3.4, False, datetime.datetime(1900, 1, 31)),
        ...          ('CCC', 42, 7.8, True, datetime.datetime(2100, 12, 25)))
        >>> # define a schema for the index
        ... from whoosh.fields import *
        >>> schema = Schema(f0=TEXT(stored=True),
        ...                 f1=NUMERIC(int, stored=True),
        ...                 f2=NUMERIC(float, stored=True),
        ...                 f3=BOOLEAN(stored=True),
        ...                 f4=DATETIME(stored=True))
        >>> # load index
        ... dirname = 'example.whoosh'
        >>> if not os.path.exists(dirname):
        ...     os.mkdir(dirname)
        ...
        >>> etl.totextindex(table, dirname, schema=schema)

    Keyword arguments:

    table
        A table container with the data to be loaded.
    index_or_dirname
        Either an instance of `whoosh.index.Index` or a string containing the
        directory path where the index is to be stored.
    schema
        Index schema to use if creating the index.
    indexname
        String containing the name of the index, if multiple indexes are stored
        in the same directory.
    merge
        Merge small segments during commit?
    optimize
        Merge all segments together?

    """
    import whoosh.index
    import whoosh.writing

    # deal with polymorphic argument
    if isinstance(index_or_dirname, string_types):
        dirname = index_or_dirname
        index = whoosh.index.create_in(dirname, schema,
                                       indexname=indexname)
        needs_closing = True
    elif isinstance(index_or_dirname, whoosh.index.Index):
        index = index_or_dirname
        needs_closing = False
    else:
        raise ArgumentError('expected string or index, found %r'
                            % index_or_dirname)

    writer = index.writer()
    try:

        for d in dicts(table):
            writer.add_document(**d)
        writer.commit(merge=merge, optimize=optimize,
                      mergetype=whoosh.writing.CLEAR)

    except:
        writer.cancel()
        raise

    finally:
        if needs_closing:
            index.close()
Esempio n. 7
0
def itersearchindex(index_or_dirname, query, limit, pagenum, pagelen,
                    indexname, docnum_field, score_field, fieldboosts,
                    search_kwargs):
    import whoosh.index
    import whoosh.query
    import whoosh.qparser

    if not search_kwargs:
        search_kwargs = dict()

    if isinstance(index_or_dirname, string_types):
        dirname = index_or_dirname
        index = whoosh.index.open_dir(dirname,
                                      indexname=indexname,
                                      readonly=True)
        needs_closing = True
    elif isinstance(index_or_dirname, whoosh.index.Index):
        index = index_or_dirname
        needs_closing = False
    else:
        raise ArgumentError('expected string or index, found %r' %
                            index_or_dirname)

    try:

        # figure out header
        hdr = tuple()
        if docnum_field is not None:
            hdr += (docnum_field, )
        if score_field is not None:
            hdr += (score_field, )
        stored_names = tuple(index.schema.stored_names())
        hdr += stored_names
        yield hdr

        # parse the query
        if isinstance(query, string_types):
            # search all fields by default
            parser = whoosh.qparser.MultifieldParser(index.schema.names(),
                                                     index.schema,
                                                     fieldboosts=fieldboosts)
            query = parser.parse(query)
        elif isinstance(query, whoosh.query.Query):
            pass
        else:
            raise ArgumentError(
                'expected string or whoosh.query.Query, found %r' % query)

        # make a function to turn docs into tuples
        astuple = operator.itemgetter(*index.schema.stored_names())

        with index.searcher() as searcher:
            if limit is not None:
                results = searcher.search(query, limit=limit, **search_kwargs)
            else:
                results = searcher.search_page(query,
                                               pagenum,
                                               pagelen=pagelen,
                                               **search_kwargs)

            if docnum_field is None and score_field is None:

                for doc in results:
                    yield astuple(doc)

            else:

                for (docnum, score), doc in izip(results.items(), results):
                    row = tuple()
                    if docnum_field is not None:
                        row += (docnum, )
                    if score_field is not None:
                        row += (score, )
                    row += astuple(doc)
                    yield row

    except:
        raise

    finally:
        if needs_closing:
            # close the index if we're the ones who opened it
            index.close()


# TODO guess schema
Esempio n. 8
0
def totextindex(table,
                index_or_dirname,
                schema=None,
                indexname=None,
                merge=False,
                optimize=False):
    """
    Load all rows from `table` into a Whoosh index. N.B., this will clear any
    existing data in the index before loading. E.g.::

        >>> import petl as etl
        >>> import datetime
        >>> import os
        >>> # here is the table we want to load into an index
        ... table = (('f0', 'f1', 'f2', 'f3', 'f4'),
        ...          ('AAA', 12, 4.3, True, datetime.datetime.now()),
        ...          ('BBB', 6, 3.4, False, datetime.datetime(1900, 1, 31)),
        ...          ('CCC', 42, 7.8, True, datetime.datetime(2100, 12, 25)))
        >>> # define a schema for the index
        ... from whoosh.fields import *
        >>> schema = Schema(f0=TEXT(stored=True),
        ...                 f1=NUMERIC(int, stored=True),
        ...                 f2=NUMERIC(float, stored=True),
        ...                 f3=BOOLEAN(stored=True),
        ...                 f4=DATETIME(stored=True))
        >>> # load index
        ... dirname = 'example.whoosh'
        >>> if not os.path.exists(dirname):
        ...     os.mkdir(dirname)
        ...
        >>> etl.totextindex(table, dirname, schema=schema)

    Keyword arguments:

    table
        A table container with the data to be loaded.
    index_or_dirname
        Either an instance of `whoosh.index.Index` or a string containing the
        directory path where the index is to be stored.
    schema
        Index schema to use if creating the index.
    indexname
        String containing the name of the index, if multiple indexes are stored
        in the same directory.
    merge
        Merge small segments during commit?
    optimize
        Merge all segments together?

    """
    import whoosh.index
    import whoosh.writing

    # deal with polymorphic argument
    if isinstance(index_or_dirname, string_types):
        dirname = index_or_dirname
        index = whoosh.index.create_in(dirname, schema, indexname=indexname)
        needs_closing = True
    elif isinstance(index_or_dirname, whoosh.index.Index):
        index = index_or_dirname
        needs_closing = False
    else:
        raise ArgumentError('expected string or index, found %r' %
                            index_or_dirname)

    writer = index.writer()
    try:

        for d in dicts(table):
            writer.add_document(**d)
        writer.commit(merge=merge,
                      optimize=optimize,
                      mergetype=whoosh.writing.CLEAR)

    except:
        writer.cancel()
        raise

    finally:
        if needs_closing:
            index.close()
Esempio n. 9
0
def closeIndex(searcher, index):
  try:
    searcher.close()
    index.close()
  except Exception as ex:
    print ex