Python process_documents Examples

Example #1

def process(foreign_id=None, index=False, retry=False):
    """Re-process documents in the given collection."""
    collection_id = None
    if foreign_id:
        collection_id = get_collection(foreign_id).id
    process_documents(collection_id=collection_id,
                      index_only=index,
                      failed_only=retry)

Example #2

File: memory_leak.py Project: wayne9qiu/aleph

# Notes on debugging memory leaks in the worker

# apt-get update && apt-get install graphviz
# pip install objgraph

import objgraph
import gc
import random
from aleph.logic.documents import process_documents

objgraph.show_most_common_types()

process_documents(21)
# objgraph.show_most_common_types()

objgraph.show_chain(
    objgraph.find_backref_chain(
        random.choice(objgraph.by_type("_DebugQueryTuple")),
        objgraph.is_proper_module),
    filename="chain.png",
)  # noqa
gc.collect()

Example #3

File: memory_leak.py Project: pudo/aleph

# Notes on debugging memory leaks in the worker

# apt-get update && apt-get install graphviz
# pip install objgraph

import objgraph
import gc
import random
from aleph.logic.documents import process_documents
objgraph.show_most_common_types()

process_documents(21)
# objgraph.show_most_common_types()

objgraph.show_chain(objgraph.find_backref_chain(random.choice(objgraph.by_type('_DebugQueryTuple')), objgraph.is_proper_module), filename='chain.png')  # noqa
gc.collect()