def test_function_step_with_context():
document_store = JsonDocumentStore("/tmp/test-json-store",
force_initialize=True)
document_store.add(create_document())
new_document_store = JsonDocumentStore("/tmp/test-json-store2",
force_initialize=True)
def my_function(doc, context):
doc.metadata.cheese = context.transaction_id
logging.error("Hello")
return doc
assert new_document_store.count() == 0
pipeline = Pipeline(document_store)
pipeline.add_step(my_function)
pipeline.set_sink(new_document_store)
stats = pipeline.run().statistics
assert stats.documents_processed == 1
assert stats.document_exceptions == 0
assert new_document_store.count() == 1
assert new_document_store.get_document(
0).metadata.cheese == pipeline.context.transaction_id
print(new_document_store.get_document(0).log)
def test_function_step_with_exception():
document_store = JsonDocumentStore("/tmp/test-json-store",
force_initialize=True)
document_store.add(create_document())
new_document_store = JsonDocumentStore("/tmp/test-json-store2",
force_initialize=True)
def my_function(doc):
doc.metadata.cheese = "fishstick"
raise Exception("hello world")
return doc
assert new_document_store.count() == 0
pipeline = Pipeline(document_store, stop_on_exception=False)
pipeline.add_step(my_function)
pipeline.set_sink(new_document_store)
stats = pipeline.run().statistics
assert stats.documents_processed == 1
assert stats.document_exceptions == 1
assert new_document_store.count() == 1
assert len(new_document_store.get_document(0).exceptions) == 1
print(new_document_store.get_document(0).exceptions)
def test_pipeline_example():
document_store = JsonDocumentStore("/tmp/test-json-store",
force_initialize=True)
document_store.add(create_document())
if Path("/tmp/test-json-store2/index..json").is_file():
os.remove("/tmp/test-json-store2")
new_document_store = JsonDocumentStore("/tmp/test-json-store2",
force_initialize=True)
assert new_document_store.count() == 0
pipeline = Pipeline(document_store)
pipeline.set_sink(new_document_store)
stats = pipeline.run().statistics
assert stats.documents_processed == 1
assert new_document_store.count() == 1