def test_node_dict_chained():
strategy = NaiveStrategy(GraphExecutionContextType=BufferingGraphExecutionContext)
def f():
return {'id': 1, 'name': 'foo'}
def uppercase_name(values):
return {**values, 'name': values['name'].upper()}
graph = Graph(f, uppercase_name)
context = strategy.execute(graph)
output = context.get_buffer()
assert len(output) == 1
assert output[0] == ({'id': 1, 'name': 'FOO'},)
def g():
yield {'id': 1, 'name': 'foo'}
yield {'id': 2, 'name': 'bar'}
graph = Graph(g, uppercase_name)
context = strategy.execute(graph)
output = context.get_buffer()
assert len(output) == 2
assert output[0] == ({'id': 1, 'name': 'FOO'},)
assert output[1] == ({'id': 2, 'name': 'BAR'},)
def test_node_dict_chained():
strategy = NaiveStrategy(GraphExecutionContextType=BufferingGraphExecutionContext)
def f():
return {"id": 1, "name": "foo"}
def uppercase_name(values):
return {**values, "name": values["name"].upper()}
graph = Graph(f, uppercase_name)
context = strategy.execute(graph)
output = context.get_buffer()
assert len(output) == 1
assert output[0] == ({"id": 1, "name": "FOO"},)
def g():
yield {"id": 1, "name": "foo"}
yield {"id": 2, "name": "bar"}
graph = Graph(g, uppercase_name)
context = strategy.execute(graph)
output = context.get_buffer()
assert len(output) == 2
assert output[0] == ({"id": 1, "name": "FOO"},)
assert output[1] == ({"id": 2, "name": "BAR"},)
def test_node_tuple_chained():
strategy = NaiveStrategy(GraphExecutionContextType=BufferingGraphExecutionContext)
def uppercase(*args):
return tuple(map(str.upper, args))
def f():
return 'foo', 'bar'
graph = Graph(f, uppercase)
context = strategy.execute(graph)
output = context.get_buffer()
assert len(output) == 1
assert output[0] == ('FOO', 'BAR')
def g():
yield 'foo', 'bar'
yield 'foo', 'baz'
graph = Graph(g, uppercase)
context = strategy.execute(graph)
output = context.get_buffer()
assert len(output) == 2
assert output[0] == ('FOO', 'BAR')
assert output[1] == ('FOO', 'BAZ')
def test_node_tuple_chained():
strategy = NaiveStrategy(GraphExecutionContextType=BufferingGraphExecutionContext)
def uppercase(*args):
return tuple(map(str.upper, args))
def f():
return "foo", "bar"
graph = Graph(f, uppercase)
context = strategy.execute(graph)
output = context.get_buffer()
assert len(output) == 1
assert output[0] == ("FOO", "BAR")
def g():
yield "foo", "bar"
yield "foo", "baz"
graph = Graph(g, uppercase)
context = strategy.execute(graph)
output = context.get_buffer()
assert len(output) == 2
assert output[0] == ("FOO", "BAR")
assert output[1] == ("FOO", "BAZ")
def test_execution():
graph = Graph()
graph.add_chain(*chain)
strategy = NaiveStrategy()
ctx = strategy.execute(graph)
assert ctx.results == [1, 4, 9, 16, 25, 36, 49, 64, 81]
def test_empty_execution_context():
graph = Graph()
ctx = GraphExecutionContext(graph)
assert not len(ctx.nodes)
assert not len(ctx.plugins)
assert not ctx.alive
def run(graph, *chain, strategy=None, plugins=None):
strategy = create_strategy(strategy)
if len(chain):
warnings.warn(
'DEPRECATED. You should pass a Graph instance instead of a chain.')
from bonobo import Graph
graph = Graph(graph, *chain)
return strategy.execute(graph, plugins=plugins)
def test_lifecycle_of_graph_with_recoverable_error():
graph = Graph([1, 2, 3], raise_an_error, print)
with GraphExecutionContext(graph) as context:
assert context.started
assert context.alive
assert not context.stopped
assert context.started
assert not context.alive
assert context.stopped
assert not context.xstatus
def test_lifecycle_of_graph_with_unrecoverable_error():
graph = Graph([1, 2, 3], raise_an_unrecoverrable_error, print)
with GraphExecutionContext(graph) as context:
assert context.started and context.alive and not context.stopped
context.write(BEGIN, EMPTY, END)
context.loop()
assert context.started
assert not context.alive
assert context.stopped
assert not context.xstatus
def test_lifecycle_of_nonempty_graph():
graph = Graph([1, 2, 3], print)
with GraphExecutionContext(graph) as context:
assert context.started
assert context.alive
assert not context.stopped
assert context.started
assert not context.alive
assert context.stopped
assert not context.xstatus
def test_simple_execution_context():
graph = Graph()
graph.add_chain(*chain)
ctx = GraphExecutionContext(graph)
assert len(ctx.nodes) == len(chain)
assert not len(ctx.plugins)
for i, node in enumerate(chain):
assert ctx[i].wrapped is node
assert not ctx.alive
ctx.recv(BEGIN, Bag(), END)
assert not ctx.alive
ctx.start()
assert ctx.alive
def run(graph, *chain, strategy=None, plugins=None):
if len(chain):
warnings.warn('DEPRECATED. You should pass a Graph instance instead of a chain.')
from bonobo import Graph
graph = Graph(graph, *chain)
strategy = create_strategy(strategy)
plugins = []
if _is_interactive_console():
from bonobo.ext.console import ConsoleOutputPlugin
if ConsoleOutputPlugin not in plugins:
plugins.append(ConsoleOutputPlugin)
if _is_jupyter_notebook():
from bonobo.ext.jupyter import JupyterOutputPlugin
if JupyterOutputPlugin not in plugins:
plugins.append(JupyterOutputPlugin)
return strategy.execute(graph, plugins=plugins)
}
"""
from random import randint
from bonobo import Bag, Graph
def extract():
yield Bag(topic='foo')
yield Bag(topic='bar')
yield Bag(topic='baz')
def transform(topic: str):
return Bag.inherit(title=topic.title(), rand=randint(10, 99))
def load(topic: str, title: str, rand: int):
print('{} ({}) wait={}'.format(title, topic, rand))
graph = Graph()
graph.add_chain(extract, transform, load)
if __name__ == '__main__':
from bonobo import run
run(graph)
from bonobo import FileReader, Graph, get_examples_path
def skip_comments(line):
if not line.startswith('#'):
yield line
graph = Graph(
FileReader(path=get_examples_path('datasets/passwd.txt')),
skip_comments,
lambda s: s.split(':'),
lambda l: l[0],
print,
)
if __name__ == '__main__':
import bonobo
bonobo.run(graph)
' '.join(
filter(None,
(row.get('postal_code', None), row.get('city', None)))),
row.get('county', None),
row.get('country'),
)))
print(' - {}: {address}'.format(t.blue('address'),
address=', '.join(address)))
print(' - {}: {links}'.format(t.blue('links'),
links=', '.join(row['links'])))
print(' - {}: {geometry}'.format(t.blue('geometry'), **row))
print(' - {}: {source}'.format(t.blue('source'),
source='datanova/' + API_DATASET))
graph = Graph(
OpenDataSoftAPI(dataset=API_DATASET,
netloc=API_NETLOC,
timezone='Europe/Paris'),
normalize,
filter_france,
Tee(display),
JsonWriter(path=get_examples_path('datasets/fablabs.txt')),
)
if __name__ == '__main__':
from bonobo import run
run(graph)
"""
from random import randint
from bonobo import Graph
def extract():
yield {'topic': 'foo'}
yield {'topic': 'bar'}
yield {'topic': 'baz'}
def transform(row: dict):
return {
'topic': row['topic'].title(),
'randint': randint(10, 99),
}
def load(row: dict):
print(row)
graph = Graph(extract, transform, load)
if __name__ == '__main__':
from bonobo import run
run(graph)
from bonobo import Graph, ThreadPoolExecutorStrategy
def yield_from(*args):
yield from args
# Represent our data processor as a simple directed graph of callables.
graph = Graph(
lambda: (x for x in ('foo', 'bar', 'baz')),
str.upper,
print,
)
# Use a thread pool.
executor = ThreadPoolExecutorStrategy()
# Run the thing.
executor.execute(graph)
from bonobo import CsvReader, Graph, get_examples_path
graph = Graph(
CsvReader(path=get_examples_path('datasets/coffeeshops.txt')),
print,
)
if __name__ == '__main__':
import bonobo
bonobo.run(graph)