def get_output(self, row):
if self.ioformat == settings.IOFORMAT_ARG0:
return row
if self.ioformat == settings.IOFORMAT_KWARGS:
return Bag(**row)
raise NotImplementedError('Unsupported format.')
def execute(self, graph, *args, plugins=None, services=None, **kwargs):
context = self.create_graph_execution_context(graph,
plugins=plugins,
services=services)
context.recv(BEGIN, Bag(), END)
executor = self.create_executor()
futures = []
for plugin_context in context.plugins:
def _runner(plugin_context=plugin_context):
try:
plugin_context.start()
plugin_context.loop()
plugin_context.stop()
except Exception as exc:
print_error(exc,
traceback.format_exc(),
prefix='Error in plugin context',
context=plugin_context)
futures.append(executor.submit(_runner))
for node_context in context.nodes:
def _runner(node_context=node_context):
try:
node_context.start()
except Exception as exc:
print_error(exc,
traceback.format_exc(),
prefix='Could not start node context',
context=node_context)
node_context.input.on_end()
else:
node_context.loop()
try:
node_context.stop()
except Exception as exc:
print_error(exc,
traceback.format_exc(),
prefix='Could not stop node context',
context=node_context)
futures.append(executor.submit(_runner))
while context.alive:
time.sleep(0.2)
for plugin_context in context.plugins:
plugin_context.shutdown()
executor.shutdown()
return context
def arg0_to_kwargs(row):
"""
Transform items in a stream from "arg0" format (each call only has one positional argument, which is a dict-like
object) to "kwargs" format (each call only has keyword arguments that represent a row).
:param row:
:return: bonobo.Bag
"""
return Bag(**row)
def execute(self, graph, *args, plugins=None, **kwargs):
context = self.create_graph_execution_context(graph, plugins=plugins)
context.write(BEGIN, Bag(), END)
# TODO: how to run plugins in "naive" mode ?
context.start()
context.loop()
context.stop()
return context
def create_buffer(self, context, engine, connection, table):
"""
This context processor creates a "buffer" of yet to be persisted elements, and commits the remaining elements
when the transformation ends.
:param engine:
:param connection:
"""
buffer = yield Queue()
for row in self.commit(table, connection, buffer, force=True):
context.send(Bag(row))
def _resolve(input_bag, output):
# NotModified means to send the input unmodified to output.
if output is NOT_MODIFIED:
return input_bag
if is_error(output):
return output
# If it does not look like a bag, let's create one for easier manipulation
if hasattr(output, 'apply'):
# Already a bag? Check if we need to set parent.
if INHERIT_INPUT in output.flags:
output.set_parent(input_bag)
else:
# Not a bag? Let's encapsulate it.
output = Bag(output)
return output
def _count_counter(self, context):
counter = ValueHolder(0)
yield counter
context.send(Bag(counter._value))
def read(self, fs, file):
for line in self.loader(file).items():
yield Bag(*line)