def get_status_handler(self, advanced=False):
""" Returns a status handler that returns True
each time it receives a True from parents.
Experimental: if "advanced" is set to True,
uses an intelligent status handler that looks for parent nodes
until it finds a splitting node and consumes
all the buffer on each branch to avoid high memory usage """
if advanced:
# def print_status(info, status_getter):
# def new_getter():
# v = status_getter()
# print 'getter for %s: %s' % (info, v)
#
# return new_getter
# buffer_num_getters = [print_status(node.name,
# node.get_input_buffer)
# for node in self.consumer_nodes]
buffer_num_getters = [node.get_input_buffer
for node in self.consumer_nodes]
final_consumer = runner(status_lookup,
dict(buffer_num_getters=buffer_num_getters))
for consumer in self.consumers:
final_consumer.connect_in("sources", consumer)
return final_consumer('out')
else:
final_consumer = runner(all_true_longest)
for num, consumer in enumerate(self.consumers):
final_consumer.connect_in("sources", consumer)
return final_consumer('out')
def initialize(self):
""" Creates the Node objects from the dictionnary data structure,
set up the links between them and prelink_up then link_up the consumers.
"""
for node_id, node_data in self.nodes.items():
self.node_objects[node_id] = Node(
node_id,
runner(node_data['component'],
node_data.get('args', dict()),
handle_bypass=False),
in_port=node_data.get('in_port'),
out_port=node_data.get('out_port'),
joiner=node_data.get('joiner'),
splitter=node_data.get('splitter'),
joiner_strip_bypass=node_data.get('joiner_strip_bypass', True),
deffer_to_process=node_data.get('deffer_to_process', False))
for node_id, node_data in self.nodes.items():
node = self.node_objects[node_id]
for in_node in node_data.get('before', list()):
node.add_input(self.node_objects.get(in_node))
self.node_objects.get(in_node).add_output(node)
for out_node in node_data.get('after', list()):
node.add_output(self.node_objects.get(out_node))
self.node_objects.get(out_node).add_input(node)
for node in self.consumer_nodes:
node.prelink_up()
for node in self.consumer_nodes:
node.link_up()
def get_generator(self):
""" Sets up all the sources in the runner and returns an iterator over
the output port of the junction runner.
Warning, this function shouldn't be called twice !
"""
if self.runner is None:
extra_args = {
'ports': list(map(itemgetter(1), self.input_sources))
}
func_spec = inspect.getargspec(self.joiner)
func_kwargs = func_spec[0][len(func_spec[3]) * -1:]
if "metadata" in func_kwargs:
extra_args['metadata'] = \
{'get_buffer_state':self.node.get_buffer_state,
'get_input_buffer':self.node.get_input_buffer}
self.runner = runner(self.joiner, extra_args, handle_bypass=False)
for input_source, name in self.input_sources:
generator = input_source
if self.strip_bypass:
generator = self.continuation_stripper(generator)
self.runner.connect_in('sources', generator)
if self.strip_bypass:
return self.output_checker(self.runner('out'))
else:
return self.runner('out')
def prepare_splitter(self, splitter_function, size):
args = inspect.getargspec(splitter_function).args
kwargs = dict()
if "size" in args:
kwargs['size'] = size
if "ports" in args:
kwargs['ports'] = list(
map(operator.attrgetter('name'), self.output_ports))
return runner(splitter_function, kwargs, handle_bypass=False)
def get_default_splitter(self, size):
""" Returns a default splitter (basically the same than
an itertools tee: pass the same data in all outputs) """
return runner(splitm, dict(size=size), handle_bypass=False)