def refine(self, graph, tick, known_inputs):
"""
Refines the graph - here, means that the sub-graph defined as body is connected with the parent graph.
Then the original node with the given task is removed - i.e. the graph defined as body of the NestedGraphTask
replaces the node with the given task.
@param graph: parent graph
@param tick: identifies the node the given NestedGraphTask is attached to.
@param known_inputs: Known inputs are not needed at the moment - passed to comply with the general refine
method signature.
"""
parentpath = _dataflow_path(graph, tick)
in_connections = self._in_connections(graph, tick)
out_connections = self._out_connections(graph, tick)
# remove task
self._remove_task(graph, tick)
# insert the subgraph
insert_subgraph(graph, self.body_graph, tick)
for source, dest in in_connections:
graph.connect(source, dest)
for source, dest in out_connections:
graph.connect(source, dest)
# adjust path elements in the tasks of the subgraph
addedticks = _filter_for_common_parent(graph, tick)
for t in addedticks:
_extend_dataflow_path(graph, t, parentpath)
return defer.succeed(None)
def refine(self, graph, tick, known_inputs):
"""
Refines the graph - here, means that the sub-graph defined as body is connected with the parent graph.
Then the original node with the given task is removed - i.e. the graph defined as body of the NestedGraphTask
replaces the node with the given task.
@param graph: parent graph
@param tick: identifies the node the given NestedGraphTask is attached to.
@param known_inputs: Known inputs are not needed at the moment - passed to comply with the general refine
method signature.
"""
parentpath =_dataflow_path(graph, tick)
in_connections = self._in_connections(graph, tick)
out_connections = self._out_connections(graph, tick)
# remove task
self._remove_task(graph, tick)
# insert the subgraph
insert_subgraph(graph, self.body_graph, tick)
for source, dest in in_connections:
graph.connect(source, dest)
for source, dest in out_connections:
graph.connect(source, dest)
# adjust path elements in the tasks of the subgraph
addedticks=_filter_for_common_parent(graph, tick)
for t in addedticks:
_extend_dataflow_path(graph, t, parentpath)
return defer.succeed(None)
def adjust_graph(self, g, tick, input_list):
parentpath = _dataflow_path(g, tick)
# task to read from list at tick: (tick,1) -
# no inputs are needed for this task since data passed at refinement time
map_task = _mapPortsTask(self.iterator_port, input_list)
map_task_tick = graph.START_TICK + 1 << tick
mappath = "%s.map" % parentpath
g.add_task(map_task_tick, map_task, {'name': 'map', 'path': mappath})
# iterator port data
source, _ = [
conn for conn in g.get_in_connections(tick)
if conn[1].port == self.iterator_port
][0]
dest = graph.Endpoint(map_task_tick, 'inputlist')
g.connect(source, dest)
# connect CONTEXT
source = graph.Endpoint(START_TICK, CONTEXT)
dest = graph.Endpoint(map_task_tick, CONTEXT)
g.connect(source, dest)
# tick to extend for the iterations
iterations_tick = map_task_tick + 1
# task to collect elements into list at tick: (tick,3) -
# no inputs are needed for this task since data are passed in at refinement time
reduce_inputnames = [
source.port for source, dest in self.body_graph.get_in_connections(
graph.FINAL_TICK)
]
_, reduce_out_dest = g.get_out_connections(tick)[0]
reduce_task = _reducePortsTask(len(input_list), reduce_inputnames,
self.outputname)
reduce_tick = iterations_tick + 1
reducepath = "%s.reduce" % parentpath
g.add_task(reduce_tick, reduce_task, {
'name': 'reduce',
'path': reducepath
})
# context
source = graph.Endpoint(START_TICK, CONTEXT)
dest = graph.Endpoint(reduce_tick, CONTEXT)
g.connect(source, dest)
reduce_task_source = graph.Endpoint(reduce_tick, self.outputname)
# iterations
input_map = {
dest.port: source
for source, dest in g.get_in_connections(tick)
if dest.port != self.iterator_port
}
for iteration_counter in range(len(input_list)):
# tick to expand the sub-graph tasks from
iteration_tick = graph.START_TICK + iteration_counter + 1 << iterations_tick
# connections for the inputs -
iterator_port_source = graph.Endpoint(
map_task_tick,
"%s_%s" % (self.iterator_port, iteration_counter + 1))
in_connections = self._in_connections(iteration_tick, input_map,
iterator_port_source)
# connection for the output tuple
out_connections = self._out_connections(iteration_tick,
reduce_tick,
iteration_counter)
# insert the subgraph
insert_subgraph(g, self.body_graph, iteration_tick)
# add connections
for source, dest in in_connections:
g.connect(source, dest)
for source, dest in out_connections:
g.connect(source, dest)
addedticks = _filter_for_common_parent(g, iteration_tick)
for t in addedticks:
parentpath_it = "%s.iterations.%i" % (parentpath,
iteration_counter + 1)
_extend_dataflow_path(g, t, parentpath_it)
self._remove_task(g, tick)
g.connect(reduce_task_source, reduce_out_dest)
def adjust_graph(self, g, tick, input_list):
parentpath=_dataflow_path(g, tick)
# task to read from list at tick: (tick,1) -
# no inputs are needed for this task since data passed at refinement time
map_task=_mapPortsTask(self.iterator_port, input_list)
map_task_tick=graph.START_TICK+1<<tick
mappath="%s.map"%parentpath
g.add_task(map_task_tick, map_task, {'name':'map', 'path':mappath})
# iterator port data
source, _ = [conn for conn in g.get_in_connections(tick) if conn[1].port == self.iterator_port][0]
dest = graph.Endpoint(map_task_tick, 'inputlist')
g.connect(source, dest)
# connect CONTEXT
source=graph.Endpoint(START_TICK, CONTEXT)
dest=graph.Endpoint(map_task_tick, CONTEXT)
g.connect(source, dest)
# tick to extend for the iterations
iterations_tick=map_task_tick+1
# task to collect elements into list at tick: (tick,3) -
# no inputs are needed for this task since data are passed in at refinement time
reduce_inputnames=[source.port for source,dest in self.body_graph.get_in_connections(graph.FINAL_TICK)]
_, reduce_out_dest = g.get_out_connections(tick)[0]
reduce_task=_reducePortsTask(len(input_list), reduce_inputnames, self.outputname)
reduce_tick=iterations_tick+1
reducepath="%s.reduce"%parentpath
g.add_task(reduce_tick, reduce_task, {'name':'reduce', 'path':reducepath})
# context
source=graph.Endpoint(START_TICK, CONTEXT)
dest=graph.Endpoint(reduce_tick, CONTEXT)
g.connect(source, dest)
reduce_task_source=graph.Endpoint(reduce_tick, self.outputname)
# iterations
input_map = {dest.port: source for source, dest in g.get_in_connections(tick) if dest.port != self.iterator_port}
for iteration_counter in range(len(input_list)):
# tick to expand the sub-graph tasks from
iteration_tick = graph.START_TICK + iteration_counter+1 << iterations_tick
# connections for the inputs -
iterator_port_source=graph.Endpoint(map_task_tick, "%s_%s"%(self.iterator_port,iteration_counter+1))
in_connections = self._in_connections(iteration_tick, input_map, iterator_port_source)
# connection for the output tuple
out_connections=self._out_connections(iteration_tick, reduce_tick, iteration_counter)
# insert the subgraph
insert_subgraph(g, self.body_graph, iteration_tick)
# add connections
for source, dest in in_connections:
g.connect(source, dest)
for source, dest in out_connections:
g.connect(source, dest)
addedticks=_filter_for_common_parent(g, iteration_tick)
for t in addedticks:
parentpath_it="%s.iterations.%i"%(parentpath,iteration_counter+1)
_extend_dataflow_path(g, t, parentpath_it)
self._remove_task(g, tick)
g.connect(reduce_task_source,reduce_out_dest)
