def refine(self, graph, tick, known_inputs): """ Refines the graph - here, means that the sub-graph defined as body is connected with the parent graph. Then the original node with the given task is removed - i.e. the graph defined as body of the NestedGraphTask replaces the node with the given task. @param graph: parent graph @param tick: identifies the node the given NestedGraphTask is attached to. @param known_inputs: Known inputs are not needed at the moment - passed to comply with the general refine method signature. """ parentpath = _dataflow_path(graph, tick) in_connections = self._in_connections(graph, tick) out_connections = self._out_connections(graph, tick) # remove task self._remove_task(graph, tick) # insert the subgraph insert_subgraph(graph, self.body_graph, tick) for source, dest in in_connections: graph.connect(source, dest) for source, dest in out_connections: graph.connect(source, dest) # adjust path elements in the tasks of the subgraph addedticks = _filter_for_common_parent(graph, tick) for t in addedticks: _extend_dataflow_path(graph, t, parentpath) return defer.succeed(None)
def refine(self, graph, tick, known_inputs): """ Refines the graph - here, means that the sub-graph defined as body is connected with the parent graph. Then the original node with the given task is removed - i.e. the graph defined as body of the NestedGraphTask replaces the node with the given task. @param graph: parent graph @param tick: identifies the node the given NestedGraphTask is attached to. @param known_inputs: Known inputs are not needed at the moment - passed to comply with the general refine method signature. """ parentpath =_dataflow_path(graph, tick) in_connections = self._in_connections(graph, tick) out_connections = self._out_connections(graph, tick) # remove task self._remove_task(graph, tick) # insert the subgraph insert_subgraph(graph, self.body_graph, tick) for source, dest in in_connections: graph.connect(source, dest) for source, dest in out_connections: graph.connect(source, dest) # adjust path elements in the tasks of the subgraph addedticks=_filter_for_common_parent(graph, tick) for t in addedticks: _extend_dataflow_path(graph, t, parentpath) return defer.succeed(None)
def adjust_graph(self, g, tick, input_list): parentpath = _dataflow_path(g, tick) # task to read from list at tick: (tick,1) - # no inputs are needed for this task since data passed at refinement time map_task = _mapPortsTask(self.iterator_port, input_list) map_task_tick = graph.START_TICK + 1 << tick mappath = "%s.map" % parentpath g.add_task(map_task_tick, map_task, {'name': 'map', 'path': mappath}) # iterator port data source, _ = [ conn for conn in g.get_in_connections(tick) if conn[1].port == self.iterator_port ][0] dest = graph.Endpoint(map_task_tick, 'inputlist') g.connect(source, dest) # connect CONTEXT source = graph.Endpoint(START_TICK, CONTEXT) dest = graph.Endpoint(map_task_tick, CONTEXT) g.connect(source, dest) # tick to extend for the iterations iterations_tick = map_task_tick + 1 # task to collect elements into list at tick: (tick,3) - # no inputs are needed for this task since data are passed in at refinement time reduce_inputnames = [ source.port for source, dest in self.body_graph.get_in_connections( graph.FINAL_TICK) ] _, reduce_out_dest = g.get_out_connections(tick)[0] reduce_task = _reducePortsTask(len(input_list), reduce_inputnames, self.outputname) reduce_tick = iterations_tick + 1 reducepath = "%s.reduce" % parentpath g.add_task(reduce_tick, reduce_task, { 'name': 'reduce', 'path': reducepath }) # context source = graph.Endpoint(START_TICK, CONTEXT) dest = graph.Endpoint(reduce_tick, CONTEXT) g.connect(source, dest) reduce_task_source = graph.Endpoint(reduce_tick, self.outputname) # iterations input_map = { dest.port: source for source, dest in g.get_in_connections(tick) if dest.port != self.iterator_port } for iteration_counter in range(len(input_list)): # tick to expand the sub-graph tasks from iteration_tick = graph.START_TICK + iteration_counter + 1 << iterations_tick # connections for the inputs - iterator_port_source = graph.Endpoint( map_task_tick, "%s_%s" % (self.iterator_port, iteration_counter + 1)) in_connections = self._in_connections(iteration_tick, input_map, iterator_port_source) # connection for the output tuple out_connections = self._out_connections(iteration_tick, reduce_tick, iteration_counter) # insert the subgraph insert_subgraph(g, self.body_graph, iteration_tick) # add connections for source, dest in in_connections: g.connect(source, dest) for source, dest in out_connections: g.connect(source, dest) addedticks = _filter_for_common_parent(g, iteration_tick) for t in addedticks: parentpath_it = "%s.iterations.%i" % (parentpath, iteration_counter + 1) _extend_dataflow_path(g, t, parentpath_it) self._remove_task(g, tick) g.connect(reduce_task_source, reduce_out_dest)
def adjust_graph(self, g, tick, input_list): parentpath=_dataflow_path(g, tick) # task to read from list at tick: (tick,1) - # no inputs are needed for this task since data passed at refinement time map_task=_mapPortsTask(self.iterator_port, input_list) map_task_tick=graph.START_TICK+1<<tick mappath="%s.map"%parentpath g.add_task(map_task_tick, map_task, {'name':'map', 'path':mappath}) # iterator port data source, _ = [conn for conn in g.get_in_connections(tick) if conn[1].port == self.iterator_port][0] dest = graph.Endpoint(map_task_tick, 'inputlist') g.connect(source, dest) # connect CONTEXT source=graph.Endpoint(START_TICK, CONTEXT) dest=graph.Endpoint(map_task_tick, CONTEXT) g.connect(source, dest) # tick to extend for the iterations iterations_tick=map_task_tick+1 # task to collect elements into list at tick: (tick,3) - # no inputs are needed for this task since data are passed in at refinement time reduce_inputnames=[source.port for source,dest in self.body_graph.get_in_connections(graph.FINAL_TICK)] _, reduce_out_dest = g.get_out_connections(tick)[0] reduce_task=_reducePortsTask(len(input_list), reduce_inputnames, self.outputname) reduce_tick=iterations_tick+1 reducepath="%s.reduce"%parentpath g.add_task(reduce_tick, reduce_task, {'name':'reduce', 'path':reducepath}) # context source=graph.Endpoint(START_TICK, CONTEXT) dest=graph.Endpoint(reduce_tick, CONTEXT) g.connect(source, dest) reduce_task_source=graph.Endpoint(reduce_tick, self.outputname) # iterations input_map = {dest.port: source for source, dest in g.get_in_connections(tick) if dest.port != self.iterator_port} for iteration_counter in range(len(input_list)): # tick to expand the sub-graph tasks from iteration_tick = graph.START_TICK + iteration_counter+1 << iterations_tick # connections for the inputs - iterator_port_source=graph.Endpoint(map_task_tick, "%s_%s"%(self.iterator_port,iteration_counter+1)) in_connections = self._in_connections(iteration_tick, input_map, iterator_port_source) # connection for the output tuple out_connections=self._out_connections(iteration_tick, reduce_tick, iteration_counter) # insert the subgraph insert_subgraph(g, self.body_graph, iteration_tick) # add connections for source, dest in in_connections: g.connect(source, dest) for source, dest in out_connections: g.connect(source, dest) addedticks=_filter_for_common_parent(g, iteration_tick) for t in addedticks: parentpath_it="%s.iterations.%i"%(parentpath,iteration_counter+1) _extend_dataflow_path(g, t, parentpath_it) self._remove_task(g, tick) g.connect(reduce_task_source,reduce_out_dest)