def schedule_refinement(self, g, tick, task, inputs): """ Refine the graph. :param g: Graph to be refined. :param tick: Tick of the task being refined. :param task: Task being refined. :param inputs: port -> valueref for the refiner ports. """ me = anycall.RPCSystem.default.local_worker #@UndefinedVariable meremote = anycall.RPCSystem.default.local_remoteworker #@UndefinedVariable logger.debug("Refinement request for %r. Transferring inputs to %r" % (tick, me)) in_ports = list(inputs.iterkeys()) # First, get all the required data here. data_transfers = [] for port in in_ports: valueref = inputs[port] source = self._strategy.choose_source_worker(valueref, meremote) if hasattr(task, 'refiner_reducer'): reducer = task.refiner_reducer.get(port, None) else: reducer = None if reducer is None: # Get the complete value. # We could use a `lambda x:x` reducer, but this way # the value ends up in our store, so we might not have # to transfer it again. d = me.fetch_from(source, valueref.valueid) def fetched(_): return me.get_value(valueref.valueid) d.addCallback(fetched) else: d = source.reduce(valueref.valueid, reducer) data_transfers.append(d) transfer_results = yield defer.DeferredList(data_transfers, fireOnOneErrback=True) input_values = {} for i, port in enumerate(in_ports): _, value = transfer_results[i] input_values[port] = value logger.info("Refining %r..." % tick) # Lets change the graph task.refine(g, tick, input_values) logger.info("Refining of %r completed." % tick)
def schedule_refinement(self, g, tick, task, inputs): """ Refine the graph. :param g: Graph to be refined. :param tick: Tick of the task being refined. :param task: Task being refined. :param inputs: port -> valueref for the refiner ports. """ me = anycall.RPCSystem.default.local_worker #@UndefinedVariable meremote = anycall.RPCSystem.default.local_remoteworker #@UndefinedVariable logger.debug("Refinement request for %r. Transferring inputs to %r" % (tick, me)) in_ports = list(inputs.iterkeys()) # First, get all the required data here. data_transfers = [] for port in in_ports: valueref = inputs[port] source = self._strategy.choose_source_worker(valueref, meremote) if hasattr(task, 'refiner_reducer'): reducer = task.refiner_reducer.get(port, None) else: reducer = None if reducer is None: # Get the complete value. # We could use a `lambda x:x` reducer, but this way # the value ends up in our store, so we might not have # to transfer it again. d = me.fetch_from(source, valueref.valueid) def fetched(_): return me.get_value(valueref.valueid) d.addCallback(fetched) else: d = source.reduce(valueref.valueid, reducer) data_transfers.append(d) transfer_results = yield defer.DeferredList(data_transfers, fireOnOneErrback = True) input_values = {} for i, port in enumerate(in_ports): _, value = transfer_results[i] input_values[port] = value logger.info("Refining %r..." % tick) # Lets change the graph task.refine(g, tick, input_values) logger.info("Refining of %r completed." % tick)
def schedule_refinement(self, g, tick, task, inputs): """ Refine the graph. :param g: Graph to be refined. :param tick: Tick of the task being refined. :param task: Task being refined. :param inputs: portname, value for the refiner ports. """ logger.debug("Refining task with tick %s." % tick) return task.refine(g, tick, inputs)