def schedule_refinement(self, g, tick, task, inputs):
"""
Refine the graph.
:param g: Graph to be refined.
:param tick: Tick of the task being refined.
:param task: Task being refined.
:param inputs: port -> valueref for the refiner ports.
"""
me = anycall.RPCSystem.default.local_worker #@UndefinedVariable
meremote = anycall.RPCSystem.default.local_remoteworker #@UndefinedVariable
logger.debug("Refinement request for %r. Transferring inputs to %r" %
(tick, me))
in_ports = list(inputs.iterkeys())
# First, get all the required data here.
data_transfers = []
for port in in_ports:
valueref = inputs[port]
source = self._strategy.choose_source_worker(valueref, meremote)
if hasattr(task, 'refiner_reducer'):
reducer = task.refiner_reducer.get(port, None)
else:
reducer = None
if reducer is None:
# Get the complete value.
# We could use a `lambda x:x` reducer, but this way
# the value ends up in our store, so we might not have
# to transfer it again.
d = me.fetch_from(source, valueref.valueid)
def fetched(_):
return me.get_value(valueref.valueid)
d.addCallback(fetched)
else:
d = source.reduce(valueref.valueid, reducer)
data_transfers.append(d)
transfer_results = yield defer.DeferredList(data_transfers,
fireOnOneErrback=True)
input_values = {}
for i, port in enumerate(in_ports):
_, value = transfer_results[i]
input_values[port] = value
logger.info("Refining %r..." % tick)
# Lets change the graph
task.refine(g, tick, input_values)
logger.info("Refining of %r completed." % tick)
def schedule_refinement(self, g, tick, task, inputs):
"""
Refine the graph.
:param g: Graph to be refined.
:param tick: Tick of the task being refined.
:param task: Task being refined.
:param inputs: port -> valueref for the refiner ports.
"""
me = anycall.RPCSystem.default.local_worker #@UndefinedVariable
meremote = anycall.RPCSystem.default.local_remoteworker #@UndefinedVariable
logger.debug("Refinement request for %r. Transferring inputs to %r" % (tick, me))
in_ports = list(inputs.iterkeys())
# First, get all the required data here.
data_transfers = []
for port in in_ports:
valueref = inputs[port]
source = self._strategy.choose_source_worker(valueref, meremote)
if hasattr(task, 'refiner_reducer'):
reducer = task.refiner_reducer.get(port, None)
else:
reducer = None
if reducer is None:
# Get the complete value.
# We could use a `lambda x:x` reducer, but this way
# the value ends up in our store, so we might not have
# to transfer it again.
d = me.fetch_from(source, valueref.valueid)
def fetched(_):
return me.get_value(valueref.valueid)
d.addCallback(fetched)
else:
d = source.reduce(valueref.valueid, reducer)
data_transfers.append(d)
transfer_results = yield defer.DeferredList(data_transfers, fireOnOneErrback = True)
input_values = {}
for i, port in enumerate(in_ports):
_, value = transfer_results[i]
input_values[port] = value
logger.info("Refining %r..." % tick)
# Lets change the graph
task.refine(g, tick, input_values)
logger.info("Refining of %r completed." % tick)
def schedule_refinement(self, g, tick, task, inputs):
"""
Refine the graph.
:param g: Graph to be refined.
:param tick: Tick of the task being refined.
:param task: Task being refined.
:param inputs: portname, value for the refiner ports.
"""
logger.debug("Refining task with tick %s." % tick)
return task.refine(g, tick, inputs)
