def signal(self, sig):
dlist = []
for el in self.launchers:
d = el.signal(sig)
dlist.append(d)
dfinal = gatherBoth(dlist, consumeErrors=True)
return dfinal
def interrupt_then_kill(self, delay=1.0):
dlist = []
for el in self.launchers:
d = el.interrupt_then_kill(delay)
dlist.append(d)
dfinal = gatherBoth(dlist, consumeErrors=True)
return dfinal
def signal(self, sig):
dlist = []
for el in self.launchers:
d = el.signal(sig)
dlist.append(d)
dfinal = gatherBoth(dlist, consumeErrors=True)
return dfinal
def start(self, send_furl=False):
dlist = []
for host in self.engine_hosts.keys():
count = self.engine_hosts[host]
d = self._start(host, count, send_furl)
dlist.append(d)
return gatherBoth(dlist, consumeErrors=True)
def start(self, send_furl=False):
dlist = []
for host in self.engine_hosts.keys():
count = self.engine_hosts[host]
d = self._start(host, count, send_furl)
dlist.append(d)
return gatherBoth(dlist, consumeErrors=True)
def map(self, func, *sequences):
"""
Apply func to *sequences elementwise. Like Python's builtin map.
This version is load balanced.
"""
max_len = max(len(s) for s in sequences)
for s in sequences:
if len(s)!=max_len:
raise ValueError('all sequences must have equal length')
task_args = zip(*sequences)
task_ids = []
dlist = []
for ta in task_args:
task = MapTask(func, ta, clear_before=self.clear_before,
clear_after=self.clear_after, retries=self.retries,
recovery_task=self.recovery_task, depend=self.depend)
dlist.append(self.task_controller.run(task))
dlist = gatherBoth(dlist, consumeErrors=1)
dlist.addCallback(collect_exceptions,'map')
if self.block:
def get_results(task_ids):
d = self.task_controller.barrier(task_ids)
d.addCallback(lambda _: gatherBoth([self.task_controller.get_task_result(tid) for tid in task_ids], consumeErrors=1))
d.addCallback(collect_exceptions, 'map')
return d
dlist.addCallback(get_results)
return dlist
def interrupt_then_kill(self, delay=1.0):
dlist = []
for el in self.launchers:
d = el.interrupt_then_kill(delay)
dlist.append(d)
dfinal = gatherBoth(dlist, consumeErrors=True)
return dfinal
def do_gather(engines):
nEngines = len(engines)
mapClass = Map.dists[dist]
mapObject = mapClass()
d_list = []
# Loop through and push to each engine in non-blocking mode.
# This returns a set of deferreds to deferred_ids
for index, engineid in enumerate(engines):
d = self.pull(key, targets=engineid, block=False)
d_list.append(d)
# Collect the deferred to deferred_ids
d = gatherBoth(d_list,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
# Now d has a list of deferred_ids or Failures coming
d.addCallback(error.collect_exceptions, 'scatter')
def process_did_list(did_list):
"""Turn a list of deferred_ids into a final result or failure."""
new_d_list = [
self.get_pending_deferred(did, True) for did in did_list
]
final_d = gatherBoth(new_d_list,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
final_d.addCallback(error.collect_exceptions, 'gather')
final_d.addCallback(lambda lop: [i[0] for i in lop])
final_d.addCallback(mapObject.joinPartitions)
return final_d
# Now, depending on block, we need to handle the list deferred_ids
# coming down the pipe diferently.
if block:
# If we are blocking register a callback that will transform the
# list of deferred_ids into the final result.
d.addCallback(process_did_list)
return d
else:
# Here we are going to use a _local_ PendingDeferredManager.
deferred_id = self.pdm.get_deferred_id()
# This is the deferred we will return to the user that will fire
# with the local deferred_id AFTER we have received the list of
# primary deferred_ids
d_to_return = defer.Deferred()
def do_it(did_list):
"""Produce a deferred to the final result, but first fire the
deferred we will return to the user that has the local
deferred id."""
d_to_return.callback(deferred_id)
return process_did_list(did_list)
d.addCallback(do_it)
# Now save the deferred to the final result
self.pdm.save_pending_deferred(d, deferred_id)
return d_to_return
def interrupt_then_kill(self, delay=1.0):
dlist = []
for el in self.launchers:
d = el.get_stop_deferred()
dlist.append(d)
el.interrupt_then_kill(delay)
dfinal = gatherBoth(dlist, consumeErrors=True)
dfinal.addCallback(self._handle_stop)
return dfinal
def signal(self, sig):
dlist = []
for el in self.launchers:
d = el.get_stop_deferred()
dlist.append(d)
el.signal(sig)
dfinal = gatherBoth(dlist, consumeErrors=True)
dfinal.addCallback(self._handle_stop)
return dfinal
def stop(self):
"""Stop the IPython cluster if it is running."""
if self.state=='running':
d1 = self.launcher.observe_stop()
d1.addCallback(self._handle_stop)
d2 = self.launcher.stop()
return gatherBoth([d1, d2], consumeErrors=True)
else:
raise ClusterStateError("Cluster not running")
def get_results(task_ids):
d = self.task_controller.barrier(task_ids)
d.addCallback(lambda _: gatherBoth([
self.task_controller.get_task_result(tid)
for tid in task_ids
],
consumeErrors=1))
d.addCallback(collect_exceptions, 'map')
return d
def signal(self, sig):
dlist = []
for el in self.launchers:
d = el.get_stop_deferred()
dlist.append(d)
el.signal(sig)
dfinal = gatherBoth(dlist, consumeErrors=True)
dfinal.addCallback(self._handle_stop)
return dfinal
def stop(self):
"""Stop the IPython cluster if it is running."""
if self.state=='running':
d1 = self.launcher.observe_stop()
d1.addCallback(self._handle_stop)
d2 = self.launcher.stop()
return gatherBoth([d1, d2], consumeErrors=True)
else:
raise ClusterStateError("Cluster not running")
def interrupt_then_kill(self, delay=1.0):
dlist = []
for el in self.launchers:
d = el.get_stop_deferred()
dlist.append(d)
el.interrupt_then_kill(delay)
dfinal = gatherBoth(dlist, consumeErrors=True)
dfinal.addCallback(self._handle_stop)
return dfinal
def start(self, n):
dlist = []
for i in range(n):
el = EngineLauncher(extra_args=self.extra_args)
d = el.start()
self.launchers.append(el)
dlist.append(d)
dfinal = gatherBoth(dlist, consumeErrors=True)
dfinal.addCallback(self._handle_start)
return dfinal
def process_did_list(did_list):
"""Turn a list of deferred_ids into a final result or failure."""
new_d_list = [self.get_pending_deferred(did, True) for did in did_list]
final_d = gatherBoth(new_d_list,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
final_d.addCallback(error.collect_exceptions, 'scatter')
final_d.addCallback(lambda lop: [i[0] for i in lop])
return final_d
def start(self, n):
dlist = []
for i in range(n):
el = EngineLauncher(extra_args=self.extra_args)
d = el.start()
self.launchers.append(el)
dlist.append(d)
dfinal = gatherBoth(dlist, consumeErrors=True)
dfinal.addCallback(self._handle_start)
return dfinal
def process_did_list(did_list):
"""Turn a list of deferred_ids into a final result or failure."""
new_d_list = [self.get_pending_deferred(did, True) for did in did_list]
final_d = gatherBoth(new_d_list,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
final_d.addCallback(error.collect_exceptions, 'scatter')
final_d.addCallback(lambda lop: [i[0] for i in lop])
return final_d
def do_scatter(engines):
nEngines = len(engines)
mapClass = Map.dists[dist]
mapObject = mapClass()
d_list = []
# Loop through and push to each engine in non-blocking mode.
# This returns a set of deferreds to deferred_ids
for index, engineid in enumerate(engines):
partition = mapObject.getPartition(seq, index, nEngines)
if flatten and len(partition) == 1:
d = self.push({key: partition[0]}, targets=engineid, block=False)
else:
d = self.push({key: partition}, targets=engineid, block=False)
d_list.append(d)
# Collect the deferred to deferred_ids
d = gatherBoth(d_list,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
# Now d has a list of deferred_ids or Failures coming
d.addCallback(error.collect_exceptions, 'scatter')
def process_did_list(did_list):
"""Turn a list of deferred_ids into a final result or failure."""
new_d_list = [self.get_pending_deferred(did, True) for did in did_list]
final_d = gatherBoth(new_d_list,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
final_d.addCallback(error.collect_exceptions, 'scatter')
final_d.addCallback(lambda lop: [i[0] for i in lop])
return final_d
# Now, depending on block, we need to handle the list deferred_ids
# coming down the pipe diferently.
if block:
# If we are blocking register a callback that will transform the
# list of deferred_ids into the final result.
d.addCallback(process_did_list)
return d
else:
# Here we are going to use a _local_ PendingDeferredManager.
deferred_id = self.pdm.get_deferred_id()
# This is the deferred we will return to the user that will fire
# with the local deferred_id AFTER we have received the list of
# primary deferred_ids
d_to_return = defer.Deferred()
def do_it(did_list):
"""Produce a deferred to the final result, but first fire the
deferred we will return to the user that has the local
deferred id."""
d_to_return.callback(deferred_id)
return process_did_list(did_list)
d.addCallback(do_it)
# Now save the deferred to the final result
self.pdm.save_pending_deferred(d, deferred_id)
return d_to_return
def _ssh_engine(self, hostname, count):
exec_engine = "ssh %s sh %s/%s-sshx.sh %s" % (
hostname, self.temp_dir, os.environ['USER'], self.engine_command)
cmds = exec_engine.split()
dlist = []
log.msg("about to start engines...")
for i in range(count):
log.msg('Starting engines: %s' % exec_engine)
d = getProcessOutput(cmds[0], cmds[1:], env=os.environ)
dlist.append(d)
return gatherBoth(dlist, consumeErrors=True)
def _ssh_engine(self, hostname, count):
exec_engine = "ssh %s sh %s/%s-sshx.sh %s" % (
hostname, self.temp_dir,
os.environ['USER'], self.engine_command
)
cmds = exec_engine.split()
dlist = []
log.msg("about to start engines...")
for i in range(count):
log.msg('Starting engines: %s' % exec_engine)
d = getProcessOutput(cmds[0], cmds[1:], env=os.environ)
dlist.append(d)
return gatherBoth(dlist, consumeErrors=True)
def pull_serialized(self, keys, targets='all'):
try:
dList = self._performOnEngines('pull_serialized', keys, targets=targets)
except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
for d in dList:
d.addCallback(self._logSizes)
d = gatherBoth(dList,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
d.addCallback(error.collect_exceptions, 'pull_serialized')
return d
def clear_properties(self, targets='all'):
log.msg("Clearing properties on %r" % targets)
try:
engines = self.engineList(targets)
except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
dList = [e.clear_properties() for e in engines]
d = gatherBoth(dList,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
d.addCallback(error.collect_exceptions, 'clear_properties')
return d
def clear_properties(self, targets='all'):
log.msg("Clearing properties on %r" % targets)
try:
engines = self.engineList(targets)
except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
dList = [e.clear_properties() for e in engines]
d = gatherBoth(dList,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
d.addCallback(error.collect_exceptions, 'clear_properties')
return d
def pull_serialized(self, keys, targets='all'):
try:
dList = self._performOnEngines('pull_serialized', keys, targets=targets)
except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
for d in dList:
d.addCallback(self._logSizes)
d = gatherBoth(dList,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
d.addCallback(error.collect_exceptions, 'pull_serialized')
return d
def queue_status(self, targets='all'):
log.msg("Getting queue status on %r" % targets)
try:
engines = self.engineList(targets)
except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
dList = []
for e in engines:
dList.append(e.queue_status().addCallback(lambda s:(e.id, s)))
d = gatherBoth(dList,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
d.addCallback(error.collect_exceptions, 'queue_status')
return d
def queue_status(self, targets='all'):
log.msg("Getting queue status on %r" % targets)
try:
engines = self.engineList(targets)
except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
dList = []
for e in engines:
dList.append(e.queue_status().addCallback(lambda s:(e.id, s)))
d = gatherBoth(dList,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
d.addCallback(error.collect_exceptions, 'queue_status')
return d
def start(self, n, cluster_dir):
"""Start n engines by profile or cluster_dir."""
self.cluster_dir = str(cluster_dir)
dlist = []
for i in range(n):
el = LocalEngineLauncher(work_dir=self.work_dir, config=self.config)
# Copy the engine args over to each engine launcher.
import copy
el.engine_args = copy.deepcopy(self.engine_args)
d = el.start(cluster_dir)
if i==0:
log.msg("Starting LocalEngineSetLauncher: %r" % el.args)
self.launchers.append(el)
dlist.append(d)
# The consumeErrors here could be dangerous
dfinal = gatherBoth(dlist, consumeErrors=True)
dfinal.addCallback(self.notify_start)
return dfinal
def start(self, n, cluster_dir):
"""Start n engines by profile or cluster_dir."""
self.cluster_dir = unicode(cluster_dir)
dlist = []
for i in range(n):
el = LocalEngineLauncher(work_dir=self.work_dir,
config=self.config)
# Copy the engine args over to each engine launcher.
import copy
el.engine_args = copy.deepcopy(self.engine_args)
d = el.start(cluster_dir)
if i == 0:
log.msg("Starting LocalEngineSetLauncher: %r" % el.args)
self.launchers.append(el)
dlist.append(d)
# The consumeErrors here could be dangerous
dfinal = gatherBoth(dlist, consumeErrors=True)
dfinal.addCallback(self.notify_start)
return dfinal
def _performOnEnginesAndGatherBoth(self, methodName, *args, **kwargs):
"""Called _performOnEngines and wraps result/exception into deferred."""
try:
dList = self._performOnEngines(methodName, *args, **kwargs)
except (error.InvalidEngineID, AttributeError, KeyError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
# Having fireOnOneErrback is causing problems with the determinacy
# of the system. Basically, once a single engine has errbacked, this
# method returns. In some cases, this will cause client to submit
# another command. Because the previous command is still running
# on some engines, this command will be queued. When those commands
# then errback, the second command will raise QueueCleared. Ahhh!
d = gatherBoth(dList,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
d.addCallback(error.collect_exceptions, methodName)
return d
def _performOnEnginesAndGatherBoth(self, methodName, *args, **kwargs):
"""Called _performOnEngines and wraps result/exception into deferred."""
try:
dList = self._performOnEngines(methodName, *args, **kwargs)
except (error.InvalidEngineID, AttributeError, KeyError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
# Having fireOnOneErrback is causing problems with the determinacy
# of the system. Basically, once a single engine has errbacked, this
# method returns. In some cases, this will cause client to submit
# another command. Because the previous command is still running
# on some engines, this command will be queued. When those commands
# then errback, the second command will raise QueueCleared. Ahhh!
d = gatherBoth(dList,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
d.addCallback(error.collect_exceptions, methodName)
return d
def map(self, func, *sequences):
"""
Apply func to *sequences elementwise. Like Python's builtin map.
This version is load balanced.
"""
max_len = max(len(s) for s in sequences)
for s in sequences:
if len(s) != max_len:
raise ValueError('all sequences must have equal length')
task_args = zip(*sequences)
task_ids = []
dlist = []
for ta in task_args:
task = MapTask(func,
ta,
clear_before=self.clear_before,
clear_after=self.clear_after,
retries=self.retries,
recovery_task=self.recovery_task,
depend=self.depend)
dlist.append(self.task_controller.run(task))
dlist = gatherBoth(dlist, consumeErrors=1)
dlist.addCallback(collect_exceptions, 'map')
if self.block:
def get_results(task_ids):
d = self.task_controller.barrier(task_ids)
d.addCallback(lambda _: gatherBoth([
self.task_controller.get_task_result(tid)
for tid in task_ids
],
consumeErrors=1))
d.addCallback(collect_exceptions, 'map')
return d
dlist.addCallback(get_results)
return dlist
def observe_stop(self):
dlist = [el.observe_stop() for el in self.launchers]
dfinal = gatherBoth(dlist, consumeErrors=False)
dfinal.addCallback(self.notify_stop)
return dfinal
def get_results(task_ids):
d = self.task_controller.barrier(task_ids)
d.addCallback(lambda _: gatherBoth([self.task_controller.get_task_result(tid) for tid in task_ids], consumeErrors=1))
d.addCallback(collect_exceptions, 'map')
return d
def kill(self):
dlist = []
for host in self.engine_hosts.keys():
d = self._killall(host)
dlist.append(d)
return gatherBoth(dlist, consumeErrors=True)
def kill(self):
dlist = []
for host in self.engine_hosts.keys():
d = self._killall(host)
dlist.append(d)
return gatherBoth(dlist, consumeErrors=True)
def observe_stop(self):
dlist = [el.observe_stop() for el in self.launchers]
dfinal = gatherBoth(dlist, consumeErrors=False)
dfinal.addCallback(self.notify_stop)
return dfinal
