def create_cluster(self,
function,
deps=(),
callback=None,
progress_bar=None,
verbose=False,
modules=(),
on_nodes_changed=None,
node_setup=None,
node_cleanup=None):
self.jobs = []
self.callback = callback
self.function = function
self.verbose = verbose
self.close = True
self.on_nodes_changed = on_nodes_changed
self.node_cleanup = node_cleanup
if not self.run_parallel:
if node_setup:
node_setup()
_thread.start_new_thread(self._job_runner_thread, ())
else:
# DDG: NodeAllocate is used to pass the arguments to setup during node initialization
self.cluster = dispy.JobCluster(
function,
[
dispy.NodeAllocate(node.ip_addr,
setup_args=() if node_setup else
(modules, )) for node in self.nodes
],
list(deps),
callback,
self.cluster_status,
pulse_interval=60,
# Note, exceptions in setup seems to be swallowed up and
# never shown.
setup=node_setup or setup,
cleanup=node_cleanup or True,
loglevel=dispy.logger.CRITICAL,
# if communication is lost to a node, his jobs will be
# automatically rescheduled to another one.. (but
# jobs must be reentrant! Because if rescheduled and the
# disconnected node is really alive (temporal netsplit) the
# job will run multiple times and maybe in parallel)
reentrant=True,
ip_addr=self.ip_address)
self.http_server = dispy.httpd.DispyHTTPServer(self.cluster,
poll_sec=2)
# wait for all nodes to be created
print(" >> Waiting %d seconds to initialize all nodes... " % DELAY)
time.sleep(DELAY)
self.progress_bar = progress_bar
def create_cluster(self, function, deps=(), callback=None, progress_bar=None, verbose=False, modules=()):
self.jobs = []
self.callback = callback
self.function = function
self.verbose = verbose
self.close = True
if self.run_parallel:
# DDG: NodeAllocate is used to pass the arguments to setup during node initialization
self.cluster = dispy.JobCluster(function, [dispy.NodeAllocate(node.ip_addr, setup_args=(modules,))
for node in self.nodes], list(deps),
callback, self.cluster_status, pulse_interval=60, setup=setup,
loglevel=dispy.logger.CRITICAL, reentrant=True, ip_addr=self.ip_address)
self.http_server = dispy.httpd.DispyHTTPServer(self.cluster, poll_sec=2)
# wait for all nodes to be created
time.sleep(DELAY)
self.progress_bar = progress_bar