def start(self, n=2):
"""Start the IPython cluster with n engines.
Parameters
----------
n : int
The number of engine to start.
"""
# We might want to add logic to test if the cluster has started
# by another process....
if not self.state=='running':
self.launcher = IPClusterLauncher(os.getcwd())
self.launcher.ipcluster_n = n
self.launcher.ipcluster_subcommand = 'start'
d = self.launcher.start()
d.addCallback(self._handle_start)
return d
else:
raise ClusterStateError('Cluster is already running')
def start(self, n=2):
"""Start the IPython cluster with n engines.
Parameters
----------
n : int
The number of engine to start.
"""
# We might want to add logic to test if the cluster has started
# by another process....
if not self.state=='running':
self.launcher = IPClusterLauncher(os.getcwd())
self.launcher.ipcluster_n = n
self.launcher.ipcluster_subcommand = 'start'
d = self.launcher.start()
d.addCallback(self._handle_start)
return d
else:
raise ClusterStateError('Cluster is already running')
class AsyncCluster(object):
"""An class that wraps the :command:`ipcluster` script."""
def __init__(self, profile='default', cluster_dir=None, ipython_dir=None,
auto_create=False, auto_stop=True):
"""Create a class to manage an IPython cluster.
This class calls the :command:`ipcluster` command with the right
options to start an IPython cluster. Typically a cluster directory
must be created (:command:`ipcluster create`) and configured before
using this class. Configuration is done by editing the
configuration files in the top level of the cluster directory.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
auto_create : bool
Automatically create the cluster directory it is dones't exist.
This will usually only make sense if using a local cluster
(default=False).
auto_stop : bool
Automatically stop the cluster when this instance is garbage
collected (default=True). This is useful if you want the cluster
to live beyond your current process. There is also an instance
attribute ``auto_stop`` to change this behavior.
"""
self._setup_cluster_dir(profile, cluster_dir, ipython_dir, auto_create)
self.state = 'before'
self.launcher = None
self.client_connector = None
self.auto_stop = auto_stop
def __del__(self):
if self.auto_stop and self.state=='running':
print("Auto stopping the cluster...")
self.stop()
@property
def location(self):
if hasattr(self, 'cluster_dir_obj'):
return self.cluster_dir_obj.location
else:
return ''
@property
def running(self):
if self.state=='running':
return True
else:
return False
def _setup_cluster_dir(self, profile, cluster_dir, ipython_dir, auto_create):
if ipython_dir is None:
ipython_dir = get_ipython_dir()
if cluster_dir is not None:
try:
self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
except ClusterDirError:
pass
if profile is not None:
try:
self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
ipython_dir, profile)
except ClusterDirError:
pass
if auto_create or profile=='default':
# This should call 'ipcluster create --profile default
self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
ipython_dir, profile)
else:
raise ClusterDirError('Cluster dir not found.')
@make_deferred
def start(self, n=2):
"""Start the IPython cluster with n engines.
Parameters
----------
n : int
The number of engine to start.
"""
# We might want to add logic to test if the cluster has started
# by another process....
if not self.state=='running':
self.launcher = IPClusterLauncher(os.getcwd())
self.launcher.ipcluster_n = n
self.launcher.ipcluster_subcommand = 'start'
d = self.launcher.start()
d.addCallback(self._handle_start)
return d
else:
raise ClusterStateError('Cluster is already running')
@make_deferred
def stop(self):
"""Stop the IPython cluster if it is running."""
if self.state=='running':
d1 = self.launcher.observe_stop()
d1.addCallback(self._handle_stop)
d2 = self.launcher.stop()
return gatherBoth([d1, d2], consumeErrors=True)
else:
raise ClusterStateError("Cluster not running")
def get_multiengine_client(self, delay=DELAY, max_tries=MAX_TRIES):
"""Get the multiengine client for the running cluster.
If this fails, it means that the cluster has not finished starting.
Usually waiting a few seconds are re-trying will solve this.
"""
if self.client_connector is None:
self.client_connector = AsyncClientConnector()
return self.client_connector.get_multiengine_client(
cluster_dir=self.cluster_dir_obj.location,
delay=delay, max_tries=max_tries
)
def get_task_client(self, delay=DELAY, max_tries=MAX_TRIES):
"""Get the task client for the running cluster.
If this fails, it means that the cluster has not finished starting.
Usually waiting a few seconds are re-trying will solve this.
"""
if self.client_connector is None:
self.client_connector = AsyncClientConnector()
return self.client_connector.get_task_client(
cluster_dir=self.cluster_dir_obj.location,
delay=delay, max_tries=max_tries
)
def get_ipengine_logs(self):
return self.get_logs_by_name('ipengine')
def get_ipcontroller_logs(self):
return self.get_logs_by_name('ipcontroller')
def get_ipcluster_logs(self):
return self.get_logs_by_name('ipcluster')
def get_logs_by_name(self, name='ipcluster'):
log_dir = self.cluster_dir_obj.log_dir
logs = {}
for log in os.listdir(log_dir):
if log.startswith(name + '-') and log.endswith('.log'):
with open(os.path.join(log_dir, log), 'r') as f:
logs[log] = f.read()
return logs
def get_logs(self):
d = self.get_ipcluster_logs()
d.update(self.get_ipengine_logs())
d.update(self.get_ipcontroller_logs())
return d
def _handle_start(self, r):
self.state = 'running'
def _handle_stop(self, r):
self.state = 'after'
class AsyncCluster(object):
"""An class that wraps the :command:`ipcluster` script."""
def __init__(self, profile='default', cluster_dir=None, ipython_dir=None,
auto_create=False, auto_stop=True):
"""Create a class to manage an IPython cluster.
This class calls the :command:`ipcluster` command with the right
options to start an IPython cluster. Typically a cluster directory
must be created (:command:`ipcluster create`) and configured before
using this class. Configuration is done by editing the
configuration files in the top level of the cluster directory.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
auto_create : bool
Automatically create the cluster directory it is dones't exist.
This will usually only make sense if using a local cluster
(default=False).
auto_stop : bool
Automatically stop the cluster when this instance is garbage
collected (default=True). This is useful if you want the cluster
to live beyond your current process. There is also an instance
attribute ``auto_stop`` to change this behavior.
"""
self._setup_cluster_dir(profile, cluster_dir, ipython_dir, auto_create)
self.state = 'before'
self.launcher = None
self.client_connector = None
self.auto_stop = auto_stop
def __del__(self):
if self.auto_stop and self.state=='running':
print "Auto stopping the cluster..."
self.stop()
@property
def location(self):
if hasattr(self, 'cluster_dir_obj'):
return self.cluster_dir_obj.location
else:
return ''
@property
def running(self):
if self.state=='running':
return True
else:
return False
def _setup_cluster_dir(self, profile, cluster_dir, ipython_dir, auto_create):
if ipython_dir is None:
ipython_dir = get_ipython_dir()
if cluster_dir is not None:
try:
self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
except ClusterDirError:
pass
if profile is not None:
try:
self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
ipython_dir, profile)
except ClusterDirError:
pass
if auto_create or profile=='default':
# This should call 'ipcluster create --profile default
self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
ipython_dir, profile)
else:
raise ClusterDirError('Cluster dir not found.')
@make_deferred
def start(self, n=2):
"""Start the IPython cluster with n engines.
Parameters
----------
n : int
The number of engine to start.
"""
# We might want to add logic to test if the cluster has started
# by another process....
if not self.state=='running':
self.launcher = IPClusterLauncher(os.getcwd())
self.launcher.ipcluster_n = n
self.launcher.ipcluster_subcommand = 'start'
d = self.launcher.start()
d.addCallback(self._handle_start)
return d
else:
raise ClusterStateError('Cluster is already running')
@make_deferred
def stop(self):
"""Stop the IPython cluster if it is running."""
if self.state=='running':
d1 = self.launcher.observe_stop()
d1.addCallback(self._handle_stop)
d2 = self.launcher.stop()
return gatherBoth([d1, d2], consumeErrors=True)
else:
raise ClusterStateError("Cluster not running")
def get_multiengine_client(self, delay=DELAY, max_tries=MAX_TRIES):
"""Get the multiengine client for the running cluster.
If this fails, it means that the cluster has not finished starting.
Usually waiting a few seconds are re-trying will solve this.
"""
if self.client_connector is None:
self.client_connector = AsyncClientConnector()
return self.client_connector.get_multiengine_client(
cluster_dir=self.cluster_dir_obj.location,
delay=delay, max_tries=max_tries
)
def get_task_client(self, delay=DELAY, max_tries=MAX_TRIES):
"""Get the task client for the running cluster.
If this fails, it means that the cluster has not finished starting.
Usually waiting a few seconds are re-trying will solve this.
"""
if self.client_connector is None:
self.client_connector = AsyncClientConnector()
return self.client_connector.get_task_client(
cluster_dir=self.cluster_dir_obj.location,
delay=delay, max_tries=max_tries
)
def get_ipengine_logs(self):
return self.get_logs_by_name('ipengine')
def get_ipcontroller_logs(self):
return self.get_logs_by_name('ipcontroller')
def get_ipcluster_logs(self):
return self.get_logs_by_name('ipcluster')
def get_logs_by_name(self, name='ipcluster'):
log_dir = self.cluster_dir_obj.log_dir
logs = {}
for log in os.listdir(log_dir):
if log.startswith(name + '-') and log.endswith('.log'):
with open(os.path.join(log_dir, log), 'r') as f:
logs[log] = f.read()
return logs
def get_logs(self):
d = self.get_ipcluster_logs()
d.update(self.get_ipengine_logs())
d.update(self.get_ipcontroller_logs())
return d
def _handle_start(self, r):
self.state = 'running'
def _handle_stop(self, r):
self.state = 'after'
