def start(self, n=2): """Start the IPython cluster with n engines. Parameters ---------- n : int The number of engine to start. """ # We might want to add logic to test if the cluster has started # by another process.... if not self.state=='running': self.launcher = IPClusterLauncher(os.getcwd()) self.launcher.ipcluster_n = n self.launcher.ipcluster_subcommand = 'start' d = self.launcher.start() d.addCallback(self._handle_start) return d else: raise ClusterStateError('Cluster is already running')
class AsyncCluster(object): """An class that wraps the :command:`ipcluster` script.""" def __init__(self, profile='default', cluster_dir=None, ipython_dir=None, auto_create=False, auto_stop=True): """Create a class to manage an IPython cluster. This class calls the :command:`ipcluster` command with the right options to start an IPython cluster. Typically a cluster directory must be created (:command:`ipcluster create`) and configured before using this class. Configuration is done by editing the configuration files in the top level of the cluster directory. Parameters ---------- profile : str The name of a cluster directory profile (default="default"). The cluster directory "cluster_<profile>" will be searched for in ``os.getcwd()``, the ipython_dir and then in the directories listed in the :env:`IPCLUSTER_DIR_PATH` environment variable. cluster_dir : str The full path to a cluster directory. This is useful if profiles are not being used. ipython_dir : str The location of the ipython_dir if different from the default. This is used if the cluster directory is being found by profile. auto_create : bool Automatically create the cluster directory it is dones't exist. This will usually only make sense if using a local cluster (default=False). auto_stop : bool Automatically stop the cluster when this instance is garbage collected (default=True). This is useful if you want the cluster to live beyond your current process. There is also an instance attribute ``auto_stop`` to change this behavior. """ self._setup_cluster_dir(profile, cluster_dir, ipython_dir, auto_create) self.state = 'before' self.launcher = None self.client_connector = None self.auto_stop = auto_stop def __del__(self): if self.auto_stop and self.state=='running': print("Auto stopping the cluster...") self.stop() @property def location(self): if hasattr(self, 'cluster_dir_obj'): return self.cluster_dir_obj.location else: return '' @property def running(self): if self.state=='running': return True else: return False def _setup_cluster_dir(self, profile, cluster_dir, ipython_dir, auto_create): if ipython_dir is None: ipython_dir = get_ipython_dir() if cluster_dir is not None: try: self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir) except ClusterDirError: pass if profile is not None: try: self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile( ipython_dir, profile) except ClusterDirError: pass if auto_create or profile=='default': # This should call 'ipcluster create --profile default self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile( ipython_dir, profile) else: raise ClusterDirError('Cluster dir not found.') @make_deferred def start(self, n=2): """Start the IPython cluster with n engines. Parameters ---------- n : int The number of engine to start. """ # We might want to add logic to test if the cluster has started # by another process.... if not self.state=='running': self.launcher = IPClusterLauncher(os.getcwd()) self.launcher.ipcluster_n = n self.launcher.ipcluster_subcommand = 'start' d = self.launcher.start() d.addCallback(self._handle_start) return d else: raise ClusterStateError('Cluster is already running') @make_deferred def stop(self): """Stop the IPython cluster if it is running.""" if self.state=='running': d1 = self.launcher.observe_stop() d1.addCallback(self._handle_stop) d2 = self.launcher.stop() return gatherBoth([d1, d2], consumeErrors=True) else: raise ClusterStateError("Cluster not running") def get_multiengine_client(self, delay=DELAY, max_tries=MAX_TRIES): """Get the multiengine client for the running cluster. If this fails, it means that the cluster has not finished starting. Usually waiting a few seconds are re-trying will solve this. """ if self.client_connector is None: self.client_connector = AsyncClientConnector() return self.client_connector.get_multiengine_client( cluster_dir=self.cluster_dir_obj.location, delay=delay, max_tries=max_tries ) def get_task_client(self, delay=DELAY, max_tries=MAX_TRIES): """Get the task client for the running cluster. If this fails, it means that the cluster has not finished starting. Usually waiting a few seconds are re-trying will solve this. """ if self.client_connector is None: self.client_connector = AsyncClientConnector() return self.client_connector.get_task_client( cluster_dir=self.cluster_dir_obj.location, delay=delay, max_tries=max_tries ) def get_ipengine_logs(self): return self.get_logs_by_name('ipengine') def get_ipcontroller_logs(self): return self.get_logs_by_name('ipcontroller') def get_ipcluster_logs(self): return self.get_logs_by_name('ipcluster') def get_logs_by_name(self, name='ipcluster'): log_dir = self.cluster_dir_obj.log_dir logs = {} for log in os.listdir(log_dir): if log.startswith(name + '-') and log.endswith('.log'): with open(os.path.join(log_dir, log), 'r') as f: logs[log] = f.read() return logs def get_logs(self): d = self.get_ipcluster_logs() d.update(self.get_ipengine_logs()) d.update(self.get_ipcontroller_logs()) return d def _handle_start(self, r): self.state = 'running' def _handle_stop(self, r): self.state = 'after'
class AsyncCluster(object): """An class that wraps the :command:`ipcluster` script.""" def __init__(self, profile='default', cluster_dir=None, ipython_dir=None, auto_create=False, auto_stop=True): """Create a class to manage an IPython cluster. This class calls the :command:`ipcluster` command with the right options to start an IPython cluster. Typically a cluster directory must be created (:command:`ipcluster create`) and configured before using this class. Configuration is done by editing the configuration files in the top level of the cluster directory. Parameters ---------- profile : str The name of a cluster directory profile (default="default"). The cluster directory "cluster_<profile>" will be searched for in ``os.getcwd()``, the ipython_dir and then in the directories listed in the :env:`IPCLUSTER_DIR_PATH` environment variable. cluster_dir : str The full path to a cluster directory. This is useful if profiles are not being used. ipython_dir : str The location of the ipython_dir if different from the default. This is used if the cluster directory is being found by profile. auto_create : bool Automatically create the cluster directory it is dones't exist. This will usually only make sense if using a local cluster (default=False). auto_stop : bool Automatically stop the cluster when this instance is garbage collected (default=True). This is useful if you want the cluster to live beyond your current process. There is also an instance attribute ``auto_stop`` to change this behavior. """ self._setup_cluster_dir(profile, cluster_dir, ipython_dir, auto_create) self.state = 'before' self.launcher = None self.client_connector = None self.auto_stop = auto_stop def __del__(self): if self.auto_stop and self.state=='running': print "Auto stopping the cluster..." self.stop() @property def location(self): if hasattr(self, 'cluster_dir_obj'): return self.cluster_dir_obj.location else: return '' @property def running(self): if self.state=='running': return True else: return False def _setup_cluster_dir(self, profile, cluster_dir, ipython_dir, auto_create): if ipython_dir is None: ipython_dir = get_ipython_dir() if cluster_dir is not None: try: self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir) except ClusterDirError: pass if profile is not None: try: self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile( ipython_dir, profile) except ClusterDirError: pass if auto_create or profile=='default': # This should call 'ipcluster create --profile default self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile( ipython_dir, profile) else: raise ClusterDirError('Cluster dir not found.') @make_deferred def start(self, n=2): """Start the IPython cluster with n engines. Parameters ---------- n : int The number of engine to start. """ # We might want to add logic to test if the cluster has started # by another process.... if not self.state=='running': self.launcher = IPClusterLauncher(os.getcwd()) self.launcher.ipcluster_n = n self.launcher.ipcluster_subcommand = 'start' d = self.launcher.start() d.addCallback(self._handle_start) return d else: raise ClusterStateError('Cluster is already running') @make_deferred def stop(self): """Stop the IPython cluster if it is running.""" if self.state=='running': d1 = self.launcher.observe_stop() d1.addCallback(self._handle_stop) d2 = self.launcher.stop() return gatherBoth([d1, d2], consumeErrors=True) else: raise ClusterStateError("Cluster not running") def get_multiengine_client(self, delay=DELAY, max_tries=MAX_TRIES): """Get the multiengine client for the running cluster. If this fails, it means that the cluster has not finished starting. Usually waiting a few seconds are re-trying will solve this. """ if self.client_connector is None: self.client_connector = AsyncClientConnector() return self.client_connector.get_multiengine_client( cluster_dir=self.cluster_dir_obj.location, delay=delay, max_tries=max_tries ) def get_task_client(self, delay=DELAY, max_tries=MAX_TRIES): """Get the task client for the running cluster. If this fails, it means that the cluster has not finished starting. Usually waiting a few seconds are re-trying will solve this. """ if self.client_connector is None: self.client_connector = AsyncClientConnector() return self.client_connector.get_task_client( cluster_dir=self.cluster_dir_obj.location, delay=delay, max_tries=max_tries ) def get_ipengine_logs(self): return self.get_logs_by_name('ipengine') def get_ipcontroller_logs(self): return self.get_logs_by_name('ipcontroller') def get_ipcluster_logs(self): return self.get_logs_by_name('ipcluster') def get_logs_by_name(self, name='ipcluster'): log_dir = self.cluster_dir_obj.log_dir logs = {} for log in os.listdir(log_dir): if log.startswith(name + '-') and log.endswith('.log'): with open(os.path.join(log_dir, log), 'r') as f: logs[log] = f.read() return logs def get_logs(self): d = self.get_ipcluster_logs() d.update(self.get_ipengine_logs()) d.update(self.get_ipcontroller_logs()) return d def _handle_start(self, r): self.state = 'running' def _handle_stop(self, r): self.state = 'after'