def get_result(self, i=None, targets=None, block=None):
"""
Get a previous result.
When code is executed in an engine, a dict is created and returned. This
method retrieves that dict for previous commands.
:Parameters:
i : int
The number of the result to get
targets : id or list of ids
The engine to use for the execution
block : boolean
If False, this method will return the actual result. If False,
a `PendingResult` is returned which can be used to get the result
at a later time.
"""
targets, block = self._findTargetsAndBlock(targets, block)
result = blockingCallFromThread(self.smultiengine.get_result, i, targets=targets, block=block)
if block:
result = ResultList(result)
else:
result = PendingResult(self, result)
result.add_callback(wrapResultList)
return result
def get_result(self, i=None, targets=None, block=None):
"""
Get a previous result.
When code is executed in an engine, a dict is created and returned. This
method retrieves that dict for previous commands.
:Parameters:
i : int
The number of the result to get
targets : id or list of ids
The engine to use for the execution
block : boolean
If False, this method will return the actual result. If False,
a `PendingResult` is returned which can be used to get the result
at a later time.
"""
targets, block = self._findTargetsAndBlock(targets, block)
result = blockingCallFromThread(self.smultiengine.get_result,
i,
targets=targets,
block=block)
if block:
result = ResultList(result)
else:
result = PendingResult(self, result)
result.add_callback(wrapResultList)
return result
def execute(self, lines, targets=None, block=None):
"""
Execute code on a set of engines.
:Parameters:
lines : str
The Python code to execute as a string
targets : id or list of ids
The engine to use for the execution
block : boolean
If False, this method will return the actual result. If False,
a `PendingResult` is returned which can be used to get the result
at a later time.
"""
targets, block = self._findTargetsAndBlock(targets, block)
result = blockingCallFromThread(self.smultiengine.execute,
lines,
targets=targets,
block=block)
if block:
result = ResultList(result)
else:
result = PendingResult(self, result)
result.add_callback(wrapResultList)
return result
def spin(self):
"""
Touch the scheduler, to resume scheduling without submitting a task.
This method only needs to be called in unusual situations where the
scheduler is idle for some reason.
"""
return blockingCallFromThread(self.task_controller.spin)
def barrier(self, taskids):
"""Block until a set of tasks are completed.
:Parameters:
taskids : list, tuple
A sequence of taskids to block on.
"""
return blockingCallFromThread(self.task_controller.barrier, taskids)
def _blockFromThread(self, function, *args, **kwargs):
block = kwargs.get('block', None)
if block is None:
raise error.MissingBlockArgument("'block' keyword argument is missing")
result = blockingCallFromThread(function, *args, **kwargs)
if not block:
result = PendingResult(self, result)
return result
def spin(self):
"""
Touch the scheduler, to resume scheduling without submitting a task.
This method only needs to be called in unusual situations where the
scheduler is idle for some reason.
"""
return blockingCallFromThread(self.task_controller.spin)
def barrier(self, taskids):
"""Block until a set of tasks are completed.
:Parameters:
taskids : list, tuple
A sequence of taskids to block on.
"""
return blockingCallFromThread(self.task_controller.barrier, taskids)
def start(self, n=2):
"""Start the IPython cluster with n engines.
Parameters
----------
n : int
The number of engine to start.
"""
return blockingCallFromThread(self.async_cluster.start, n)
def get_client(self, profile='default', cluster_dir=None,
furl_or_file=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
client = blockingCallFromThread(
self.async_cc.get_client, profile, cluster_dir,
furl_or_file, ipython_dir,
delay, max_tries
)
return client.adapt_to_blocking_client()
def start(self, n=2):
"""Start the IPython cluster with n engines.
Parameters
----------
n : int
The number of engine to start.
"""
return blockingCallFromThread(self.async_cluster.start, n)
def abort(self, taskid):
"""
Abort a task by taskid.
:Parameters:
taskid : int
The taskid of the task to be aborted.
"""
return blockingCallFromThread(self.task_controller.abort, taskid)
def abort(self, taskid):
"""
Abort a task by taskid.
:Parameters:
taskid : int
The taskid of the task to be aborted.
"""
return blockingCallFromThread(self.task_controller.abort, taskid)
def get_client(self, profile='default', cluster_dir=None,
furl_or_file=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
client = blockingCallFromThread(
self.async_cc.get_client, profile, cluster_dir,
furl_or_file, ipython_dir,
delay, max_tries
)
return client.adapt_to_blocking_client()
def _blockFromThread(self, function, *args, **kwargs):
block = kwargs.get('block', None)
if block is None:
raise error.MissingBlockArgument(
"'block' keyword argument is missing")
result = blockingCallFromThread(function, *args, **kwargs)
if not block:
result = PendingResult(self, result)
return result
def clear(self,taskids=None):
"""
Clear all previously run tasks from the task controller.
This is needed because the task controller keep all task results
in memory. This can be a problem is there are many completed
tasks. Users should call this periodically to clean out these
cached task results.
"""
return blockingCallFromThread(self.task_controller.clear, taskids)
def clear(self):
"""
Clear all previously run tasks from the task controller.
This is needed because the task controller keep all task results
in memory. This can be a problem is there are many completed
tasks. Users should call this periodically to clean out these
cached task results.
"""
return blockingCallFromThread(self.task_controller.clear)
def _bcft(self, *args, **kwargs):
try:
result = blockingCallFromThread(*args, **kwargs)
except DeadReferenceError:
raise error.ConnectionError(
"""A connection error has occurred in trying to connect to the
controller. This is usually caused by the controller dying or
being restarted. To resolve this issue try recreating the
task client.""")
else:
return result
def _bcft(self, *args, **kwargs):
try:
result = blockingCallFromThread(*args, **kwargs)
except DeadReferenceError:
raise error.ConnectionError(
"""A connection error has occurred in trying to connect to the
controller. This is usually caused by the controller dying or
being restarted. To resolve this issue try recreating the
multiengine client."""
)
else:
return result
def get_task_client(furl_or_file=''):
"""Get the blocking Task client.
:Parameters:
furl_or_file : str
A furl or a filename containing a furl. If empty, the
default furl_file will be used
:Returns:
The connected TaskClient instance
"""
client = blockingCallFromThread(_client_tub.get_task_client, furl_or_file)
return client.adapt_to_blocking_client()
def queue_status(self, verbose=False):
"""
Get a dictionary with the current state of the task queue.
:Parameters:
verbose : boolean
If True, return a list of taskids. If False, simply give
the number of tasks with each status.
:Returns:
A dict with the queue status.
"""
return blockingCallFromThread(self.task_controller.queue_status, verbose)
def get_task_result(self, taskid, block=False):
"""
Get a task result by taskid.
:Parameters:
taskid : int
The taskid of the task to be retrieved.
block : boolean
Should I block until the task is done?
:Returns: A `TaskResult` object that encapsulates the task result.
"""
return blockingCallFromThread(self.task_controller.get_task_result,
taskid, block)
def queue_status(self, verbose=False):
"""
Get a dictionary with the current state of the task queue.
:Parameters:
verbose : boolean
If True, return a list of taskids. If False, simply give
the number of tasks with each status.
:Returns:
A dict with the queue status.
"""
return blockingCallFromThread(self.task_controller.queue_status,
verbose)
def get_task_client(furl_or_file=''):
"""Get the blocking Task client.
:Parameters:
furl_or_file : str
A furl or a filename containing a furl. If empty, the
default furl_file will be used
:Returns:
The connected TaskClient instance
"""
client = blockingCallFromThread(_client_tub.get_task_client,
furl_or_file)
return client.adapt_to_blocking_client()
def get_task_result(self, taskid, block=False):
"""
Get a task result by taskid.
:Parameters:
taskid : int
The taskid of the task to be retrieved.
block : boolean
Should I block until the task is done?
:Returns: A `TaskResult` object that encapsulates the task result.
"""
return blockingCallFromThread(self.task_controller.get_task_result,
taskid, block)
def flush(self):
"""
Clear all pending deferreds/results from the controller.
For each `PendingResult` that is created by this client, the controller
holds on to the result for that `PendingResult`. This can be a problem
if there are a large number of `PendingResult` objects that are created.
Once the result of the `PendingResult` has been retrieved, the result
is removed from the controller, but if a user doesn't get a result (
they just ignore the `PendingResult`) the result is kept forever on the
controller. This method allows the user to clear out all un-retrieved
results on the controller.
"""
r = blockingCallFromThread(self.smultiengine.clear_pending_deferreds)
return r
def flush(self):
"""
Clear all pending deferreds/results from the controller.
For each `PendingResult` that is created by this client, the controller
holds on to the result for that `PendingResult`. This can be a problem
if there are a large number of `PendingResult` objects that are created.
Once the result of the `PendingResult` has been retrieved, the result
is removed from the controller, but if a user doesn't get a result (
they just ignore the `PendingResult`) the result is kept forever on the
controller. This method allows the user to clear out all un-retrieved
results on the controller.
"""
r = blockingCallFromThread(self.smultiengine.clear_pending_deferreds)
return r
def run(self, task, block=False):
"""Run a task on the `TaskController`.
See the documentation of the `MapTask` and `StringTask` classes for
details on how to build a task of different types.
:Parameters:
task : an `ITask` implementer
:Returns: The int taskid of the submitted task. Pass this to
`get_task_result` to get the `TaskResult` object.
"""
tid = blockingCallFromThread(self.task_controller.run, task)
if block:
return self.get_task_result(tid, block=True)
else:
return tid
def run(self, task, block=False):
"""Run a task on the `TaskController`.
See the documentation of the `MapTask` and `StringTask` classes for
details on how to build a task of different types.
:Parameters:
task : an `ITask` implementer
:Returns: The int taskid of the submitted task. Pass this to
`get_task_result` to get the `TaskResult` object.
"""
tid = blockingCallFromThread(self.task_controller.run, task)
if block:
return self.get_task_result(tid, block=True)
else:
return tid
def get_multiengine_client(self, profile='default', cluster_dir=None,
furl_or_file=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get the multiengine client.
Usually only the ``profile`` option will be needed. If a FURL file
can't be found by its profile, use ``cluster_dir`` or
``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
The multiengine client instance.
"""
client = blockingCallFromThread(
self.async_cc.get_multiengine_client, profile, cluster_dir,
furl_or_file, ipython_dir, delay, max_tries
)
return client.adapt_to_blocking_client()
def get_multiengine_client(self, profile='default', cluster_dir=None,
furl_or_file=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get the multiengine client.
Usually only the ``profile`` option will be needed. If a FURL file
can't be found by its profile, use ``cluster_dir`` or
``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
The multiengine client instance.
"""
client = blockingCallFromThread(
self.async_cc.get_multiengine_client, profile, cluster_dir,
furl_or_file, ipython_dir, delay, max_tries
)
return client.adapt_to_blocking_client()
def execute(self, lines, targets=None, block=None):
"""
Execute code on a set of engines.
:Parameters:
lines : str
The Python code to execute as a string
targets : id or list of ids
The engine to use for the execution
block : boolean
If False, this method will return the actual result. If False,
a `PendingResult` is returned which can be used to get the result
at a later time.
"""
targets, block = self._findTargetsAndBlock(targets, block)
result = blockingCallFromThread(self.smultiengine.execute, lines,
targets=targets, block=block)
if block:
result = ResultList(result)
else:
result = PendingResult(self, result)
result.add_callback(wrapResultList)
return result
def stop(self):
"""Stop the IPython cluster if it is running."""
return blockingCallFromThread(self.async_cluster.stop)
def stop(self):
"""Stop the IPython cluster if it is running."""
return blockingCallFromThread(self.async_cluster.stop)
def get_pending_deferred(self, deferredID, block):
return blockingCallFromThread(self.smultiengine.get_pending_deferred, deferredID, block)
def get_ids(self):
"""
Returns the ids of currently registered engines.
"""
result = blockingCallFromThread(self.smultiengine.get_ids)
return result
def get_ids(self):
"""
Returns the ids of currently registered engines.
"""
result = blockingCallFromThread(self.smultiengine.get_ids)
return result
def get_pending_deferred(self, deferredID, block):
return blockingCallFromThread(self.smultiengine.get_pending_deferred,
deferredID, block)
