class AsyncClientConnector(object):
"""A class for getting remote references and clients from furls.
This start a single :class:`Tub` for all remote reference and caches
references.
"""
def __init__(self):
self._remote_refs = {}
self.tub = Tub()
self.tub.startService()
def _find_furl(self, profile='default', cluster_dir=None,
furl_or_file=None, furl_file_name=None,
ipython_dir=None):
"""Find a FURL file.
If successful, this returns a FURL file that exists on the file
system. The contents of the file have not been checked though. This
is because we often have to deal with FURL file whose buffers have
not been flushed.
This raises an :exc:`~IPython.kernel.fcutil.FURLError` exception
if a FURL file can't be found.
This tries the following:
1. By the name ``furl_or_file``.
2. By ``cluster_dir`` and ``furl_file_name``.
3. By cluster profile with a default of ``default``. This uses
``ipython_dir``.
"""
# Try by furl_or_file
if furl_or_file is not None:
if is_valid_furl_or_file(furl_or_file):
return furl_or_file
if furl_file_name is None:
raise FURLError('A furl_file_name must be provided if furl_or_file is not')
# Try by cluster_dir
if cluster_dir is not None:
cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
sdir = cluster_dir_obj.security_dir
furl_file = os.path.join(sdir, furl_file_name)
validate_furl_or_file(furl_file)
return furl_file
# Try by profile
if ipython_dir is None:
ipython_dir = get_ipython_dir()
if profile is not None:
cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
ipython_dir, profile)
sdir = cluster_dir_obj.security_dir
furl_file = os.path.join(sdir, furl_file_name)
validate_furl_or_file(furl_file)
return furl_file
raise FURLError('Could not find a valid FURL file.')
def get_reference(self, furl_or_file):
"""Get a remote reference using a furl or a file containing a furl.
Remote references are cached locally so once a remote reference
has been retrieved for a given furl, the cached version is
returned.
Parameters
----------
furl_or_file : str
A furl or a filename containing a furl. This should already be
validated, but might not yet exist.
Returns
-------
A deferred to a remote reference
"""
furl = furl_or_file
if furl in self._remote_refs:
d = defer.succeed(self._remote_refs[furl])
else:
d = self.tub.getReference(furl)
d.addCallback(self._save_ref, furl)
return d
def _save_ref(self, ref, furl):
"""Cache a remote reference by its furl."""
self._remote_refs[furl] = ref
return ref
def get_task_client(self, profile='default', cluster_dir=None,
furl_or_file=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get the task controller client.
This method is a simple wrapper around `get_client` that passes in
the default name of the task client FURL file. Usually only
the ``profile`` option will be needed. If a FURL file can't be
found by its profile, use ``cluster_dir`` or ``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
A deferred to the actual client class.
"""
return self.get_client(
profile, cluster_dir, furl_or_file,
'ipcontroller-tc.furl', ipython_dir,
delay, max_tries
)
def get_multiengine_client(self, profile='default', cluster_dir=None,
furl_or_file=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get the multiengine controller client.
This method is a simple wrapper around `get_client` that passes in
the default name of the task client FURL file. Usually only
the ``profile`` option will be needed. If a FURL file can't be
found by its profile, use ``cluster_dir`` or ``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
A deferred to the actual client class.
"""
return self.get_client(
profile, cluster_dir, furl_or_file,
'ipcontroller-mec.furl', ipython_dir,
delay, max_tries
)
def get_client(self, profile='default', cluster_dir=None,
furl_or_file=None, furl_file_name=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get a remote reference and wrap it in a client by furl.
This method is a simple wrapper around `get_client` that passes in
the default name of the task client FURL file. Usually only
the ``profile`` option will be needed. If a FURL file can't be
found by its profile, use ``cluster_dir`` or ``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
furl_file_name : str
The filename (not the full path) of the FURL. This must be
provided if ``furl_or_file`` is not.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
A deferred to the actual client class. Or a failure to a
:exc:`FURLError`.
"""
try:
furl_file = self._find_furl(
profile, cluster_dir, furl_or_file,
furl_file_name, ipython_dir
)
# If this succeeds, we know the furl file exists and has a .furl
# extension, but it could still be empty. That is checked each
# connection attempt.
except FURLError:
return defer.fail(failure.Failure())
def _wrap_remote_reference(rr):
d = rr.callRemote('get_client_name')
d.addCallback(lambda name: import_item(name))
def adapt(client_interface):
client = client_interface(rr)
client.tub = self.tub
return client
d.addCallback(adapt)
return d
d = self._try_to_connect(furl_file, delay, max_tries, attempt=0)
d.addCallback(_wrap_remote_reference)
d.addErrback(self._handle_error, furl_file)
return d
def _handle_error(self, f, furl_file):
raise ClientConnectorError('Could not connect to the controller '
'using the FURL file. This usually means that i) the controller '
'was not started or ii) a firewall was blocking the client from '
'connecting to the controller: %s' % furl_file)
@inlineCallbacks
def _try_to_connect(self, furl_or_file, delay, max_tries, attempt):
"""Try to connect to the controller with retry logic."""
if attempt < max_tries:
log.msg("Connecting [%r]" % attempt)
try:
self.furl = find_furl(furl_or_file)
# Uncomment this to see the FURL being tried.
# log.msg("FURL: %s" % self.furl)
rr = yield self.get_reference(self.furl)
log.msg("Connected: %s" % furl_or_file)
except:
if attempt==max_tries-1:
# This will propagate the exception all the way to the top
# where it can be handled.
raise
else:
yield sleep_deferred(delay)
rr = yield self._try_to_connect(
furl_or_file, 1.5*delay, max_tries, attempt+1
)
returnValue(rr)
else:
returnValue(rr)
else:
raise ClientConnectorError(
'Could not connect to controller, max_tries (%r) exceeded. '
'This usually means that i) the controller was not started, '
'or ii) a firewall was blocking the client from connecting '
'to the controller.' % max_tries
)
class ClientConnector(object):
"""
This class gets remote references from furls and returns the wrapped clients.
This class is also used in `client.py` and `asyncclient.py` to create
a single per client-process Tub.
"""
def __init__(self):
self._remote_refs = {}
self.tub = Tub()
self.tub.startService()
def get_reference(self, furl_or_file):
"""
Get a remote reference using a furl or a file containing a furl.
Remote references are cached locally so once a remote reference
has been retrieved for a given furl, the cached version is
returned.
:Parameters:
furl_or_file : str
A furl or a filename containing a furl
:Returns:
A deferred to a remote reference
"""
furl = find_furl(furl_or_file)
if furl in self._remote_refs:
d = defer.succeed(self._remote_refs[furl])
else:
d = self.tub.getReference(furl)
d.addCallback(self.save_ref, furl)
return d
def save_ref(self, ref, furl):
"""
Cache a remote reference by its furl.
"""
self._remote_refs[furl] = ref
return ref
def get_task_client(self, furl_or_file=''):
"""
Get the task controller client.
This method is a simple wrapper around `get_client` that allow
`furl_or_file` to be empty, in which case, the furls is taken
from the default furl file given in the configuration.
:Parameters:
furl_or_file : str
A furl or a filename containing a furl. If empty, the
default furl_file will be used
:Returns:
A deferred to the actual client class
"""
task_co = client_co['client_interfaces']['task']
if furl_or_file:
ff = furl_or_file
else:
ff = task_co['furl_file']
return self.get_client(ff)
def get_multiengine_client(self, furl_or_file=''):
"""
Get the multiengine controller client.
This method is a simple wrapper around `get_client` that allow
`furl_or_file` to be empty, in which case, the furls is taken
from the default furl file given in the configuration.
:Parameters:
furl_or_file : str
A furl or a filename containing a furl. If empty, the
default furl_file will be used
:Returns:
A deferred to the actual client class
"""
task_co = client_co['client_interfaces']['multiengine']
if furl_or_file:
ff = furl_or_file
else:
ff = task_co['furl_file']
return self.get_client(ff)
def get_client(self, furl_or_file):
"""
Get a remote reference and wrap it in a client by furl.
This method first gets a remote reference and then calls its
`get_client_name` method to find the apprpriate client class
that should be used to wrap the remote reference.
:Parameters:
furl_or_file : str
A furl or a filename containing a furl
:Returns:
A deferred to the actual client class
"""
furl = find_furl(furl_or_file)
d = self.get_reference(furl)
def wrap_remote_reference(rr):
d = rr.callRemote('get_client_name')
d.addCallback(lambda name: import_item(name))
def adapt(client_interface):
client = client_interface(rr)
client.tub = self.tub
return client
d.addCallback(adapt)
return d
d.addCallback(wrap_remote_reference)
return d
class AsyncClientConnector(object):
"""A class for getting remote references and clients from furls.
This start a single :class:`Tub` for all remote reference and caches
references.
"""
def __init__(self):
self._remote_refs = {}
self.tub = Tub()
self.tub.startService()
def _find_furl(self, profile='default', cluster_dir=None,
furl_or_file=None, furl_file_name=None,
ipython_dir=None):
"""Find a FURL file.
If successful, this returns a FURL file that exists on the file
system. The contents of the file have not been checked though. This
is because we often have to deal with FURL file whose buffers have
not been flushed.
This raises an :exc:`~IPython.kernel.fcutil.FURLError` exception
if a FURL file can't be found.
This tries the following:
1. By the name ``furl_or_file``.
2. By ``cluster_dir`` and ``furl_file_name``.
3. By cluster profile with a default of ``default``. This uses
``ipython_dir``.
"""
# Try by furl_or_file
if furl_or_file is not None:
if is_valid_furl_or_file(furl_or_file):
return furl_or_file
if furl_file_name is None:
raise FURLError('A furl_file_name must be provided if furl_or_file is not')
# Try by cluster_dir
if cluster_dir is not None:
cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
sdir = cluster_dir_obj.security_dir
furl_file = os.path.join(sdir, furl_file_name)
validate_furl_or_file(furl_file)
return furl_file
# Try by profile
if ipython_dir is None:
ipython_dir = get_ipython_dir()
if profile is not None:
cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
ipython_dir, profile)
sdir = cluster_dir_obj.security_dir
furl_file = os.path.join(sdir, furl_file_name)
validate_furl_or_file(furl_file)
return furl_file
raise FURLError('Could not find a valid FURL file.')
def get_reference(self, furl_or_file):
"""Get a remote reference using a furl or a file containing a furl.
Remote references are cached locally so once a remote reference
has been retrieved for a given furl, the cached version is
returned.
Parameters
----------
furl_or_file : str
A furl or a filename containing a furl. This should already be
validated, but might not yet exist.
Returns
-------
A deferred to a remote reference
"""
furl = furl_or_file
if furl in self._remote_refs:
d = defer.succeed(self._remote_refs[furl])
else:
d = self.tub.getReference(furl)
d.addCallback(self._save_ref, furl)
return d
def _save_ref(self, ref, furl):
"""Cache a remote reference by its furl."""
self._remote_refs[furl] = ref
return ref
def get_task_client(self, profile='default', cluster_dir=None,
furl_or_file=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get the task controller client.
This method is a simple wrapper around `get_client` that passes in
the default name of the task client FURL file. Usually only
the ``profile`` option will be needed. If a FURL file can't be
found by its profile, use ``cluster_dir`` or ``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
A deferred to the actual client class.
"""
return self.get_client(
profile, cluster_dir, furl_or_file,
'ipcontroller-tc.furl', ipython_dir,
delay, max_tries
)
def get_multiengine_client(self, profile='default', cluster_dir=None,
furl_or_file=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get the multiengine controller client.
This method is a simple wrapper around `get_client` that passes in
the default name of the task client FURL file. Usually only
the ``profile`` option will be needed. If a FURL file can't be
found by its profile, use ``cluster_dir`` or ``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
A deferred to the actual client class.
"""
return self.get_client(
profile, cluster_dir, furl_or_file,
'ipcontroller-mec.furl', ipython_dir,
delay, max_tries
)
def get_client(self, profile='default', cluster_dir=None,
furl_or_file=None, furl_file_name=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get a remote reference and wrap it in a client by furl.
This method is a simple wrapper around `get_client` that passes in
the default name of the task client FURL file. Usually only
the ``profile`` option will be needed. If a FURL file can't be
found by its profile, use ``cluster_dir`` or ``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
furl_file_name : str
The filename (not the full path) of the FURL. This must be
provided if ``furl_or_file`` is not.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
A deferred to the actual client class. Or a failure to a
:exc:`FURLError`.
"""
try:
furl_file = self._find_furl(
profile, cluster_dir, furl_or_file,
furl_file_name, ipython_dir
)
# If this succeeds, we know the furl file exists and has a .furl
# extension, but it could still be empty. That is checked each
# connection attempt.
except FURLError:
return defer.fail(failure.Failure())
def _wrap_remote_reference(rr):
d = rr.callRemote('get_client_name')
d.addCallback(lambda name: import_item(name))
def adapt(client_interface):
client = client_interface(rr)
client.tub = self.tub
return client
d.addCallback(adapt)
return d
d = self._try_to_connect(furl_file, delay, max_tries, attempt=0)
d.addCallback(_wrap_remote_reference)
d.addErrback(self._handle_error, furl_file)
return d
def _handle_error(self, f, furl_file):
raise ClientConnectorError('Could not connect to the controller '
'using the FURL file. This usually means that i) the controller '
'was not started or ii) a firewall was blocking the client from '
'connecting to the controller: %s' % furl_file)
@inlineCallbacks
def _try_to_connect(self, furl_or_file, delay, max_tries, attempt):
"""Try to connect to the controller with retry logic."""
if attempt < max_tries:
log.msg("Connecting [%r]" % attempt)
try:
self.furl = find_furl(furl_or_file)
# Uncomment this to see the FURL being tried.
# log.msg("FURL: %s" % self.furl)
rr = yield self.get_reference(self.furl)
log.msg("Connected: %s" % furl_or_file)
except:
if attempt==max_tries-1:
# This will propagate the exception all the way to the top
# where it can be handled.
raise
else:
yield sleep_deferred(delay)
rr = yield self._try_to_connect(
furl_or_file, 1.5*delay, max_tries, attempt+1
)
returnValue(rr)
else:
returnValue(rr)
else:
raise ClientConnectorError(
'Could not connect to controller, max_tries (%r) exceeded. '
'This usually means that i) the controller was not started, '
'or ii) a firewall was blocking the client from connecting '
'to the controller.' % max_tries
)