Ejemplo n.º 1
0
class AsyncClientConnector(object):
    """A class for getting remote references and clients from furls.

    This start a single :class:`Tub` for all remote reference and caches
    references.
    """

    def __init__(self):
        self._remote_refs = {}
        self.tub = Tub()
        self.tub.startService()

    def _find_furl(self, profile='default', cluster_dir=None, 
                   furl_or_file=None, furl_file_name=None,
                   ipython_dir=None):
        """Find a FURL file.

        If successful, this returns a FURL file that exists on the file
        system. The contents of the file have not been checked though. This
        is because we often have to deal with FURL file whose buffers have
        not been flushed.

        This raises an :exc:`~IPython.kernel.fcutil.FURLError` exception 
        if a FURL file can't be found.

        This tries the following:
        
        1. By the name ``furl_or_file``.
        2. By ``cluster_dir`` and ``furl_file_name``.
        3. By cluster profile with a default of ``default``. This uses
           ``ipython_dir``.
        """
        # Try by furl_or_file
        if furl_or_file is not None:
            if is_valid_furl_or_file(furl_or_file):
                return furl_or_file

        if furl_file_name is None:
            raise FURLError('A furl_file_name must be provided if furl_or_file is not')

        # Try by cluster_dir
        if cluster_dir is not None:
            cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
            sdir = cluster_dir_obj.security_dir
            furl_file = os.path.join(sdir, furl_file_name)
            validate_furl_or_file(furl_file)
            return furl_file

        # Try by profile
        if ipython_dir is None:
            ipython_dir = get_ipython_dir()
        if profile is not None:
            cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
                ipython_dir, profile)
            sdir = cluster_dir_obj.security_dir
            furl_file = os.path.join(sdir, furl_file_name)
            validate_furl_or_file(furl_file)
            return furl_file

        raise FURLError('Could not find a valid FURL file.')

    def get_reference(self, furl_or_file):
        """Get a remote reference using a furl or a file containing a furl.

        Remote references are cached locally so once a remote reference
        has been retrieved for a given furl, the cached version is 
        returned.

        Parameters
        ----------
        furl_or_file : str
            A furl or a filename containing a furl. This should already be
            validated, but might not yet exist.

        Returns
        -------
        A deferred to a remote reference
        """
        furl = furl_or_file
        if furl in self._remote_refs:
            d = defer.succeed(self._remote_refs[furl])
        else:
            d = self.tub.getReference(furl)
            d.addCallback(self._save_ref, furl)
        return d
        
    def _save_ref(self, ref, furl):
        """Cache a remote reference by its furl."""
        self._remote_refs[furl] = ref
        return ref
        
    def get_task_client(self, profile='default', cluster_dir=None,
                        furl_or_file=None, ipython_dir=None,
                        delay=DELAY, max_tries=MAX_TRIES):
        """Get the task controller client.
        
        This method is a simple wrapper around `get_client` that passes in
        the default name of the task client FURL file.  Usually only
        the ``profile`` option will be needed.  If a FURL file can't be
        found by its profile, use ``cluster_dir`` or ``furl_or_file``.
        
        Parameters
        ----------
        profile : str
            The name of a cluster directory profile (default="default"). The
            cluster directory "cluster_<profile>" will be searched for
            in ``os.getcwd()``, the ipython_dir and then in the directories
            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
        cluster_dir : str
            The full path to a cluster directory.  This is useful if profiles
            are not being used.
        furl_or_file : str
            A furl or a filename containing a FURL. This is useful if you 
            simply know the location of the FURL file.
        ipython_dir : str
            The location of the ipython_dir if different from the default.
            This is used if the cluster directory is being found by profile.
        delay : float
            The initial delay between re-connection attempts. Susequent delays
            get longer according to ``delay[i] = 1.5*delay[i-1]``.
        max_tries : int
            The max number of re-connection attempts.

        Returns
        -------
        A deferred to the actual client class.
        """
        return self.get_client(
            profile, cluster_dir, furl_or_file, 
            'ipcontroller-tc.furl', ipython_dir,
            delay, max_tries
        )

    def get_multiengine_client(self, profile='default', cluster_dir=None,
                               furl_or_file=None, ipython_dir=None,
                               delay=DELAY, max_tries=MAX_TRIES):
        """Get the multiengine controller client.
        
        This method is a simple wrapper around `get_client` that passes in
        the default name of the task client FURL file.  Usually only
        the ``profile`` option will be needed.  If a FURL file can't be
        found by its profile, use ``cluster_dir`` or ``furl_or_file``.
        
        Parameters
        ----------
        profile : str
            The name of a cluster directory profile (default="default"). The
            cluster directory "cluster_<profile>" will be searched for
            in ``os.getcwd()``, the ipython_dir and then in the directories
            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
        cluster_dir : str
            The full path to a cluster directory.  This is useful if profiles
            are not being used.
        furl_or_file : str
            A furl or a filename containing a FURL. This is useful if you 
            simply know the location of the FURL file.
        ipython_dir : str
            The location of the ipython_dir if different from the default.
            This is used if the cluster directory is being found by profile.
        delay : float
            The initial delay between re-connection attempts. Susequent delays
            get longer according to ``delay[i] = 1.5*delay[i-1]``.
        max_tries : int
            The max number of re-connection attempts.

        Returns
        -------
        A deferred to the actual client class.
        """
        return self.get_client(
            profile, cluster_dir, furl_or_file, 
            'ipcontroller-mec.furl', ipython_dir,
            delay, max_tries
        )
    
    def get_client(self, profile='default', cluster_dir=None,
                   furl_or_file=None, furl_file_name=None, ipython_dir=None,
                   delay=DELAY, max_tries=MAX_TRIES):
        """Get a remote reference and wrap it in a client by furl.

        This method is a simple wrapper around `get_client` that passes in
        the default name of the task client FURL file.  Usually only
        the ``profile`` option will be needed.  If a FURL file can't be
        found by its profile, use ``cluster_dir`` or ``furl_or_file``.
        
        Parameters
        ----------
        profile : str
            The name of a cluster directory profile (default="default"). The
            cluster directory "cluster_<profile>" will be searched for
            in ``os.getcwd()``, the ipython_dir and then in the directories
            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
        cluster_dir : str
            The full path to a cluster directory.  This is useful if profiles
            are not being used.
        furl_or_file : str
            A furl or a filename containing a FURL. This is useful if you 
            simply know the location of the FURL file.
        furl_file_name : str
            The filename (not the full path) of the FURL. This must be
            provided if ``furl_or_file`` is not.
        ipython_dir : str
            The location of the ipython_dir if different from the default.
            This is used if the cluster directory is being found by profile.
        delay : float
            The initial delay between re-connection attempts. Susequent delays
            get longer according to ``delay[i] = 1.5*delay[i-1]``.
        max_tries : int
            The max number of re-connection attempts.

        Returns
        -------
        A deferred to the actual client class. Or a failure to a 
        :exc:`FURLError`.
        """
        try:
            furl_file = self._find_furl(
                profile, cluster_dir, furl_or_file,
                furl_file_name, ipython_dir
            )
            # If this succeeds, we know the furl file exists and has a .furl
            # extension, but it could still be empty. That is checked each
            # connection attempt.
        except FURLError:
            return defer.fail(failure.Failure())

        def _wrap_remote_reference(rr):
            d = rr.callRemote('get_client_name')
            d.addCallback(lambda name: import_item(name))
            def adapt(client_interface):
                client = client_interface(rr)
                client.tub = self.tub
                return client
            d.addCallback(adapt)

            return d

        d = self._try_to_connect(furl_file, delay, max_tries, attempt=0)
        d.addCallback(_wrap_remote_reference)
        d.addErrback(self._handle_error, furl_file)
        return d

    def _handle_error(self, f, furl_file):
        raise ClientConnectorError('Could not connect to the controller '
            'using the FURL file. This usually means that i) the controller '
            'was not started or ii) a firewall was blocking the client from '
            'connecting to the controller: %s' % furl_file)

    @inlineCallbacks
    def _try_to_connect(self, furl_or_file, delay, max_tries, attempt):
        """Try to connect to the controller with retry logic."""
        if attempt < max_tries:
            log.msg("Connecting [%r]" % attempt)
            try:
                self.furl = find_furl(furl_or_file)
                # Uncomment this to see the FURL being tried.
                # log.msg("FURL: %s" % self.furl)
                rr = yield self.get_reference(self.furl)
                log.msg("Connected: %s" % furl_or_file)
            except:
                if attempt==max_tries-1:
                    # This will propagate the exception all the way to the top
                    # where it can be handled.
                    raise
                else:
                    yield sleep_deferred(delay)
                    rr = yield self._try_to_connect(
                        furl_or_file, 1.5*delay, max_tries, attempt+1
                    )
                    returnValue(rr)
            else:
                returnValue(rr)
        else:
            raise ClientConnectorError(
                'Could not connect to controller, max_tries (%r) exceeded. '
                'This usually means that i) the controller was not started, '
                'or ii) a firewall was blocking the client from connecting '
                'to the controller.' % max_tries
            )
Ejemplo n.º 2
0
class ClientConnector(object):
    """
    This class gets remote references from furls and returns the wrapped clients.
    
    This class is also used in `client.py` and `asyncclient.py` to create 
    a single per client-process Tub.
    """
    
    def __init__(self):
        self._remote_refs = {}
        self.tub = Tub()
        self.tub.startService()
    
    def get_reference(self, furl_or_file):
        """
        Get a remote reference using a furl or a file containing a furl.
        
        Remote references are cached locally so once a remote reference
        has been retrieved for a given furl, the cached version is 
        returned.
        
        :Parameters:
            furl_or_file : str
                A furl or a filename containing a furl
        
        :Returns:
            A deferred to a remote reference
        """
        furl = find_furl(furl_or_file)
        if furl in self._remote_refs:
            d = defer.succeed(self._remote_refs[furl])
        else:
            d = self.tub.getReference(furl)
            d.addCallback(self.save_ref, furl)
        return d
        
    def save_ref(self, ref, furl):
        """
        Cache a remote reference by its furl.
        """
        self._remote_refs[furl] = ref
        return ref
        
    def get_task_client(self, furl_or_file=''):
        """
        Get the task controller client.
        
        This method is a simple wrapper around `get_client` that allow
        `furl_or_file` to be empty, in which case, the furls is taken
        from the default furl file given in the configuration.
        
        :Parameters:
            furl_or_file : str
                A furl or a filename containing a furl.  If empty, the
                default furl_file will be used
                
        :Returns:
            A deferred to the actual client class
        """
        task_co = client_co['client_interfaces']['task']
        if furl_or_file:
            ff = furl_or_file
        else:
            ff = task_co['furl_file']
        return self.get_client(ff)

    def get_multiengine_client(self, furl_or_file=''):
        """
        Get the multiengine controller client.
        
        This method is a simple wrapper around `get_client` that allow
        `furl_or_file` to be empty, in which case, the furls is taken
        from the default furl file given in the configuration.
        
        :Parameters:
            furl_or_file : str
                A furl or a filename containing a furl.  If empty, the
                default furl_file will be used
                
        :Returns:
            A deferred to the actual client class
        """
        task_co = client_co['client_interfaces']['multiengine']
        if furl_or_file:
            ff = furl_or_file
        else:
            ff = task_co['furl_file']
        return self.get_client(ff)
    
    def get_client(self, furl_or_file):
        """
        Get a remote reference and wrap it in a client by furl.
        
        This method first gets a remote reference and then calls its 
        `get_client_name` method to find the apprpriate client class
        that should be used to wrap the remote reference.
        
        :Parameters:
            furl_or_file : str
                A furl or a filename containing a furl
        
        :Returns:
            A deferred to the actual client class
        """
        furl = find_furl(furl_or_file)
        d = self.get_reference(furl)
        def wrap_remote_reference(rr):
            d = rr.callRemote('get_client_name')
            d.addCallback(lambda name: import_item(name))
            def adapt(client_interface):
                client = client_interface(rr)
                client.tub = self.tub
                return client
            d.addCallback(adapt)

            return d
        d.addCallback(wrap_remote_reference)
        return d
Ejemplo n.º 3
0
class AsyncClientConnector(object):
    """A class for getting remote references and clients from furls.

    This start a single :class:`Tub` for all remote reference and caches
    references.
    """

    def __init__(self):
        self._remote_refs = {}
        self.tub = Tub()
        self.tub.startService()

    def _find_furl(self, profile='default', cluster_dir=None, 
                   furl_or_file=None, furl_file_name=None,
                   ipython_dir=None):
        """Find a FURL file.

        If successful, this returns a FURL file that exists on the file
        system. The contents of the file have not been checked though. This
        is because we often have to deal with FURL file whose buffers have
        not been flushed.

        This raises an :exc:`~IPython.kernel.fcutil.FURLError` exception 
        if a FURL file can't be found.

        This tries the following:
        
        1. By the name ``furl_or_file``.
        2. By ``cluster_dir`` and ``furl_file_name``.
        3. By cluster profile with a default of ``default``. This uses
           ``ipython_dir``.
        """
        # Try by furl_or_file
        if furl_or_file is not None:
            if is_valid_furl_or_file(furl_or_file):
                return furl_or_file

        if furl_file_name is None:
            raise FURLError('A furl_file_name must be provided if furl_or_file is not')

        # Try by cluster_dir
        if cluster_dir is not None:
            cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
            sdir = cluster_dir_obj.security_dir
            furl_file = os.path.join(sdir, furl_file_name)
            validate_furl_or_file(furl_file)
            return furl_file

        # Try by profile
        if ipython_dir is None:
            ipython_dir = get_ipython_dir()
        if profile is not None:
            cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
                ipython_dir, profile)
            sdir = cluster_dir_obj.security_dir
            furl_file = os.path.join(sdir, furl_file_name)
            validate_furl_or_file(furl_file)
            return furl_file

        raise FURLError('Could not find a valid FURL file.')

    def get_reference(self, furl_or_file):
        """Get a remote reference using a furl or a file containing a furl.

        Remote references are cached locally so once a remote reference
        has been retrieved for a given furl, the cached version is 
        returned.

        Parameters
        ----------
        furl_or_file : str
            A furl or a filename containing a furl. This should already be
            validated, but might not yet exist.

        Returns
        -------
        A deferred to a remote reference
        """
        furl = furl_or_file
        if furl in self._remote_refs:
            d = defer.succeed(self._remote_refs[furl])
        else:
            d = self.tub.getReference(furl)
            d.addCallback(self._save_ref, furl)
        return d
        
    def _save_ref(self, ref, furl):
        """Cache a remote reference by its furl."""
        self._remote_refs[furl] = ref
        return ref
        
    def get_task_client(self, profile='default', cluster_dir=None,
                        furl_or_file=None, ipython_dir=None,
                        delay=DELAY, max_tries=MAX_TRIES):
        """Get the task controller client.
        
        This method is a simple wrapper around `get_client` that passes in
        the default name of the task client FURL file.  Usually only
        the ``profile`` option will be needed.  If a FURL file can't be
        found by its profile, use ``cluster_dir`` or ``furl_or_file``.
        
        Parameters
        ----------
        profile : str
            The name of a cluster directory profile (default="default"). The
            cluster directory "cluster_<profile>" will be searched for
            in ``os.getcwd()``, the ipython_dir and then in the directories
            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
        cluster_dir : str
            The full path to a cluster directory.  This is useful if profiles
            are not being used.
        furl_or_file : str
            A furl or a filename containing a FURL. This is useful if you 
            simply know the location of the FURL file.
        ipython_dir : str
            The location of the ipython_dir if different from the default.
            This is used if the cluster directory is being found by profile.
        delay : float
            The initial delay between re-connection attempts. Susequent delays
            get longer according to ``delay[i] = 1.5*delay[i-1]``.
        max_tries : int
            The max number of re-connection attempts.

        Returns
        -------
        A deferred to the actual client class.
        """
        return self.get_client(
            profile, cluster_dir, furl_or_file, 
            'ipcontroller-tc.furl', ipython_dir,
            delay, max_tries
        )

    def get_multiengine_client(self, profile='default', cluster_dir=None,
                               furl_or_file=None, ipython_dir=None,
                               delay=DELAY, max_tries=MAX_TRIES):
        """Get the multiengine controller client.
        
        This method is a simple wrapper around `get_client` that passes in
        the default name of the task client FURL file.  Usually only
        the ``profile`` option will be needed.  If a FURL file can't be
        found by its profile, use ``cluster_dir`` or ``furl_or_file``.
        
        Parameters
        ----------
        profile : str
            The name of a cluster directory profile (default="default"). The
            cluster directory "cluster_<profile>" will be searched for
            in ``os.getcwd()``, the ipython_dir and then in the directories
            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
        cluster_dir : str
            The full path to a cluster directory.  This is useful if profiles
            are not being used.
        furl_or_file : str
            A furl or a filename containing a FURL. This is useful if you 
            simply know the location of the FURL file.
        ipython_dir : str
            The location of the ipython_dir if different from the default.
            This is used if the cluster directory is being found by profile.
        delay : float
            The initial delay between re-connection attempts. Susequent delays
            get longer according to ``delay[i] = 1.5*delay[i-1]``.
        max_tries : int
            The max number of re-connection attempts.

        Returns
        -------
        A deferred to the actual client class.
        """
        return self.get_client(
            profile, cluster_dir, furl_or_file, 
            'ipcontroller-mec.furl', ipython_dir,
            delay, max_tries
        )
    
    def get_client(self, profile='default', cluster_dir=None,
                   furl_or_file=None, furl_file_name=None, ipython_dir=None,
                   delay=DELAY, max_tries=MAX_TRIES):
        """Get a remote reference and wrap it in a client by furl.

        This method is a simple wrapper around `get_client` that passes in
        the default name of the task client FURL file.  Usually only
        the ``profile`` option will be needed.  If a FURL file can't be
        found by its profile, use ``cluster_dir`` or ``furl_or_file``.
        
        Parameters
        ----------
        profile : str
            The name of a cluster directory profile (default="default"). The
            cluster directory "cluster_<profile>" will be searched for
            in ``os.getcwd()``, the ipython_dir and then in the directories
            listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
        cluster_dir : str
            The full path to a cluster directory.  This is useful if profiles
            are not being used.
        furl_or_file : str
            A furl or a filename containing a FURL. This is useful if you 
            simply know the location of the FURL file.
        furl_file_name : str
            The filename (not the full path) of the FURL. This must be
            provided if ``furl_or_file`` is not.
        ipython_dir : str
            The location of the ipython_dir if different from the default.
            This is used if the cluster directory is being found by profile.
        delay : float
            The initial delay between re-connection attempts. Susequent delays
            get longer according to ``delay[i] = 1.5*delay[i-1]``.
        max_tries : int
            The max number of re-connection attempts.

        Returns
        -------
        A deferred to the actual client class. Or a failure to a 
        :exc:`FURLError`.
        """
        try:
            furl_file = self._find_furl(
                profile, cluster_dir, furl_or_file,
                furl_file_name, ipython_dir
            )
            # If this succeeds, we know the furl file exists and has a .furl
            # extension, but it could still be empty. That is checked each
            # connection attempt.
        except FURLError:
            return defer.fail(failure.Failure())

        def _wrap_remote_reference(rr):
            d = rr.callRemote('get_client_name')
            d.addCallback(lambda name: import_item(name))
            def adapt(client_interface):
                client = client_interface(rr)
                client.tub = self.tub
                return client
            d.addCallback(adapt)

            return d

        d = self._try_to_connect(furl_file, delay, max_tries, attempt=0)
        d.addCallback(_wrap_remote_reference)
        d.addErrback(self._handle_error, furl_file)
        return d

    def _handle_error(self, f, furl_file):
        raise ClientConnectorError('Could not connect to the controller '
            'using the FURL file. This usually means that i) the controller '
            'was not started or ii) a firewall was blocking the client from '
            'connecting to the controller: %s' % furl_file)

    @inlineCallbacks
    def _try_to_connect(self, furl_or_file, delay, max_tries, attempt):
        """Try to connect to the controller with retry logic."""
        if attempt < max_tries:
            log.msg("Connecting [%r]" % attempt)
            try:
                self.furl = find_furl(furl_or_file)
                # Uncomment this to see the FURL being tried.
                # log.msg("FURL: %s" % self.furl)
                rr = yield self.get_reference(self.furl)
                log.msg("Connected: %s" % furl_or_file)
            except:
                if attempt==max_tries-1:
                    # This will propagate the exception all the way to the top
                    # where it can be handled.
                    raise
                else:
                    yield sleep_deferred(delay)
                    rr = yield self._try_to_connect(
                        furl_or_file, 1.5*delay, max_tries, attempt+1
                    )
                    returnValue(rr)
            else:
                returnValue(rr)
        else:
            raise ClientConnectorError(
                'Could not connect to controller, max_tries (%r) exceeded. '
                'This usually means that i) the controller was not started, '
                'or ii) a firewall was blocking the client from connecting '
                'to the controller.' % max_tries
            )