Exemplo n.º 1
0
    def __init__(self, workdir=None, datadir=None, backends_filename=None, scheduler=None, storage=None):
        super(Weboob, self).__init__(modules_path=False, scheduler=scheduler, storage=storage)

        # Create WORKDIR
        if workdir is None:
            if 'WEBOOB_WORKDIR' in os.environ:
                workdir = os.environ['WEBOOB_WORKDIR']
            else:
                workdir = os.path.join(os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')), 'weboob')

        self.workdir = os.path.realpath(workdir)
        self._create_dir(workdir)

        # Create DATADIR
        if datadir is None:
            if 'WEBOOB_DATADIR' in os.environ:
                datadir = os.environ['WEBOOB_DATADIR']
            elif 'WEBOOB_WORKDIR' in os.environ:
                datadir = os.environ['WEBOOB_WORKDIR']
            else:
                datadir = os.path.join(os.environ.get('XDG_DATA_HOME', os.path.join(os.path.expanduser('~'), '.local', 'share')), 'weboob')

        _datadir = os.path.realpath(datadir)
        self._create_dir(_datadir)

        # Modules management
        self.repositories = Repositories(workdir, _datadir, self.VERSION)
        self.modules_loader = RepositoryModulesLoader(self.repositories)

        # Backend instances config
        if not backends_filename:
            backends_filename = os.environ.get('WEBOOB_BACKENDS', os.path.join(self.workdir, self.BACKENDS_FILENAME))
        elif not backends_filename.startswith('/'):
            backends_filename = os.path.join(self.workdir, backends_filename)
        self.backends_config = BackendsConfig(backends_filename)
Exemplo n.º 2
0
    def __init__(self, workdir=None, backends_filename=None, scheduler=None, storage=None):
        self.logger = getLogger('weboob')
        self.backend_instances = {}
        self.callbacks = {'login':   lambda backend_name, value: None,
                          'captcha': lambda backend_name, image: None,
                         }

        # Scheduler
        if scheduler is None:
            scheduler = Scheduler()
        self.scheduler = scheduler

        # Create WORKDIR
        if workdir is not None:
            datadir = workdir
        elif 'WEBOOB_WORKDIR' in os.environ:
            datadir = workdir = os.environ.get('WEBOOB_WORKDIR')
        else:
            old_workdir = os.path.join(os.path.expanduser('~'), '.weboob')
            xdg_config_home = os.path.join(os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')), 'weboob')
            xdg_data_home = os.path.join(os.environ.get('XDG_DATA_HOME', os.path.join(os.path.expanduser('~'), '.local', 'share')), 'weboob')

            if os.path.isdir(old_workdir):
                self.logger.warning('You are using "%s" as working directory. Files are moved into %s and %s.'
                                    % (old_workdir, xdg_config_home, xdg_data_home))
                self._create_dir(xdg_config_home)
                self._create_dir(xdg_data_home)
                for f in os.listdir(old_workdir):
                    if f in Repositories.SHARE_DIRS:
                        dest = xdg_data_home
                    else:
                        dest = xdg_config_home
                    shutil.move(os.path.join(old_workdir, f), dest)
                shutil.rmtree(old_workdir)
            workdir = xdg_config_home
            datadir = xdg_data_home

        self.workdir = os.path.realpath(workdir)
        self._create_dir(workdir)

        # Repositories management
        self.repositories = Repositories(workdir, datadir, self.VERSION)

        # Backends loader
        self.modules_loader = ModulesLoader(self.repositories)

        # Backend instances config
        if not backends_filename:
            backends_filename = os.environ.get('WEBOOB_BACKENDS', os.path.join(self.workdir, self.BACKENDS_FILENAME))
        elif not backends_filename.startswith('/'):
            backends_filename = os.path.join(self.workdir, backends_filename)
        self.backends_config = BackendsConfig(backends_filename)

        # Storage
        self.storage = storage
Exemplo n.º 3
0
class Weboob(WebNip):
    """
    The main class of Weboob, used to manage backends, modules repositories and
    call methods on all loaded backends.

    :param workdir: optional parameter to set path of the working directory
    :type workdir: str
    :param datadir: optional parameter to set path of the data directory
    :type datadir: str
    :param backends_filename: name of the *backends* file, where configuration of
                              backends is stored
    :type backends_filename: str
    :param storage: provide a storage where backends can save data
    :type storage: :class:`weboob.tools.storage.IStorage`
    """
    BACKENDS_FILENAME = 'backends'

    def __init__(self, workdir=None, datadir=None, backends_filename=None, scheduler=None, storage=None):
        super(Weboob, self).__init__(modules_path=False, scheduler=scheduler, storage=storage)

        # Create WORKDIR
        if workdir is None:
            if 'WEBOOB_WORKDIR' in os.environ:
                workdir = os.environ['WEBOOB_WORKDIR']
            else:
                workdir = os.path.join(os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')), 'weboob')

        self.workdir = os.path.realpath(workdir)
        self._create_dir(workdir)

        # Create DATADIR
        if datadir is None:
            if 'WEBOOB_DATADIR' in os.environ:
                datadir = os.environ['WEBOOB_DATADIR']
            elif 'WEBOOB_WORKDIR' in os.environ:
                datadir = os.environ['WEBOOB_WORKDIR']
            else:
                datadir = os.path.join(os.environ.get('XDG_DATA_HOME', os.path.join(os.path.expanduser('~'), '.local', 'share')), 'weboob')

        _datadir = os.path.realpath(datadir)
        self._create_dir(_datadir)

        # Modules management
        self.repositories = Repositories(workdir, _datadir, self.VERSION)
        self.modules_loader = RepositoryModulesLoader(self.repositories)

        # Backend instances config
        if not backends_filename:
            backends_filename = os.environ.get('WEBOOB_BACKENDS', os.path.join(self.workdir, self.BACKENDS_FILENAME))
        elif not backends_filename.startswith('/'):
            backends_filename = os.path.join(self.workdir, backends_filename)
        self.backends_config = BackendsConfig(backends_filename)

    def _create_dir(self, name):
        if not os.path.exists(name):
            os.makedirs(name)
        elif not os.path.isdir(name):
            self.logger.error(u'"%s" is not a directory', name)

    def update(self, progress=PrintProgress()):
        """
        Update modules from repositories.
        """
        self.repositories.update(progress)

        modules_to_check = set([module_name for _, module_name, _ in self.backends_config.iter_backends()])
        for module_name in modules_to_check:
            minfo = self.repositories.get_module_info(module_name)
            if minfo and not minfo.is_installed():
                self.repositories.install(minfo, progress)

    def build_backend(self, module_name, params=None, storage=None, name=None, nofail=False):
        """
        Create a single backend which is not listed in configuration.

        :param module_name: name of module
        :param params: parameters to give to backend
        :type params: :class:`dict`
        :param storage: storage to use
        :type storage: :class:`weboob.tools.storage.IStorage`
        :param name: name of backend
        :type name: :class:`basestring`
        :rtype: :class:`weboob.tools.backend.Module`
        :param nofail: if true, this call can't fail
        :type nofail: :class:`bool`
        """
        minfo = self.repositories.get_module_info(module_name)
        if minfo is None:
            raise ModuleLoadError(module_name, 'Module does not exist.')

        if not minfo.is_installed():
            self.repositories.install(minfo)

        return super(Weboob, self).build_backend(module_name, params, storage, name, nofail)

    def load_backends(self, caps=None, names=None, modules=None, exclude=None, storage=None, errors=None):
        """
        Load backends listed in config file.

        :param caps: load backends which implement all of specified caps
        :type caps: tuple[:class:`weboob.capabilities.base.Capability`]
        :param names: load backends in list
        :type names: tuple[:class:`str`]
        :param modules: load backends which module is in list
        :type modules: tuple[:class:`str`]
        :param exclude: do not load backends in list
        :type exclude: tuple[:class:`str`]
        :param storage: use this storage if specified
        :type storage: :class:`weboob.tools.storage.IStorage`
        :param errors: if specified, store every errors in this list
        :type errors: list[:class:`LoadError`]
        :returns: loaded backends
        :rtype: dict[:class:`str`, :class:`weboob.tools.backend.Module`]
        """
        loaded = {}
        if storage is None:
            storage = self.storage

        if not self.repositories.check_repositories():
            self.logger.error(u'Repositories are not consistent with the sources.list')
            raise VersionsMismatchError(u'Versions mismatch, please run "weboob-config update"')

        for backend_name, module_name, params in self.backends_config.iter_backends():
            if '_enabled' in params and not params['_enabled'].lower() in ('1', 'y', 'true', 'on', 'yes') or \
               names is not None and backend_name not in names or \
               modules is not None and module_name not in modules or \
               exclude is not None and backend_name in exclude:
                continue

            minfo = self.repositories.get_module_info(module_name)
            if minfo is None:
                self.logger.warning(u'Backend "%s" is referenced in %s but was not found. '
                                    u'Perhaps a missing repository or a removed module?', module_name, self.backends_config.confpath)
                continue

            if caps is not None and not minfo.has_caps(caps):
                continue

            if not minfo.is_installed():
                self.repositories.install(minfo)

            module = None
            try:
                module = self.modules_loader.get_or_load_module(module_name)
            except ModuleLoadError as e:
                self.logger.error(u'Unable to load module "%s": %s', module_name, e)
                continue

            if backend_name in self.backend_instances:
                self.logger.warning(u'Oops, the backend "%s" is already loaded. Unload it before reloading...', backend_name)
                self.unload_backends(backend_name)

            try:
                backend_instance = module.create_instance(self, backend_name, params, storage)
            except Module.ConfigError as e:
                if errors is not None:
                    errors.append(self.LoadError(backend_name, e))
            else:
                self.backend_instances[backend_name] = loaded[backend_name] = backend_instance
        return loaded

    def load_or_install_module(self, module_name):
        """ Load a backend, and install it if not done before """
        try:
            return self.modules_loader.get_or_load_module(module_name)
        except ModuleLoadError:
            self.repositories.install(module_name)
            return self.modules_loader.get_or_load_module(module_name)
Exemplo n.º 4
0
class Weboob(object):
    """
    The main class of Weboob, used to manage backends and call methods.

    :param workdir: optional parameter to set path of the working directory
    :type workdir: str
    :param backends_filename: name of the *backends* file, where configuration of
                              backends is stored
    :type backends_filename: str
    :param scheduler: what scheduler to use; default is :class:`weboob.core.scheduler.Scheduler`
    :type scheduler: :class:`weboob.core.scheduler.IScheduler`
    :param storage: provide a storage where backends can save data
    :type storage: :class:`weboob.tools.storage.IStorage`
    """
    VERSION = '0.e'
    BACKENDS_FILENAME = 'backends'

    def __init__(self, workdir=None, backends_filename=None, scheduler=None, storage=None):
        self.logger = getLogger('weboob')
        self.backend_instances = {}
        self.callbacks = {'login':   lambda backend_name, value: None,
                          'captcha': lambda backend_name, image: None,
                         }

        # Scheduler
        if scheduler is None:
            scheduler = Scheduler()
        self.scheduler = scheduler

        # Create WORKDIR
        if workdir is not None:
            datadir = workdir
        elif 'WEBOOB_WORKDIR' in os.environ:
            datadir = workdir = os.environ.get('WEBOOB_WORKDIR')
        else:
            old_workdir = os.path.join(os.path.expanduser('~'), '.weboob')
            xdg_config_home = os.path.join(os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')), 'weboob')
            xdg_data_home = os.path.join(os.environ.get('XDG_DATA_HOME', os.path.join(os.path.expanduser('~'), '.local', 'share')), 'weboob')

            if os.path.isdir(old_workdir):
                self.logger.warning('You are using "%s" as working directory. Files are moved into %s and %s.'
                                    % (old_workdir, xdg_config_home, xdg_data_home))
                self._create_dir(xdg_config_home)
                self._create_dir(xdg_data_home)
                for f in os.listdir(old_workdir):
                    if f in Repositories.SHARE_DIRS:
                        dest = xdg_data_home
                    else:
                        dest = xdg_config_home
                    shutil.move(os.path.join(old_workdir, f), dest)
                shutil.rmtree(old_workdir)
            workdir = xdg_config_home
            datadir = xdg_data_home

        self.workdir = os.path.realpath(workdir)
        self._create_dir(workdir)

        # Repositories management
        self.repositories = Repositories(workdir, datadir, self.VERSION)

        # Backends loader
        self.modules_loader = ModulesLoader(self.repositories)

        # Backend instances config
        if not backends_filename:
            backends_filename = os.environ.get('WEBOOB_BACKENDS', os.path.join(self.workdir, self.BACKENDS_FILENAME))
        elif not backends_filename.startswith('/'):
            backends_filename = os.path.join(self.workdir, backends_filename)
        self.backends_config = BackendsConfig(backends_filename)

        # Storage
        self.storage = storage

    def _create_dir(self, name):
        if not os.path.exists(name):
            os.makedirs(name)
        elif not os.path.isdir(name):
            self.logger.error(u'"%s" is not a directory' % name)

    def __deinit__(self):
        self.deinit()

    def deinit(self):
        """
        Call this method when you stop using Weboob, to
        properly unload all correctly.
        """
        self.unload_backends()

    def update(self, progress=IProgress()):
        """
        Update modules from repositories.
        """
        self.repositories.update(progress)

        modules_to_check = set([module_name for name, module_name, params in self.backends_config.iter_backends()])
        for module_name in modules_to_check:
            minfo = self.repositories.get_module_info(module_name)
            if minfo and not minfo.is_installed():
                self.repositories.install(minfo, progress)

    class LoadError(Exception):
        """
        Raised when a backend is unabled to load.

        :param backend_name: name of backend we can't load
        :param exception: exception object
        """
        def __init__(self, backend_name, exception):
            Exception.__init__(self, unicode(exception))
            self.backend_name = backend_name

    def build_backend(self, module_name, params=None, storage=None):
        """
        Create a single backend which is not listed
        in configuration.

        :param module_name: name of module
        :param params: parameters to give to backend
        :type params: :class:`dict`
        :param storage: storage to use
        :type storage: :class:`weboob.tools.storage.IStorage`
        :rtype: :class:`weboob.tools.backend.BaseBackend`
        """
        minfo = self.repositories.get_module_info(module_name)
        if minfo is None:
            raise ModuleLoadError(module_name, 'Module does not exist.')

        if not minfo.is_installed():
            self.repositories.install(minfo)

        module = self.modules_loader.get_or_load_module(module_name)

        backend_instance = module.create_instance(self, module_name, params or {}, storage)
        return backend_instance

    def load_backends(self, caps=None, names=None, modules=None, storage=None, errors=None):
        """
        Load backends listed in config file.

        :param caps: load backends which implement all of specified caps
        :type caps: tuple[:class:`weboob.capabilities.base.ICapBase`]
        :param names: load backends with instance name in list
        :type names: tuple[:class:`str`]
        :param modules: load backends which module is in list
        :type modules: tuple[:class:`str`]
        :param storage: use this storage if specified
        :type storage: :class:`weboob.tools.storage.IStorage`
        :param errors: if specified, store every errors in this list
        :type errors: list[:class:`LoadError`]
        :returns: loaded backends
        :rtype: dict[:class:`str`, :class:`weboob.tools.backend.BaseBackend`]
        """
        loaded = {}
        if storage is None:
            storage = self.storage

        for instance_name, module_name, params in self.backends_config.iter_backends():
            if '_enabled' in params and not params['_enabled'].lower() in ('1', 'y', 'true', 'on', 'yes') or \
               names is not None and instance_name not in names or \
               modules is not None and module_name not in modules:
                continue

            minfo = self.repositories.get_module_info(module_name)
            if minfo is None:
                self.logger.warning(u'Backend "%s" is referenced in %s but was not found. '
                                     'Perhaps a missing repository?' % (module_name, self.backends_config.confpath))
                continue

            if caps is not None and not minfo.has_caps(caps):
                continue

            if not minfo.is_installed():
                self.repositories.install(minfo)

            module = None
            try:
                module = self.modules_loader.get_or_load_module(module_name)
            except ModuleLoadError, e:
                self.logger.error(e)
                continue

            if instance_name in self.backend_instances:
                self.logger.warning(u'Oops, the backend "%s" is already loaded. Unload it before reloading...' % instance_name)
                self.unload_backends(instance_name)

            try:
                backend_instance = module.create_instance(self, instance_name, params, storage)
            except BaseBackend.ConfigError, e:
                if errors is not None:
                    errors.append(self.LoadError(instance_name, e))
            else:
Exemplo n.º 5
0
class Weboob(object):
    """
    The main class of Weboob, used to manage backends and call methods.

    :param workdir: optional parameter to set path of the working directory
    :type workdir: str
    :param backends_filename: name of the *backends* file, where configuration of
                              backends is stored
    :type backends_filename: str
    :param scheduler: what scheduler to use; default is :class:`weboob.core.scheduler.Scheduler`
    :type scheduler: :class:`weboob.core.scheduler.IScheduler`
    :param storage: provide a storage where backends can save data
    :type storage: :class:`weboob.tools.storage.IStorage`
    """
    VERSION = '0.h'
    BACKENDS_FILENAME = 'backends'

    def __init__(self, workdir=None, backends_filename=None, scheduler=None, storage=None):
        self.logger = getLogger('weboob')
        self.backend_instances = {}
        self.callbacks = {'login':   lambda backend_name, value: None,
                          'captcha': lambda backend_name, image: None,
                         }

        # Scheduler
        if scheduler is None:
            scheduler = Scheduler()
        self.scheduler = scheduler

        # Create WORKDIR
        if workdir is not None:
            datadir = workdir
        elif 'WEBOOB_WORKDIR' in os.environ:
            datadir = workdir = os.environ.get('WEBOOB_WORKDIR')
        else:
            old_workdir = os.path.join(os.path.expanduser('~'), '.weboob')
            xdg_config_home = os.path.join(os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')), 'weboob')
            xdg_data_home = os.path.join(os.environ.get('XDG_DATA_HOME', os.path.join(os.path.expanduser('~'), '.local', 'share')), 'weboob')

            if os.path.isdir(old_workdir):
                self.logger.warning('You are using "%s" as working directory. Files are moved into %s and %s.'
                                    % (old_workdir, xdg_config_home, xdg_data_home))
                self._create_dir(xdg_config_home)
                self._create_dir(xdg_data_home)
                for f in os.listdir(old_workdir):
                    if f in Repositories.SHARE_DIRS:
                        dest = xdg_data_home
                    else:
                        dest = xdg_config_home
                    shutil.move(os.path.join(old_workdir, f), dest)
                shutil.rmtree(old_workdir)
            workdir = xdg_config_home
            datadir = xdg_data_home

        self.workdir = os.path.realpath(workdir)
        self._create_dir(workdir)

        # Repositories management
        self.repositories = Repositories(workdir, datadir, self.VERSION)
        # Backends loader
        self.modules_loader = ModulesLoader(self.repositories)

        # Backend instances config
        if not backends_filename:
            backends_filename = os.environ.get('WEBOOB_BACKENDS', os.path.join(self.workdir, self.BACKENDS_FILENAME))
        elif not backends_filename.startswith('/'):
            backends_filename = os.path.join(self.workdir, backends_filename)
        self.backends_config = BackendsConfig(backends_filename)

        # Storage
        self.storage = storage

    def _create_dir(self, name):
        if not os.path.exists(name):
            os.makedirs(name)
        elif not os.path.isdir(name):
            self.logger.error(u'"%s" is not a directory' % name)

    def __deinit__(self):
        self.deinit()

    def deinit(self):
        """
        Call this method when you stop using Weboob, to
        properly unload all correctly.
        """
        self.unload_backends()

    def update(self, progress=IProgress()):
        """
        Update modules from repositories.
        """
        self.repositories.update(progress)

        modules_to_check = set([module_name for name, module_name, params in self.backends_config.iter_backends()])
        for module_name in modules_to_check:
            minfo = self.repositories.get_module_info(module_name)
            if minfo and not minfo.is_installed():
                self.repositories.install(minfo, progress)

    class LoadError(Exception):
        """
        Raised when a backend is unabled to load.

        :param backend_name: name of backend we can't load
        :param exception: exception object
        """
        def __init__(self, backend_name, exception):
            Exception.__init__(self, unicode(exception))
            self.backend_name = backend_name

    def build_backend(self, module_name, params=None, storage=None):
        """
        Create a single backend which is not listed
        in configuration.

        :param module_name: name of module
        :param params: parameters to give to backend
        :type params: :class:`dict`
        :param storage: storage to use
        :type storage: :class:`weboob.tools.storage.IStorage`
        :rtype: :class:`weboob.tools.backend.BaseBackend`
        """
        minfo = self.repositories.get_module_info(module_name)
        if minfo is None:
            raise ModuleLoadError(module_name, 'Module does not exist.')

        if not minfo.is_installed():
            self.repositories.install(minfo)

        module = self.modules_loader.get_or_load_module(module_name)

        backend_instance = module.create_instance(self, module_name, params or {}, storage)
        return backend_instance

    def load_backends(self, caps=None, names=None, modules=None, exclude=None, storage=None, errors=None):
        """
        Load backends listed in config file.

        :param caps: load backends which implement all of specified caps
        :type caps: tuple[:class:`weboob.capabilities.base.ICapBase`]
        :param names: load backends with instance name in list
        :type names: tuple[:class:`str`]
        :param modules: load backends which module is in list
        :type modules: tuple[:class:`str`]
        :param exclude: do not load modules in list
        :type exclude: tuple[:class:`str`]
        :param storage: use this storage if specified
        :type storage: :class:`weboob.tools.storage.IStorage`
        :param errors: if specified, store every errors in this list
        :type errors: list[:class:`LoadError`]
        :returns: loaded backends
        :rtype: dict[:class:`str`, :class:`weboob.tools.backend.BaseBackend`]
        """
        loaded = {}
        if storage is None:
            storage = self.storage

        if not self.repositories.check_repositories():
            self.logger.error(u'Repositories are not consistent with the sources.list')
            raise VersionsMismatchError(u'Versions mismatch, please run "weboob-config update"')

        for instance_name, module_name, params in self.backends_config.iter_backends():
            if '_enabled' in params and not params['_enabled'].lower() in ('1', 'y', 'true', 'on', 'yes') or \
               names is not None and instance_name not in names or \
               modules is not None and module_name not in modules or \
               exclude is not None and module_name in exclude:
                continue

            minfo = self.repositories.get_module_info(module_name)
            if minfo is None:
                self.logger.warning(u'Backend "%s" is referenced in %s but was not found. '
                                     'Perhaps a missing repository?' % (module_name, self.backends_config.confpath))
                continue

            if caps is not None and not minfo.has_caps(caps):
                continue

            if not minfo.is_installed():
                self.repositories.install(minfo)

            module = None
            try:
                module = self.modules_loader.get_or_load_module(module_name)
            except ModuleLoadError as e:
                self.logger.error(u'Unable to load module "%s": %s' % (module_name, e))
                continue

            if instance_name in self.backend_instances:
                self.logger.warning(u'Oops, the backend "%s" is already loaded. Unload it before reloading...' % instance_name)
                self.unload_backends(instance_name)

            try:
                backend_instance = module.create_instance(self, instance_name, params, storage)
            except BaseBackend.ConfigError as e:
                if errors is not None:
                    errors.append(self.LoadError(instance_name, e))
            else:
                self.backend_instances[instance_name] = loaded[instance_name] = backend_instance
        return loaded

    def unload_backends(self, names=None):
        """
        Unload backends.

        :param names: if specified, only unload that backends
        :type names: :class:`list`
        """
        unloaded = {}
        if isinstance(names, basestring):
            names = [names]
        elif names is None:
            names = self.backend_instances.keys()

        for name in names:
            backend = self.backend_instances.pop(name)
            with backend:
                backend.deinit()
            unloaded[backend.name] = backend

        return unloaded

    def get_backend(self, name, **kwargs):
        """
        Get a backend from its name.

        :param name: name of backend to get
        :type name: str
        :param default: if specified, get this value when the backend is not found
        :type default: whatever you want
        :raises: :class:`KeyError` if not found.
        """
        try:
            return self.backend_instances[name]
        except KeyError:
            if 'default' in kwargs:
                return kwargs['default']
            else:
                raise

    def count_backends(self):
        """
        Get number of loaded backends.
        """
        return len(self.backend_instances)

    def iter_backends(self, caps=None):
        """
        Iter on each backends.

        Note: each backend is locked when it is returned.

        :param caps: optional list of capabilities to select backends
        :type caps: tuple[:class:`weboob.capabilities.base.IBaseCap`]
        :rtype: iter[:class:`weboob.tools.backend.BaseBackend`]
        """
        for name, backend in sorted(self.backend_instances.iteritems()):
            if caps is None or backend.has_caps(caps):
                with backend:
                    yield backend

    def do(self, function, *args, **kwargs):
        """
        Do calls on loaded backends with specified arguments, in separated
        threads.

        This function has two modes:

        - If *function* is a string, it calls the method with this name on
          each backends with the specified arguments;
        - If *function* is a callable, it calls it in a separated thread with
          the locked backend instance at first arguments, and \*args and
          \*\*kwargs.

        :param function: backend's method name, or a callable object
        :type function: :class:`str`
        :param backends: list of backends to iterate on
        :type backends: list[:class:`str`]
        :param caps: iterate on backends which implement this caps
        :type caps: list[:class:`weboob.capabilities.base.IBaseCap`]
        :rtype: A :class:`weboob.core.bcall.BackendsCall` object (iterable)
        """
        backends = self.backend_instances.values()
        _backends = kwargs.pop('backends', None)
        if _backends is not None:
            if isinstance(_backends, BaseBackend):
                backends = [_backends]
            elif isinstance(_backends, basestring):
                if len(_backends) > 0:
                    try:
                        backends = [self.backend_instances[_backends]]
                    except (ValueError, KeyError):
                        backends = []
            elif isinstance(_backends, (list, tuple, set)):
                backends = []
                for backend in _backends:
                    if isinstance(backend, basestring):
                        try:
                            backends.append(self.backend_instances[backend])
                        except (ValueError, KeyError):
                            pass
                    else:
                        backends.append(backend)
            else:
                self.logger.warning(u'The "backends" value isn\'t supported: %r' % _backends)

        if 'caps' in kwargs:
            caps = kwargs.pop('caps')
            backends = [backend for backend in backends if backend.has_caps(caps)]

        # The return value MUST BE the BackendsCall instance. Please never iterate
        # here on this object, because caller might want to use other methods, like
        # wait() on callback_thread().
        # Thanks a lot.
        return BackendsCall(backends, function, *args, **kwargs)

    def schedule(self, interval, function, *args):
        """
        Schedule an event.

        :param interval: delay before calling the function
        :type interval: int
        :param function: function to call
        :type function: callabale
        :param args: arguments to give to function
        :returns: an event identificator
        """
        return self.scheduler.schedule(interval, function, *args)

    def repeat(self, interval, function, *args):
        """
        Repeat a call to a function

        :param interval: interval between two calls
        :type interval: int
        :param function: function to call
        :type function: callable
        :param args: arguments to give to function
        :returns: an event identificator
        """
        return self.scheduler.repeat(interval, function, *args)

    def cancel(self, ev):
        """
        Cancel an event

        :param ev: the event identificator
        """
        return self.scheduler.cancel(ev)

    def want_stop(self):
        """
        Plan to stop the scheduler.
        """
        return self.scheduler.want_stop()

    def loop(self):
        """
        Run the scheduler loop
        """
        return self.scheduler.run()
Exemplo n.º 6
0
class Weboob(object):
    """
    The main class of Weboob, used to manage backends and call methods.

    :param workdir: optional parameter to set path of the working directory
    :type workdir: str
    :param backends_filename: name of the *backends* file, where configuration of
                              backends is stored
    :type backends_filename: str
    :param scheduler: what scheduler to use; default is :class:`weboob.core.scheduler.Scheduler`
    :type scheduler: :class:`weboob.core.scheduler.IScheduler`
    :param storage: provide a storage where backends can save data
    :type storage: :class:`weboob.tools.storage.IStorage`
    """
    VERSION = '0.h'
    BACKENDS_FILENAME = 'backends'

    def __init__(self,
                 workdir=None,
                 backends_filename=None,
                 scheduler=None,
                 storage=None):
        self.logger = getLogger('weboob')
        self.backend_instances = {}
        self.callbacks = {
            'login': lambda backend_name, value: None,
            'captcha': lambda backend_name, image: None,
        }

        # Scheduler
        if scheduler is None:
            scheduler = Scheduler()
        self.scheduler = scheduler

        # Create WORKDIR
        if workdir is not None:
            datadir = workdir
        elif 'WEBOOB_WORKDIR' in os.environ:
            datadir = workdir = os.environ.get('WEBOOB_WORKDIR')
        else:
            old_workdir = os.path.join(os.path.expanduser('~'), '.weboob')
            xdg_config_home = os.path.join(
                os.environ.get(
                    'XDG_CONFIG_HOME',
                    os.path.join(os.path.expanduser('~'), '.config')),
                'weboob')
            xdg_data_home = os.path.join(
                os.environ.get(
                    'XDG_DATA_HOME',
                    os.path.join(os.path.expanduser('~'), '.local', 'share')),
                'weboob')

            if os.path.isdir(old_workdir):
                self.logger.warning(
                    'You are using "%s" as working directory. Files are moved into %s and %s.'
                    % (old_workdir, xdg_config_home, xdg_data_home))
                self._create_dir(xdg_config_home)
                self._create_dir(xdg_data_home)
                for f in os.listdir(old_workdir):
                    if f in Repositories.SHARE_DIRS:
                        dest = xdg_data_home
                    else:
                        dest = xdg_config_home
                    shutil.move(os.path.join(old_workdir, f), dest)
                shutil.rmtree(old_workdir)
            workdir = xdg_config_home
            datadir = xdg_data_home

        self.workdir = os.path.realpath(workdir)
        self._create_dir(workdir)

        # Repositories management
        self.repositories = Repositories(workdir, datadir, self.VERSION)
        # Backends loader
        self.modules_loader = ModulesLoader(self.repositories)

        # Backend instances config
        if not backends_filename:
            backends_filename = os.environ.get(
                'WEBOOB_BACKENDS',
                os.path.join(self.workdir, self.BACKENDS_FILENAME))
        elif not backends_filename.startswith('/'):
            backends_filename = os.path.join(self.workdir, backends_filename)
        self.backends_config = BackendsConfig(backends_filename)

        # Storage
        self.storage = storage

    def _create_dir(self, name):
        if not os.path.exists(name):
            os.makedirs(name)
        elif not os.path.isdir(name):
            self.logger.error(u'"%s" is not a directory' % name)

    def __deinit__(self):
        self.deinit()

    def deinit(self):
        """
        Call this method when you stop using Weboob, to
        properly unload all correctly.
        """
        self.unload_backends()

    def update(self, progress=IProgress()):
        """
        Update modules from repositories.
        """
        self.repositories.update(progress)

        modules_to_check = set([
            module_name for name, module_name, params in
            self.backends_config.iter_backends()
        ])
        for module_name in modules_to_check:
            minfo = self.repositories.get_module_info(module_name)
            if minfo and not minfo.is_installed():
                self.repositories.install(minfo, progress)

    class LoadError(Exception):
        """
        Raised when a backend is unabled to load.

        :param backend_name: name of backend we can't load
        :param exception: exception object
        """
        def __init__(self, backend_name, exception):
            Exception.__init__(self, unicode(exception))
            self.backend_name = backend_name

    def build_backend(self, module_name, params=None, storage=None):
        """
        Create a single backend which is not listed
        in configuration.

        :param module_name: name of module
        :param params: parameters to give to backend
        :type params: :class:`dict`
        :param storage: storage to use
        :type storage: :class:`weboob.tools.storage.IStorage`
        :rtype: :class:`weboob.tools.backend.BaseBackend`
        """
        minfo = self.repositories.get_module_info(module_name)
        if minfo is None:
            raise ModuleLoadError(module_name, 'Module does not exist.')

        if not minfo.is_installed():
            self.repositories.install(minfo)

        module = self.modules_loader.get_or_load_module(module_name)

        backend_instance = module.create_instance(self, module_name, params
                                                  or {}, storage)
        return backend_instance

    def load_backends(self,
                      caps=None,
                      names=None,
                      modules=None,
                      exclude=None,
                      storage=None,
                      errors=None):
        """
        Load backends listed in config file.

        :param caps: load backends which implement all of specified caps
        :type caps: tuple[:class:`weboob.capabilities.base.ICapBase`]
        :param names: load backends with instance name in list
        :type names: tuple[:class:`str`]
        :param modules: load backends which module is in list
        :type modules: tuple[:class:`str`]
        :param exclude: do not load modules in list
        :type exclude: tuple[:class:`str`]
        :param storage: use this storage if specified
        :type storage: :class:`weboob.tools.storage.IStorage`
        :param errors: if specified, store every errors in this list
        :type errors: list[:class:`LoadError`]
        :returns: loaded backends
        :rtype: dict[:class:`str`, :class:`weboob.tools.backend.BaseBackend`]
        """
        loaded = {}
        if storage is None:
            storage = self.storage

        if not self.repositories.check_repositories():
            self.logger.error(
                u'Repositories are not consistent with the sources.list')
            raise VersionsMismatchError(
                u'Versions mismatch, please run "weboob-config update"')

        for instance_name, module_name, params in self.backends_config.iter_backends(
        ):
            if '_enabled' in params and not params['_enabled'].lower() in ('1', 'y', 'true', 'on', 'yes') or \
               names is not None and instance_name not in names or \
               modules is not None and module_name not in modules or \
               exclude is not None and module_name in exclude:
                continue

            minfo = self.repositories.get_module_info(module_name)
            if minfo is None:
                self.logger.warning(
                    u'Backend "%s" is referenced in %s but was not found. '
                    'Perhaps a missing repository?' %
                    (module_name, self.backends_config.confpath))
                continue

            if caps is not None and not minfo.has_caps(caps):
                continue

            if not minfo.is_installed():
                self.repositories.install(minfo)

            module = None
            try:
                module = self.modules_loader.get_or_load_module(module_name)
            except ModuleLoadError as e:
                self.logger.error(u'Unable to load module "%s": %s' %
                                  (module_name, e))
                continue

            if instance_name in self.backend_instances:
                self.logger.warning(
                    u'Oops, the backend "%s" is already loaded. Unload it before reloading...'
                    % instance_name)
                self.unload_backends(instance_name)

            try:
                backend_instance = module.create_instance(
                    self, instance_name, params, storage)
            except BaseBackend.ConfigError as e:
                if errors is not None:
                    errors.append(self.LoadError(instance_name, e))
            else:
                self.backend_instances[instance_name] = loaded[
                    instance_name] = backend_instance
        return loaded

    def unload_backends(self, names=None):
        """
        Unload backends.

        :param names: if specified, only unload that backends
        :type names: :class:`list`
        """
        unloaded = {}
        if isinstance(names, basestring):
            names = [names]
        elif names is None:
            names = self.backend_instances.keys()

        for name in names:
            backend = self.backend_instances.pop(name)
            with backend:
                backend.deinit()
            unloaded[backend.name] = backend

        return unloaded

    def get_backend(self, name, **kwargs):
        """
        Get a backend from its name.

        :param name: name of backend to get
        :type name: str
        :param default: if specified, get this value when the backend is not found
        :type default: whatever you want
        :raises: :class:`KeyError` if not found.
        """
        try:
            return self.backend_instances[name]
        except KeyError:
            if 'default' in kwargs:
                return kwargs['default']
            else:
                raise

    def count_backends(self):
        """
        Get number of loaded backends.
        """
        return len(self.backend_instances)

    def iter_backends(self, caps=None):
        """
        Iter on each backends.

        Note: each backend is locked when it is returned.

        :param caps: optional list of capabilities to select backends
        :type caps: tuple[:class:`weboob.capabilities.base.IBaseCap`]
        :rtype: iter[:class:`weboob.tools.backend.BaseBackend`]
        """
        for name, backend in sorted(self.backend_instances.iteritems()):
            if caps is None or backend.has_caps(caps):
                with backend:
                    yield backend

    def do(self, function, *args, **kwargs):
        """
        Do calls on loaded backends with specified arguments, in separated
        threads.

        This function has two modes:

        - If *function* is a string, it calls the method with this name on
          each backends with the specified arguments;
        - If *function* is a callable, it calls it in a separated thread with
          the locked backend instance at first arguments, and \*args and
          \*\*kwargs.

        :param function: backend's method name, or a callable object
        :type function: :class:`str`
        :param backends: list of backends to iterate on
        :type backends: list[:class:`str`]
        :param caps: iterate on backends which implement this caps
        :type caps: list[:class:`weboob.capabilities.base.IBaseCap`]
        :rtype: A :class:`weboob.core.bcall.BackendsCall` object (iterable)
        """
        backends = self.backend_instances.values()
        _backends = kwargs.pop('backends', None)
        if _backends is not None:
            if isinstance(_backends, BaseBackend):
                backends = [_backends]
            elif isinstance(_backends, basestring):
                if len(_backends) > 0:
                    try:
                        backends = [self.backend_instances[_backends]]
                    except (ValueError, KeyError):
                        backends = []
            elif isinstance(_backends, (list, tuple, set)):
                backends = []
                for backend in _backends:
                    if isinstance(backend, basestring):
                        try:
                            backends.append(self.backend_instances[backend])
                        except (ValueError, KeyError):
                            pass
                    else:
                        backends.append(backend)
            else:
                self.logger.warning(
                    u'The "backends" value isn\'t supported: %r' % _backends)

        if 'caps' in kwargs:
            caps = kwargs.pop('caps')
            backends = [
                backend for backend in backends if backend.has_caps(caps)
            ]

        # The return value MUST BE the BackendsCall instance. Please never iterate
        # here on this object, because caller might want to use other methods, like
        # wait() on callback_thread().
        # Thanks a lot.
        return BackendsCall(backends, function, *args, **kwargs)

    def schedule(self, interval, function, *args):
        """
        Schedule an event.

        :param interval: delay before calling the function
        :type interval: int
        :param function: function to call
        :type function: callabale
        :param args: arguments to give to function
        :returns: an event identificator
        """
        return self.scheduler.schedule(interval, function, *args)

    def repeat(self, interval, function, *args):
        """
        Repeat a call to a function

        :param interval: interval between two calls
        :type interval: int
        :param function: function to call
        :type function: callable
        :param args: arguments to give to function
        :returns: an event identificator
        """
        return self.scheduler.repeat(interval, function, *args)

    def cancel(self, ev):
        """
        Cancel an event

        :param ev: the event identificator
        """
        return self.scheduler.cancel(ev)

    def want_stop(self):
        """
        Plan to stop the scheduler.
        """
        return self.scheduler.want_stop()

    def loop(self):
        """
        Run the scheduler loop
        """
        return self.scheduler.run()