Example #1
0
    def parse_args(self):
        parser = ArgumentParser()
        parser.add_argument('base_path')
        parser.add_argument('-d',
                            '--debug',
                            action='store_true',
                            default=False)
        parser.add_argument('-p',
                            '--progress',
                            action='store_true',
                            default=False)

        parser.add_argument('-c', '--collection', action='append')
        parser.add_argument('-f', '--format', action='append')
        parser.add_argument('-s', '--source', action='append', required=True)

        # Discover installed plugins
        PluginManager.discover()

        # Add arguments from updater sources
        for _, source in PluginManager.list_ordered('database-updater'):
            for argument in source.__parameters__:
                if 'name' not in argument:
                    log.warn('Invalid source argument: %r', argument)
                    continue

                parser.add_argument(
                    '--%s-%s' % (source.__key__, argument['name']),
                    **argument.get('kwargs', {}))

        # Parse command line arguments
        return parser.parse_args()
Example #2
0
    def __init__(self, sources, collections=None, formats=None):
        self.collections = collections

        # Discover installed plugins
        PluginManager.discover()

        # Retrieve formats
        self.formats = dict(
            self._load_plugins('format', formats or DEFAULT_FORMATS))

        # Retrieve sources
        self.sources = dict(
            self._load_plugins('database-updater', sources, construct=False))
Example #3
0
    def _construct_storage(self, storage_or_key):
        if isinstance(storage_or_key, ProviderStorage):
            # Use provided source
            storage = storage_or_key
        elif PluginManager.has('storage', storage_or_key):
            # Construct source by key
            storage = PluginManager.get('storage', storage_or_key)()
        else:
            raise ValueError('Unknown storage interface: %r' % storage_or_key)

        # Initialize source
        storage.initialize(self._client)
        return storage
    def __init__(self, sources, collections=None, formats=None):
        self.collections = collections

        # Discover installed plugins
        PluginManager.discover()

        # Retrieve formats
        self.formats = dict(self._load_plugins(
            'format', formats or DEFAULT_FORMATS
        ))

        # Retrieve sources
        self.sources = dict(self._load_plugins(
            'database-updater', sources,
            construct=False
        ))
    def initialize(self, client):
        super(ReleaseProvider, self).initialize(client)

        # Construct format
        cls = PluginManager.get('format', self._fmt)

        if cls is None:
            raise ValueError('Unable to find format: %r' % self._fmt)

        self.format = cls()
    def parse_args(self):
        parser = ArgumentParser()
        parser.add_argument('base_path')
        parser.add_argument('-d', '--debug', action='store_true', default=False)
        parser.add_argument('-p', '--progress', action='store_true', default=False)

        parser.add_argument('-c', '--collection', action='append')
        parser.add_argument('-f', '--format', action='append')
        parser.add_argument('-s', '--source', action='append', required=True)

        # Discover installed plugins
        PluginManager.discover()

        # Add arguments from updater sources
        for _, source in PluginManager.list_ordered('database-updater'):
            for argument in source.__parameters__:
                if 'name' not in argument:
                    log.warn('Invalid source argument: %r', argument)
                    continue

                parser.add_argument('--%s-%s' % (source.__key__, argument['name']), **argument.get('kwargs', {}))

        # Parse command line arguments
        return parser.parse_args()
Example #7
0
    def _load_plugins(kind, keys, construct=True):
        if not keys:
            return

        for name in keys:
            cls = PluginManager.get(kind, name)

            if cls is None:
                log.warn('Unable to find plugin: %r', name)
                continue

            if not cls.available:
                log.warn('Plugin %r is not available', name)
                continue

            if construct:
                yield cls.__key__, cls()
            else:
                yield cls.__key__, cls
Example #8
0
    def _load_plugins(kind, keys, construct=True):
        if not keys:
            return

        for name in keys:
            cls = PluginManager.get(kind, name)

            if cls is None:
                log.warn('Unable to find plugin: %r', name)
                continue

            if not cls.available:
                log.warn('Plugin %r is not available', name)
                continue

            if construct:
                yield cls.__key__, cls()
            else:
                yield cls.__key__, cls
Example #9
0
    def _construct_provider(self, provider_or_key):
        if isinstance(provider_or_key, Provider):
            # Class
            provider = provider_or_key
        elif isinstance(provider_or_key, six.string_types):
            # Identifier
            provider = PluginManager.get('client-provider', provider_or_key)

            if provider is None:
                raise ValueError('Unable to find provider: %r' % provider_or_key)
        else:
            raise ValueError('Unknown provider: %r' % provider_or_key)

        # Ensure provider has been constructed
        if inspect.isclass(provider):
            provider = provider()

        # Initialize provider
        provider.initialize(self)
        return provider
Example #10
0
    def _construct_provider(self, provider_or_key):
        if isinstance(provider_or_key, Provider):
            # Class
            provider = provider_or_key
        elif isinstance(provider_or_key, six.string_types):
            # Identifier
            provider = PluginManager.get('client-provider', provider_or_key)

            if provider is None:
                raise ValueError('Unable to find provider: %r' %
                                 provider_or_key)
        else:
            raise ValueError('Unknown provider: %r' % provider_or_key)

        # Ensure provider has been constructed
        if inspect.isclass(provider):
            provider = provider()

        # Initialize provider
        provider.initialize(self)
        return provider
    def run(self, base_path, **kwargs):
        if not os.path.exists(base_path):
            log.error('Path %r doesn\'t exist', base_path)
            return False

        for _, cls in self.sources.items():
            # Retrieve collections
            collections = cls.__collections__

            # Filter `collections` by provided list
            if self.collections is not None:
                collections = [
                    key for key in collections
                    if key in self.collections
                ]

            # Run updater on each collection
            for source, target in collections:
                # Build collection path
                database_path = os.path.join(
                    base_path,
                    self._build_package_name(source, target),
                    self._build_module_name(source, target)
                )

                # Ensure database exists
                if not os.path.exists(database_path):
                    log.warn('Unknown collection: %r -> %r', source, target)
                    continue

                # Run updater on database for each format
                for fmt in self.formats.itervalues():
                    if fmt.__construct__ is False:
                        continue

                    # Build updater client
                    client = UpdaterClient(fmt)

                    # Build database storage interface
                    storage = PluginManager.get('storage', 'file/database')(
                        self, source, target,
                        path=database_path
                    )

                    storage.initialize(client)

                    # Load database
                    database = Database.load(storage, source, target)

                    # Load database collections
                    database.load_collections([
                        (source, target),
                        (target, source)
                    ])

                    for collection in database.collections.itervalues():
                        # Run updater on collection
                        try:
                            s = cls(collection, kwargs)

                            if not s.run():
                                return False
                        except Exception as ex:
                            log.warn(
                                'Unable to run updater on %r (format: %r) - %s',
                                database_path, fmt, ex,
                                exc_info=True
                            )
                            return False

                        # Write collection to disk
                        try:
                            collection.write()
                        except Exception as ex:
                            log.warn('Unable to write collection to disk - %s', ex, exc_info=True)
                            return False

        return True
Example #12
0
    def run(self, base_path, **kwargs):
        if not os.path.exists(base_path):
            log.error('Path %r doesn\'t exist', base_path)
            return False

        for _, cls in self.sources.items():
            # Retrieve collections
            collections = cls.__collections__

            # Filter `collections` by provided list
            if self.collections is not None:
                collections = [
                    key for key in collections if key in self.collections
                ]

            # Run updater on each collection
            for source, target in collections:
                # Build collection path
                database_path = os.path.join(
                    base_path, self._build_package_name(source, target),
                    self._build_module_name(source, target))

                # Ensure database exists
                if not os.path.exists(database_path):
                    log.warn('Unknown collection: %r -> %r', source, target)
                    continue

                # Run updater on database for each format
                for fmt in self.formats.itervalues():
                    if fmt.__construct__ is False:
                        continue

                    # Build updater client
                    client = UpdaterClient(fmt)

                    # Build database storage interface
                    storage = PluginManager.get('storage', 'file/database')(
                        self, source, target, path=database_path)

                    storage.initialize(client)

                    # Load database
                    database = Database.load(storage, source, target)

                    # Load database collections
                    database.load_collections([(source, target),
                                               (target, source)])

                    for collection in database.collections.itervalues():
                        # Run updater on collection
                        try:
                            s = cls(collection, kwargs)

                            if not s.run():
                                return False
                        except Exception as ex:
                            log.warn(
                                'Unable to run updater on %r (format: %r) - %s',
                                database_path,
                                fmt,
                                ex,
                                exc_info=True)
                            return False

                        # Write collection to disk
                        try:
                            collection.write()
                        except Exception as ex:
                            log.warn('Unable to write collection to disk - %s',
                                     ex,
                                     exc_info=True)
                            return False

        return True