Пример #1
0
    def __init__(self, core, datastore):
        Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
        Bcfg2.Server.Plugin.StructureValidator.__init__(self)
        Bcfg2.Server.Plugin.Generator.__init__(self)
        Bcfg2.Server.Plugin.Connector.__init__(self)
        Bcfg2.Server.Plugin.Probing.__init__(self)

        self.sentinels = set()
        self.cachepath = os.path.join(self.data, 'cache')
        self.keypath = os.path.join(self.data, 'keys')
        if not os.path.exists(self.keypath):
            # create key directory if needed
            os.makedirs(self.keypath)

        # set up config files
        self.config = PackagesConfig(self)
        self.sources = PackagesSources(os.path.join(self.data, "sources.xml"),
                                       self.cachepath, core.fam, self,
                                       self.config)
Пример #2
0
    def __init__(self, core, datastore):
        Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
        Bcfg2.Server.Plugin.StructureValidator.__init__(self)
        Bcfg2.Server.Plugin.Generator.__init__(self)
        Bcfg2.Server.Plugin.Connector.__init__(self)
        Bcfg2.Server.Plugin.Probing.__init__(self)

        self.sentinels = set()
        self.cachepath = os.path.join(self.data, 'cache')
        self.keypath = os.path.join(self.data, 'keys')
        if not os.path.exists(self.keypath):
            # create key directory if needed
            os.makedirs(self.keypath)

        # set up config files
        self.config = PackagesConfig(self)
        self.sources = PackagesSources(os.path.join(self.data, "sources.xml"),
                                       self.cachepath, core.fam, self,
                                       self.config)
Пример #3
0
class Packages(Bcfg2.Server.Plugin.Plugin,
               Bcfg2.Server.Plugin.StructureValidator,
               Bcfg2.Server.Plugin.Generator,
               Bcfg2.Server.Plugin.Connector):
    name = 'Packages'
    conflicts = ['Pkgmgr']
    experimental = True
    __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload']

    def __init__(self, core, datastore):
        Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
        Bcfg2.Server.Plugin.StructureValidator.__init__(self)
        Bcfg2.Server.Plugin.Generator.__init__(self)
        Bcfg2.Server.Plugin.Connector.__init__(self)
        Bcfg2.Server.Plugin.Probing.__init__(self)

        self.sentinels = set()
        self.cachepath = os.path.join(self.data, 'cache')
        self.keypath = os.path.join(self.data, 'keys')
        if not os.path.exists(self.keypath):
            # create key directory if needed
            os.makedirs(self.keypath)

        # set up config files
        self.config = PackagesConfig(os.path.join(self.data, "packages.conf"),
                                     core.fam, self)
        self.sources = PackagesSources(os.path.join(self.data, "sources.xml"),
                                       self.cachepath, core.fam, self,
                                       self.config)

    @property
    def disableResolver(self):
        try:
            return self.config.get("global", "resolver").lower() == "disabled"
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            return False

    @property
    def disableMetaData(self):
        try:
            return self.config.get("global", "metadata").lower() == "disabled"
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            return False

    def create_config(self, entry, metadata):
        """ create yum/apt config for the specified host """
        attrib = {'encoding': 'ascii',
                  'owner': 'root',
                  'group': 'root',
                  'type': 'file',
                  'perms': '0644'}

        collection = Collection.factory(metadata, self.sources, self.data)
        entry.text = collection.get_config()
        for (key, value) in list(attrib.items()):
            entry.attrib.__setitem__(key, value)

    def HandleEntry(self, entry, metadata):
        if entry.tag == 'Package':
            collection = Collection.factory(metadata, self.sources, self.data)
            entry.set('version', 'auto')
            entry.set('type', collection.ptype)
        elif entry.tag == 'Path':
            if (self.config.has_section("global") and
                ((self.config.has_option("global", "yum_config") and
                  entry.get("name") == self.config.get("global",
                                                       "yum_config")) or
                 (self.config.has_option("global", "apt_config") and 
                  entry.get("name") == self.config.get("global",
                                                       "apt_config")))):
                self.create_config(entry, metadata)

    def HandlesEntry(self, entry, metadata):
        if entry.tag == 'Package':
            collection = Collection.factory(metadata, self.sources, self.data)
            if collection.magic_groups_match():
                return True
        elif entry.tag == 'Path':
            # managed entries for yum/apt configs
            if ((self.config.has_option("global", "yum_config") and
                 entry.get("name") == self.config.get("global",
                                                      "yum_config")) or
                (self.config.has_option("global", "apt_config") and 
                 entry.get("name") == self.config.get("global", "apt_config"))):
                return True
        return False

    def validate_structures(self, metadata, structures):
        '''Ensure client configurations include all needed prerequisites

        Arguments:
        metadata - client metadata instance
        structures - a list of structure-stage entry combinations
        '''
        collection = Collection.factory(metadata, self.sources, self.data)
        indep = lxml.etree.Element('Independent')
        self._build_packages(metadata, indep, structures,
                             collection=collection)
        collection.build_extra_structures(indep)
        structures.append(indep)

    def _build_packages(self, metadata, independent, structures,
                        collection=None):
        """ build list of packages that need to be included in the
        specification by validate_structures() """
        if self.disableResolver:
            # Config requests no resolver
            return

        if collection is None:
            collection = Collection.factory(metadata, self.sources, self.data)
        # initial is the set of packages that are explicitly specified
        # in the configuration
        initial = set()
        # base is the set of initial packages with groups expanded
        base = set()
        to_remove = []
        for struct in structures:
            for pkg in struct.xpath('//Package | //BoundPackage'):
                if pkg.get("name"):
                    initial.add(pkg.get("name"))
                elif pkg.get("group"):
                    try:
                        if pkg.get("type"):
                            gpkgs = collection.get_group(pkg.get("group"),
                                                         ptype=pkg.get("type"))
                        else:
                            gpkgs = collection.get_group(pkg.get("group"))
                        base.update(gpkgs)
                    except TypeError:
                        raise
                        self.logger.error("Could not resolve group %s" %
                                          pkg.get("group"))
                    to_remove.append(pkg)
                else:
                    self.logger.error("Packages: Malformed Package: %s" %
                                      lxml.etree.tostring(pkg))
        base.update(initial)
        for el in to_remove:
            el.getparent().remove(el)

        packages, unknown = collection.complete(base)
        if unknown:
            self.logger.info("Packages: Got %d unknown entries" % len(unknown))
            self.logger.info("Packages: %s" % list(unknown))
        newpkgs = list(packages.difference(initial))
        self.logger.debug("Packages: %d initial, %d complete, %d new" %
                          (len(initial), len(packages), len(newpkgs)))
        newpkgs.sort()
        for pkg in newpkgs:
            lxml.etree.SubElement(independent, 'BoundPackage', name=pkg,
                                  version='auto', type=collection.ptype,
                                  origin='Packages')

    def Refresh(self):
        '''Packages.Refresh() => True|False\nReload configuration
        specification and download sources\n'''
        self._load_config(force_update=True)
        return True

    def Reload(self):
        '''Packages.Refresh() => True|False\nReload configuration
        specification and sources\n'''
        self._load_config()
        return True

    def _load_config(self, force_update=False):
        '''
        Load the configuration data and setup sources

        Keyword args:
            force_update    Force downloading repo data
        '''
        self._load_sources(force_update)
        self._load_gpg_keys(force_update)

    def _load_sources(self, force_update):
        """ Load sources from the config """
        self.sentinels = set()
        cachefiles = set()

        for collection in list(Collection.collections.values()):
            cachefiles.update(collection.cachefiles)
            if not self.disableMetaData:
                collection.setup_data(force_update)
            self.sentinels.update(collection.basegroups)

        Collection.clear_cache()

        for source in self.sources:
            cachefiles.add(source.cachefile)
            if not self.disableMetaData:
                source.setup_data(force_update)

        for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")):
            if cfile not in cachefiles:
                try:
                    if os.path.isdir(cfile):
                        shutil.rmtree(cfile)
                    else:
                        os.unlink(cfile)
                except OSError:
                    err = sys.exc_info()[1]
                    logger.error("Packages: Could not remove cache file %s: %s"
                                 % (cfile, err))

    def _load_gpg_keys(self, force_update):
        """ Load gpg keys from the config """
        keyfiles = []
        keys = []
        for source in self.sources:
            for key in source.gpgkeys:
                localfile = os.path.join(self.keypath, os.path.basename(key))
                if localfile not in keyfiles:
                    keyfiles.append(localfile)
                if ((force_update and key not in keys) or
                    not os.path.exists(localfile)):
                    self.logger.info("Packages: Downloading and parsing %s" % key)
                    response = urlopen(key)
                    open(localfile, 'w').write(response.read())
                    keys.append(key)

        for kfile in glob.glob(os.path.join(self.keypath, "*")):
            if kfile not in keyfiles:
                os.unlink(kfile)

    def get_additional_data(self, metadata):
        collection = Collection.factory(metadata, self.sources, self.data)
        return dict(sources=collection.get_additional_data())
Пример #4
0
class Packages(Bcfg2.Server.Plugin.Plugin,
               Bcfg2.Server.Plugin.StructureValidator,
               Bcfg2.Server.Plugin.Generator,
               Bcfg2.Server.Plugin.Connector):
    name = 'Packages'
    conflicts = ['Pkgmgr']
    experimental = True
    __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload']

    def __init__(self, core, datastore):
        Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
        Bcfg2.Server.Plugin.StructureValidator.__init__(self)
        Bcfg2.Server.Plugin.Generator.__init__(self)
        Bcfg2.Server.Plugin.Connector.__init__(self)
        Bcfg2.Server.Plugin.Probing.__init__(self)

        self.sentinels = set()
        self.cachepath = os.path.join(self.data, 'cache')
        self.keypath = os.path.join(self.data, 'keys')
        if not os.path.exists(self.keypath):
            # create key directory if needed
            os.makedirs(self.keypath)

        # set up config files
        self.config = PackagesConfig(self)
        self.sources = PackagesSources(os.path.join(self.data, "sources.xml"),
                                       self.cachepath, core.fam, self,
                                       self.config)

    def toggle_debug(self):
        Bcfg2.Server.Plugin.Plugin.toggle_debug(self)
        self.sources.toggle_debug()

    @property
    def disableResolver(self):
        try:
            return not self.config.getboolean("global", "resolver")
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            return False
        except ValueError:
            # for historical reasons we also accept "enabled" and
            # "disabled", which are not handled according to the
            # Python docs but appear to be handled properly by
            # ConfigParser in at least some versions
            return self.config.get("global", "resolver",
                                   default="enabled").lower() == "disabled"

    @property
    def disableMetaData(self):
        try:
            return not self.config.getboolean("global", "resolver")
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            return False
        except ValueError:
            # for historical reasons we also accept "enabled" and
            # "disabled"
            return self.config.get("global", "metadata",
                                   default="enabled").lower() == "disabled"

    def create_config(self, entry, metadata):
        """ create yum/apt config for the specified host """
        attrib = {'encoding': 'ascii',
                  'owner': 'root',
                  'group': 'root',
                  'type': 'file',
                  'perms': '0644'}

        collection = self._get_collection(metadata)
        entry.text = collection.get_config()
        for (key, value) in list(attrib.items()):
            entry.attrib.__setitem__(key, value)

    def HandleEntry(self, entry, metadata):
        if entry.tag == 'Package':
            collection = self._get_collection(metadata)
            entry.set('version', 'auto')
            entry.set('version', self.config.get("global",
                                                 "version",
                                                 default="auto"))
            entry.set('type', collection.ptype)
        elif entry.tag == 'Path':
            if (entry.get("name") == self.config.get("global", "yum_config",
                                                     default="") or
                entry.get("name") == self.config.get("global", "apt_config",
                                                     default="")):
                self.create_config(entry, metadata)

    def HandlesEntry(self, entry, metadata):
        if entry.tag == 'Package':
            if self.config.getboolean("global", "magic_groups",
                                      default=True) == True:
                collection = self._get_collection(metadata)
                if collection.magic_groups_match():
                    return True
            else:
                return True
        elif entry.tag == 'Path':
            # managed entries for yum/apt configs
            if (entry.get("name") == self.config.get("global", "yum_config",
                                                     default="") or
                entry.get("name") == self.config.get("global", "apt_config",
                                                     default="")):
                return True
        return False

    def validate_structures(self, metadata, structures):
        '''Ensure client configurations include all needed prerequisites

        Arguments:
        metadata - client metadata instance
        structures - a list of structure-stage entry combinations
        '''
        collection = self._get_collection(metadata)
        indep = lxml.etree.Element('Independent')
        self._build_packages(metadata, indep, structures,
                             collection=collection)
        collection.build_extra_structures(indep)
        structures.append(indep)

    def _build_packages(self, metadata, independent, structures,
                        collection=None):
        """ build list of packages that need to be included in the
        specification by validate_structures() """
        if self.disableResolver:
            # Config requests no resolver
            return

        if collection is None:
            collection = self._get_collection(metadata)
        # initial is the set of packages that are explicitly specified
        # in the configuration
        initial = set()
        # base is the set of initial packages with groups expanded
        base = set()
        # essential pkgs are those marked as such by the distribution
        essential = collection.get_essential()
        to_remove = []
        for struct in structures:
            for pkg in struct.xpath('//Package | //BoundPackage'):
                if pkg.get("name"):
                    initial.add(pkg.get("name"))
                elif pkg.get("group"):
                    try:
                        if pkg.get("type"):
                            gpkgs = collection.get_group(pkg.get("group"),
                                                         ptype=pkg.get("type"))
                        else:
                            gpkgs = collection.get_group(pkg.get("group"))
                        base.update(gpkgs)
                    except TypeError:
                        raise
                        self.logger.error("Could not resolve group %s" %
                                          pkg.get("group"))
                    to_remove.append(pkg)
                else:
                    self.logger.error("Packages: Malformed Package: %s" %
                                      lxml.etree.tostring(pkg))
        base.update(initial | essential)
        for el in to_remove:
            el.getparent().remove(el)

        packages, unknown = collection.complete(base)
        if unknown:
            self.logger.info("Packages: Got %d unknown entries" % len(unknown))
            self.logger.info("Packages: %s" % list(unknown))
        newpkgs = list(packages.difference(initial))
        self.debug_log("Packages: %d initial, %d complete, %d new" %
                       (len(initial), len(packages), len(newpkgs)))
        newpkgs.sort()
        for pkg in newpkgs:
            lxml.etree.SubElement(independent, 'BoundPackage', name=pkg,
                                  version=self.config.get("global", "version",
                                                          default="auto"),
                                  type=collection.ptype, origin='Packages')

    def Refresh(self):
        '''Packages.Refresh() => True|False\nReload configuration
        specification and download sources\n'''
        self._load_config(force_update=True)
        return True

    def Reload(self):
        '''Packages.Refresh() => True|False\nReload configuration
        specification and sources\n'''
        self._load_config()
        return True

    def _load_config(self, force_update=False):
        '''
        Load the configuration data and setup sources

        Keyword args:
            force_update    Force downloading repo data
        '''
        self._load_sources(force_update)
        self._load_gpg_keys(force_update)

    def _load_sources(self, force_update):
        """ Load sources from the config """
        self.sentinels = set()
        cachefiles = set()

        for collection in list(Collection.collections.values()):
            cachefiles.update(collection.cachefiles)
            if not self.disableMetaData:
                collection.setup_data(force_update)
            self.sentinels.update(collection.basegroups)

        Collection.clear_cache()

        for source in self.sources:
            cachefiles.add(source.cachefile)
            if not self.disableMetaData:
                source.setup_data(force_update)

        for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")):
            if cfile not in cachefiles:
                try:
                    if os.path.isdir(cfile):
                        shutil.rmtree(cfile)
                    else:
                        os.unlink(cfile)
                except OSError:
                    err = sys.exc_info()[1]
                    self.logger.error("Packages: Could not remove cache file "
                                      "%s: %s" % (cfile, err))

    def _load_gpg_keys(self, force_update):
        """ Load gpg keys from the config """
        keyfiles = []
        keys = []
        for source in self.sources:
            for key in source.gpgkeys:
                localfile = os.path.join(self.keypath,
                                         os.path.basename(key.rstrip("/")))
                if localfile not in keyfiles:
                    keyfiles.append(localfile)
                if ((force_update and key not in keys) or
                    not os.path.exists(localfile)):
                    self.logger.info("Packages: Downloading and parsing %s" % key)
                    response = urlopen(key)
                    open(localfile, 'w').write(response.read())
                    keys.append(key)

        for kfile in glob.glob(os.path.join(self.keypath, "*")):
            if kfile not in keyfiles:
                os.unlink(kfile)

    def _get_collection(self, metadata):
        return Collection.factory(metadata, self.sources, self.data,
                                  debug=self.debug_flag)

    def get_additional_data(self, metadata):
        collection = self._get_collection(metadata)
        return dict(sources=collection.get_additional_data())
Пример #5
0
class Packages(
    Bcfg2.Server.Plugin.Plugin,
    Bcfg2.Server.Plugin.StructureValidator,
    Bcfg2.Server.Plugin.Generator,
    Bcfg2.Server.Plugin.Connector,
):
    name = "Packages"
    conflicts = ["Pkgmgr"]
    experimental = True
    __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ["Refresh", "Reload"]

    def __init__(self, core, datastore):
        Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
        Bcfg2.Server.Plugin.StructureValidator.__init__(self)
        Bcfg2.Server.Plugin.Generator.__init__(self)
        Bcfg2.Server.Plugin.Connector.__init__(self)
        Bcfg2.Server.Plugin.Probing.__init__(self)

        self.sentinels = set()
        self.cachepath = os.path.join(self.data, "cache")
        self.keypath = os.path.join(self.data, "keys")
        if not os.path.exists(self.keypath):
            # create key directory if needed
            os.makedirs(self.keypath)

        # set up config files
        self.config = PackagesConfig(os.path.join(self.data, "packages.conf"), core.fam, self)
        self.sources = PackagesSources(
            os.path.join(self.data, "sources.xml"), self.cachepath, core.fam, self, self.config
        )

    @property
    def disableResolver(self):
        try:
            return self.config.get("global", "resolver").lower() == "disabled"
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            return False

    @property
    def disableMetaData(self):
        try:
            return self.config.get("global", "metadata").lower() == "disabled"
        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
            return False

    def create_config(self, entry, metadata):
        """ create yum/apt config for the specified host """
        attrib = {"encoding": "ascii", "owner": "root", "group": "root", "type": "file", "perms": "0644"}

        collection = Collection.factory(metadata, self.sources, self.data)
        entry.text = collection.get_config()
        for (key, value) in list(attrib.items()):
            entry.attrib.__setitem__(key, value)

    def HandleEntry(self, entry, metadata):
        if entry.tag == "Package":
            collection = Collection.factory(metadata, self.sources, self.data)
            entry.set("version", "auto")
            entry.set("type", collection.ptype)
        elif entry.tag == "Path":
            if self.config.has_section("global") and (
                (
                    self.config.has_option("global", "yum_config")
                    and entry.get("name") == self.config.get("global", "yum_config")
                )
                or (
                    self.config.has_option("global", "apt_config")
                    and entry.get("name") == self.config.get("global", "apt_config")
                )
            ):
                self.create_config(entry, metadata)

    def HandlesEntry(self, entry, metadata):
        if entry.tag == "Package":
            collection = Collection.factory(metadata, self.sources, self.data)
            if collection.magic_groups_match():
                return True
        elif entry.tag == "Path":
            # managed entries for yum/apt configs
            if (
                self.config.has_option("global", "yum_config")
                and entry.get("name") == self.config.get("global", "yum_config")
            ) or (
                self.config.has_option("global", "apt_config")
                and entry.get("name") == self.config.get("global", "apt_config")
            ):
                return True
        return False

    def validate_structures(self, metadata, structures):
        """Ensure client configurations include all needed prerequisites

        Arguments:
        metadata - client metadata instance
        structures - a list of structure-stage entry combinations
        """
        collection = Collection.factory(metadata, self.sources, self.data)
        indep = lxml.etree.Element("Independent")
        self._build_packages(metadata, indep, structures, collection=collection)
        collection.build_extra_structures(indep)
        structures.append(indep)

    def _build_packages(self, metadata, independent, structures, collection=None):
        """ build list of packages that need to be included in the
        specification by validate_structures() """
        if self.disableResolver:
            # Config requests no resolver
            return

        if collection is None:
            collection = Collection.factory(metadata, self.sources, self.data)
        # initial is the set of packages that are explicitly specified
        # in the configuration
        initial = set()
        # base is the set of initial packages with groups expanded
        base = set()
        to_remove = []
        for struct in structures:
            for pkg in struct.xpath("//Package | //BoundPackage"):
                if pkg.get("name"):
                    initial.add(pkg.get("name"))
                elif pkg.get("group"):
                    try:
                        if pkg.get("type"):
                            gpkgs = collection.get_group(pkg.get("group"), ptype=pkg.get("type"))
                        else:
                            gpkgs = collection.get_group(pkg.get("group"))
                        base.update(gpkgs)
                    except TypeError:
                        raise
                        self.logger.error("Could not resolve group %s" % pkg.get("group"))
                    to_remove.append(pkg)
                else:
                    self.logger.error("Packages: Malformed Package: %s" % lxml.etree.tostring(pkg))
        base.update(initial)
        for el in to_remove:
            el.getparent().remove(el)

        packages, unknown = collection.complete(base)
        if unknown:
            self.logger.info("Packages: Got %d unknown entries" % len(unknown))
            self.logger.info("Packages: %s" % list(unknown))
        newpkgs = list(packages.difference(initial))
        self.logger.debug("Packages: %d initial, %d complete, %d new" % (len(initial), len(packages), len(newpkgs)))
        newpkgs.sort()
        for pkg in newpkgs:
            lxml.etree.SubElement(
                independent, "BoundPackage", name=pkg, version="auto", type=collection.ptype, origin="Packages"
            )

    def Refresh(self):
        """Packages.Refresh() => True|False\nReload configuration
        specification and download sources\n"""
        self._load_config(force_update=True)
        return True

    def Reload(self):
        """Packages.Refresh() => True|False\nReload configuration
        specification and sources\n"""
        self._load_config()
        return True

    def _load_config(self, force_update=False):
        """
        Load the configuration data and setup sources

        Keyword args:
            force_update    Force downloading repo data
        """
        self._load_sources(force_update)
        self._load_gpg_keys(force_update)

    def _load_sources(self, force_update):
        """ Load sources from the config """
        self.sentinels = set()
        cachefiles = set()

        for collection in list(Collection.collections.values()):
            cachefiles.update(collection.cachefiles)
            if not self.disableMetaData:
                collection.setup_data(force_update)
            self.sentinels.update(collection.basegroups)

        Collection.clear_cache()

        for source in self.sources:
            cachefiles.add(source.cachefile)
            if not self.disableMetaData:
                source.setup_data(force_update)

        for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")):
            if cfile not in cachefiles:
                try:
                    if os.path.isdir(cfile):
                        shutil.rmtree(cfile)
                    else:
                        os.unlink(cfile)
                except OSError:
                    err = sys.exc_info()[1]
                    logger.error("Packages: Could not remove cache file %s: %s" % (cfile, err))

    def _load_gpg_keys(self, force_update):
        """ Load gpg keys from the config """
        keyfiles = []
        keys = []
        for source in self.sources:
            for key in source.gpgkeys:
                localfile = os.path.join(self.keypath, os.path.basename(key))
                if localfile not in keyfiles:
                    keyfiles.append(localfile)
                if (force_update and key not in keys) or not os.path.exists(localfile):
                    self.logger.info("Packages: Downloading and parsing %s" % key)
                    response = urlopen(key)
                    open(localfile, "w").write(response.read())
                    keys.append(key)

        for kfile in glob.glob(os.path.join(self.keypath, "*")):
            if kfile not in keyfiles:
                os.unlink(kfile)

    def get_additional_data(self, metadata):
        collection = Collection.factory(metadata, self.sources, self.data)
        return dict(sources=collection.get_additional_data())