def __init__(self, name):
"""
:param name: The filename of this properties file.
.. automethod:: _write
"""
self.name = name
self.setup = get_option_parser()
def __init__(self, name):
"""
:param name: The filename of this properties file.
.. automethod:: _write
"""
self.name = name
self.setup = get_option_parser()
def __init__(self, fname):
CfgCreator.__init__(self, fname)
StructFile.__init__(self, fname)
pubkey_path = os.path.dirname(self.name) + ".pub"
pubkey_name = os.path.join(pubkey_path, os.path.basename(pubkey_path))
self.pubkey_creator = CfgPublicKeyCreator(pubkey_name)
self.setup = get_option_parser()
self.cmd = Executor()
def __init__(self, fname):
CfgCreator.__init__(self, fname)
StructFile.__init__(self, fname)
pubkey_path = os.path.dirname(self.name) + ".pub"
pubkey_name = os.path.join(pubkey_path, os.path.basename(pubkey_path))
self.pubkey_creator = CfgPublicKeyCreator(pubkey_name)
self.setup = get_option_parser()
self.cmd = Executor()
def __init__(self, filename, cachepath, packages):
"""
:param filename: The full path to ``sources.xml``
:type filename: string
:param cachepath: The full path to the directory where
:class:`Bcfg2.Server.Plugins.Packages.Source.Source`
data will be cached
:type cachepath: string
:param packages: The Packages plugin object ``sources.xml`` is
being parsed on behalf of (i.e., the calling
object)
:type packages: Bcfg2.Server.Plugins.Packages.Packages
:raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError` -
If ``sources.xml`` cannot be read
"""
Bcfg2.Server.Plugin.Debuggable.__init__(self)
Bcfg2.Server.Plugin.StructFile.__init__(self,
filename,
should_monitor=True)
#: The full path to the directory where
#: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` data
#: will be cached
self.cachepath = cachepath
if not os.path.exists(self.cachepath):
# create cache directory if needed
try:
os.makedirs(self.cachepath)
except OSError:
err = sys.exc_info()[1]
self.logger.error("Could not create Packages cache at %s: %s" %
(self.cachepath, err))
#: The Bcfg2 options dict
self.setup = get_option_parser()
#: The :class:`Bcfg2.Server.Plugins.Packages.Packages` that
#: instantiated this ``PackagesSources`` object
self.pkg_obj = packages
#: The set of all XML files that have been successfully
#: parsed. This is used by :attr:`loaded` to determine if the
#: sources have been fully parsed and the
#: :class:`Bcfg2.Server.Plugins.Packages.Packages` plugin
#: should be told to reload its data.
self.parsed = set()
def __init__(self, filename, cachepath, packages):
"""
:param filename: The full path to ``sources.xml``
:type filename: string
:param cachepath: The full path to the directory where
:class:`Bcfg2.Server.Plugins.Packages.Source.Source`
data will be cached
:type cachepath: string
:param packages: The Packages plugin object ``sources.xml`` is
being parsed on behalf of (i.e., the calling
object)
:type packages: Bcfg2.Server.Plugins.Packages.Packages
:raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError` -
If ``sources.xml`` cannot be read
"""
Bcfg2.Server.Plugin.Debuggable.__init__(self)
Bcfg2.Server.Plugin.StructFile.__init__(self, filename,
should_monitor=True)
#: The full path to the directory where
#: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` data
#: will be cached
self.cachepath = cachepath
if not os.path.exists(self.cachepath):
# create cache directory if needed
try:
os.makedirs(self.cachepath)
except OSError:
err = sys.exc_info()[1]
self.logger.error("Could not create Packages cache at %s: %s" %
(self.cachepath, err))
#: The Bcfg2 options dict
self.setup = get_option_parser()
#: The :class:`Bcfg2.Server.Plugins.Packages.Packages` that
#: instantiated this ``PackagesSources`` object
self.pkg_obj = packages
#: The set of all XML files that have been successfully
#: parsed. This is used by :attr:`loaded` to determine if the
#: sources have been fully parsed and the
#: :class:`Bcfg2.Server.Plugins.Packages.Packages` plugin
#: should be told to reload its data.
self.parsed = set()
def __init__(self, metadata, sources, cachepath, basepath, debug=False):
"""
:param metadata: The client metadata for this collection
:type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
:param sources: A list of all sources known to the server that
will be used to generate the list of sources
that apply to this client
:type sources: list of
:class:`Bcfg2.Server.Plugins.Packages.Source.Source`
objects
:param cachepath: The filesystem path where cache and other temporary
data will be stored
:type cachepath: string
:param basepath: The filesystem path to the Packages plugin
directory, where more permanent data can be
stored
:type basepath: string
:param debug: Enable debugging output
:type debug: bool
.. -----
.. autoattribute:: __package_groups__
"""
Bcfg2.Server.Plugin.Debuggable.__init__(self)
list.__init__(self, sources)
self.debug_flag = debug
self.metadata = metadata
self.basepath = basepath
self.cachepath = cachepath
self.virt_pkgs = dict()
self.fam = get_fam()
self.setup = get_option_parser()
try:
self.ptype = sources[0].ptype
except IndexError:
self.ptype = "unknown"
def __init__(self, metadata, sources, cachepath, basepath, debug=False):
"""
:param metadata: The client metadata for this collection
:type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
:param sources: A list of all sources known to the server that
will be used to generate the list of sources
that apply to this client
:type sources: list of
:class:`Bcfg2.Server.Plugins.Packages.Source.Source`
objects
:param cachepath: The filesystem path where cache and other temporary
data will be stored
:type cachepath: string
:param basepath: The filesystem path to the Packages plugin
directory, where more permanent data can be
stored
:type basepath: string
:param debug: Enable debugging output
:type debug: bool
.. -----
.. autoattribute:: __package_groups__
"""
Bcfg2.Server.Plugin.Debuggable.__init__(self)
list.__init__(self, sources)
self.debug_flag = debug
self.metadata = metadata
self.basepath = basepath
self.cachepath = cachepath
self.virt_pkgs = dict()
self.fam = get_fam()
self.setup = get_option_parser()
try:
self.ptype = sources[0].ptype
except IndexError:
self.ptype = "unknown"
def _setup_pulp():
""" Connect to a Pulp server and pass authentication credentials.
This only needs to be called once, but multiple calls won't hurt
anything.
:returns: :class:`pulp.client.api.server.PulpServer`
"""
global PULPSERVER, PULPCONFIG
if not HAS_PULP:
msg = "Packages: Cannot create Pulp collection: Pulp libraries " + "not found"
LOGGER.error(msg)
raise Bcfg2.Server.Plugin.PluginInitError(msg)
if PULPSERVER is None:
setup = get_option_parser()
try:
username = setup.cfp.get("packages:pulp", "username")
password = setup.cfp.get("packages:pulp", "password")
except ConfigParser.NoSectionError:
msg = "Packages: No [pulp] section found in bcfg2.conf"
LOGGER.error(msg)
raise Bcfg2.Server.Plugin.PluginInitError(msg)
except ConfigParser.NoOptionError:
msg = "Packages: Required option not found in bcfg2.conf: %s" % sys.exc_info()[1]
LOGGER.error(msg)
raise Bcfg2.Server.Plugin.PluginInitError(msg)
PULPCONFIG = ConsumerConfig()
serveropts = PULPCONFIG.server
PULPSERVER = server.PulpServer(
serveropts["host"], int(serveropts["port"]), serveropts["scheme"], serveropts["path"]
)
PULPSERVER.set_basic_auth_credentials(username, password)
server.set_active_server(PULPSERVER)
return PULPSERVER
def __init__(self, config, times):
self.setup = get_option_parser()
self.config = config
self.times = times
self.dryrun = self.setup['dryrun']
self.times['initialization'] = time.time()
self.tools = []
#: A dict of the state of each entry. Keys are the entries.
#: Values are boolean: True means that the entry is good,
#: False means that the entry is bad.
self.states = {}
self.whitelist = []
self.blacklist = []
self.removal = []
self.logger = logging.getLogger(__name__)
drivers = self.setup['drivers']
for driver in drivers[:]:
if (driver not in Bcfg2.Client.Tools.__all__ and
isinstance(driver, str)):
self.logger.error("Tool driver %s is not available" % driver)
drivers.remove(driver)
tclass = {}
for tool in drivers:
if not isinstance(tool, str):
tclass[time.time()] = tool
tool_class = "Bcfg2.Client.Tools.%s" % tool
try:
tclass[tool] = getattr(__import__(tool_class, globals(),
locals(), ['*']),
tool)
except ImportError:
continue
except:
self.logger.error("Tool %s unexpectedly failed to load" % tool,
exc_info=1)
for tool in list(tclass.values()):
try:
self.tools.append(tool(config))
except Bcfg2.Client.Tools.ToolInstantiationError:
continue
except:
self.logger.error("Failed to instantiate tool %s" % tool,
exc_info=1)
for tool in self.tools[:]:
for conflict in getattr(tool, 'conflicts', []):
for item in self.tools:
if item.name == conflict:
self.tools.remove(item)
self.logger.info("Loaded tool drivers:")
self.logger.info([tool.name for tool in self.tools])
deprecated = [tool.name for tool in self.tools if tool.deprecated]
if deprecated:
self.logger.warning("Loaded deprecated tool drivers:")
self.logger.warning(deprecated)
experimental = [tool.name for tool in self.tools if tool.experimental]
if experimental:
self.logger.warning("Loaded experimental tool drivers:")
self.logger.warning(experimental)
# find entries not handled by any tools
self.unhandled = [entry for struct in config
for entry in struct
if entry not in self.handled]
if self.unhandled:
self.logger.error("The following entries are not handled by any "
"tool:")
for entry in self.unhandled:
self.logger.error("%s:%s:%s" % (entry.tag, entry.get('type'),
entry.get('name')))
self.find_dups(config)
pkgs = [(entry.get('name'), entry.get('origin'))
for struct in config
for entry in struct
if entry.tag == 'Package']
if pkgs:
self.logger.debug("The following packages are specified in bcfg2:")
self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] is None])
self.logger.debug("The following packages are prereqs added by "
"Packages:")
self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == 'Packages'])
def __init__(self, basepath, xsource): # pylint: disable=R0912
"""
:param basepath: The base filesystem path under which cache
data for this source should be stored
:type basepath: string
:param xsource: The XML tag that describes this source
:type source: lxml.etree._Element
:raises: :class:`Bcfg2.Server.Plugins.Packages.Source.SourceInitError`
"""
Bcfg2.Server.Plugin.Debuggable.__init__(self)
#: The base filesystem path under which cache data for this
#: source should be stored
self.basepath = basepath
#: The XML tag that describes this source
self.xsource = xsource
#: A Bcfg2 options dict
self.setup = get_option_parser()
#: A set of package names that are deemed "essential" by this
#: source
self.essentialpkgs = set()
#: A list of the text of all 'Component' attributes of this
#: source from XML
self.components = [item.text for item in xsource.findall('Component')]
#: A list of the arches supported by this source
self.arches = [item.text for item in xsource.findall('Arch')]
#: A list of the the names of packages that are blacklisted
#: from this source
self.blacklist = [item.text for item in xsource.findall('Blacklist')]
#: A list of the the names of packages that are whitelisted in
#: this source
self.whitelist = [item.text for item in xsource.findall('Whitelist')]
#: A dict of repository options that will be included in the
#: configuration generated on the server side (if such is
#: applicable; most backends do not generate any sort of
#: repository configuration on the Bcfg2 server)
self.server_options = dict()
#: A dict of repository options that will be included in the
#: configuration generated for the client (if that is
#: supported by the backend)
self.client_options = dict()
opts = xsource.findall("Options")
for el in opts:
repoopts = dict([(k, v)
for k, v in el.attrib.items()
if k != "clientonly" and k != "serveronly"])
if el.get("clientonly", "false").lower() == "false":
self.server_options.update(repoopts)
if el.get("serveronly", "false").lower() == "false":
self.client_options.update(repoopts)
#: A list of URLs to GPG keys that apply to this source
self.gpgkeys = [el.text for el in xsource.findall("GPGKey")]
#: Whether or not to include essential packages from this source
self.essential = xsource.get('essential', 'true').lower() == 'true'
#: Whether or not to include recommended packages from this source
self.recommended = xsource.get('recommended',
'false').lower() == 'true'
#: The "rawurl" attribute from :attr:`xsource`, if applicable.
#: A trailing slash is automatically appended to this if there
#: wasn't one already present.
self.rawurl = xsource.get('rawurl', '')
if self.rawurl and not self.rawurl.endswith("/"):
self.rawurl += "/"
#: The "url" attribute from :attr:`xsource`, if applicable. A
#: trailing slash is automatically appended to this if there
#: wasn't one already present.
self.url = xsource.get('url', '')
if self.url and not self.url.endswith("/"):
self.url += "/"
#: The "version" attribute from :attr:`xsource`
self.version = xsource.get('version', '')
#: A list of predicates that are used to determine if this
#: source applies to a given
#: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata`
#: object.
self.conditions = []
#: Formerly, :ref:`server-plugins-generators-packages` only
#: supported applying package sources to groups; that is, they
#: could not be assigned by more complicated logic like
#: per-client repositories and group or client negation. This
#: attribute attempts to provide for some limited backwards
#: compat with older code that relies on this.
self.groups = []
for el in xsource.iterancestors():
if el.tag == "Group":
if el.get("negate", "false").lower() == "true":
self.conditions.append(lambda m, el=el:
el.get("name") not in m.groups)
else:
self.groups.append(el.get("name"))
self.conditions.append(lambda m, el=el:
el.get("name") in m.groups)
elif el.tag == "Client":
if el.get("negate", "false").lower() == "true":
self.conditions.append(lambda m, el=el:
el.get("name") != m.hostname)
else:
self.conditions.append(lambda m, el=el:
el.get("name") == m.hostname)
#: A set of all package names in this source. This will not
#: necessarily be populated, particularly by backends that
#: reimplement large portions of
#: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`
self.pkgnames = set()
#: A dict of ``<package name>`` -> ``<list of dependencies>``.
#: This will not necessarily be populated, particularly by
#: backends that reimplement large portions of
#: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`
self.deps = dict()
#: A dict of ``<package name>`` -> ``<list of provided
#: symbols>``. This will not necessarily be populated,
#: particularly by backends that reimplement large portions of
#: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`
self.provides = dict()
#: The file (or directory) used for this source's cache data
self.cachefile = os.path.join(self.basepath,
"cache-%s" % self.cachekey)
if not self.rawurl:
baseurl = self.url + "%(version)s/%(component)s/%(arch)s/"
else:
baseurl = self.rawurl
#: A list of dicts, each of which describes the URL to one
#: repository contained in this source. Each dict contains
#: the following keys:
#:
#: * ``version``: The version of the repo (``None`` for
#: ``rawurl`` repos)
#: * ``component``: The component use to form this URL
#: (``None`` for ``rawurl`` repos)
#: * ``arch``: The architecture of this repo
#: * ``baseurl``: Either the ``rawurl`` attribute, or the
#: format string built from the ``url`` attribute
#: * ``url``: The actual URL to the repository
self.url_map = []
for arch in self.arches:
if self.url:
usettings = [dict(version=self.version, component=comp,
arch=arch)
for comp in self.components]
else: # rawurl given
usettings = [dict(version=self.version, component=None,
arch=arch)]
for setting in usettings:
if not self.rawurl:
setting['baseurl'] = self.url
else:
setting['baseurl'] = self.rawurl
setting['url'] = baseurl % setting
self.url_map.extend(usettings)
def __init__(self, config, times):
self.setup = get_option_parser()
self.config = config
self.times = times
self.dryrun = self.setup['dryrun']
self.times['initialization'] = time.time()
self.tools = []
#: A dict of the state of each entry. Keys are the entries.
#: Values are boolean: True means that the entry is good,
#: False means that the entry is bad.
self.states = {}
self.whitelist = []
self.blacklist = []
self.removal = []
self.logger = logging.getLogger(__name__)
drivers = self.setup['drivers']
for driver in drivers[:]:
if (driver not in Bcfg2.Client.Tools.__all__
and isinstance(driver, str)):
self.logger.error("Tool driver %s is not available" % driver)
drivers.remove(driver)
tclass = {}
for tool in drivers:
if not isinstance(tool, str):
tclass[time.time()] = tool
tool_class = "Bcfg2.Client.Tools.%s" % tool
try:
tclass[tool] = getattr(
__import__(tool_class, globals(), locals(), ['*']), tool)
except ImportError:
continue
except:
self.logger.error("Tool %s unexpectedly failed to load" % tool,
exc_info=1)
for tool in list(tclass.values()):
try:
self.tools.append(tool(config))
except Bcfg2.Client.Tools.ToolInstantiationError:
continue
except:
self.logger.error("Failed to instantiate tool %s" % tool,
exc_info=1)
for tool in self.tools[:]:
for conflict in getattr(tool, 'conflicts', []):
for item in self.tools:
if item.name == conflict:
self.tools.remove(item)
self.logger.info("Loaded tool drivers:")
self.logger.info([tool.name for tool in self.tools])
deprecated = [tool.name for tool in self.tools if tool.deprecated]
if deprecated:
self.logger.warning("Loaded deprecated tool drivers:")
self.logger.warning(deprecated)
experimental = [tool.name for tool in self.tools if tool.experimental]
if experimental:
self.logger.warning("Loaded experimental tool drivers:")
self.logger.warning(experimental)
# find entries not handled by any tools
self.unhandled = [
entry for struct in config for entry in struct
if entry not in self.handled
]
if self.unhandled:
self.logger.error("The following entries are not handled by any "
"tool:")
for entry in self.unhandled:
self.logger.error(
"%s:%s:%s" %
(entry.tag, entry.get('type'), entry.get('name')))
self.find_dups(config)
pkgs = [(entry.get('name'), entry.get('origin')) for struct in config
for entry in struct if entry.tag == 'Package']
if pkgs:
self.logger.debug("The following packages are specified in bcfg2:")
self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] is None])
self.logger.debug("The following packages are prereqs added by "
"Packages:")
self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == 'Packages'])
def __init__(self, basepath, xsource): # pylint: disable=R0912
"""
:param basepath: The base filesystem path under which cache
data for this source should be stored
:type basepath: string
:param xsource: The XML tag that describes this source
:type source: lxml.etree._Element
:raises: :class:`Bcfg2.Server.Plugins.Packages.Source.SourceInitError`
"""
Bcfg2.Server.Plugin.Debuggable.__init__(self)
#: The base filesystem path under which cache data for this
#: source should be stored
self.basepath = basepath
#: The XML tag that describes this source
self.xsource = xsource
#: A Bcfg2 options dict
self.setup = get_option_parser()
#: A set of package names that are deemed "essential" by this
#: source
self.essentialpkgs = set()
#: A list of the text of all 'Component' attributes of this
#: source from XML
self.components = [item.text for item in xsource.findall('Component')]
#: A list of the arches supported by this source
self.arches = [item.text for item in xsource.findall('Arch')]
#: A list of the the names of packages that are blacklisted
#: from this source
self.blacklist = [item.text for item in xsource.findall('Blacklist')]
#: A list of the the names of packages that are whitelisted in
#: this source
self.whitelist = [item.text for item in xsource.findall('Whitelist')]
#: Whether or not to include deb-src lines in the generated APT
#: configuration
self.debsrc = xsource.get('debsrc', 'false') == 'true'
#: A dict of repository options that will be included in the
#: configuration generated on the server side (if such is
#: applicable; most backends do not generate any sort of
#: repository configuration on the Bcfg2 server)
self.server_options = dict()
#: A dict of repository options that will be included in the
#: configuration generated for the client (if that is
#: supported by the backend)
self.client_options = dict()
opts = xsource.findall("Options")
for el in opts:
repoopts = dict([(k, v) for k, v in el.attrib.items()
if k != "clientonly" and k != "serveronly"])
if el.get("clientonly", "false").lower() == "false":
self.server_options.update(repoopts)
if el.get("serveronly", "false").lower() == "false":
self.client_options.update(repoopts)
#: A list of URLs to GPG keys that apply to this source
self.gpgkeys = [el.text for el in xsource.findall("GPGKey")]
#: Whether or not to include essential packages from this source
self.essential = xsource.get('essential', 'true').lower() == 'true'
#: Whether or not to include recommended packages from this source
self.recommended = xsource.get('recommended',
'false').lower() == 'true'
#: The "rawurl" attribute from :attr:`xsource`, if applicable.
#: A trailing slash is automatically appended to this if there
#: wasn't one already present.
self.rawurl = xsource.get('rawurl', '')
if self.rawurl and not self.rawurl.endswith("/"):
self.rawurl += "/"
#: The "url" attribute from :attr:`xsource`, if applicable. A
#: trailing slash is automatically appended to this if there
#: wasn't one already present.
self.url = xsource.get('url', '')
if self.url and not self.url.endswith("/"):
self.url += "/"
#: The "version" attribute from :attr:`xsource`
self.version = xsource.get('version', '')
#: A list of predicates that are used to determine if this
#: source applies to a given
#: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata`
#: object.
self.conditions = []
#: Formerly, :ref:`server-plugins-generators-packages` only
#: supported applying package sources to groups; that is, they
#: could not be assigned by more complicated logic like
#: per-client repositories and group or client negation. This
#: attribute attempts to provide for some limited backwards
#: compat with older code that relies on this.
self.groups = []
for el in xsource.iterancestors():
if el.tag == "Group":
if el.get("negate", "false").lower() == "true":
self.conditions.append(
lambda m, el=el: el.get("name") not in m.groups)
else:
self.groups.append(el.get("name"))
self.conditions.append(
lambda m, el=el: el.get("name") in m.groups)
elif el.tag == "Client":
if el.get("negate", "false").lower() == "true":
self.conditions.append(
lambda m, el=el: el.get("name") != m.hostname)
else:
self.conditions.append(
lambda m, el=el: el.get("name") == m.hostname)
#: A set of all package names in this source. This will not
#: necessarily be populated, particularly by backends that
#: reimplement large portions of
#: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`
self.pkgnames = set()
#: A dict of ``<package name>`` -> ``<list of dependencies>``.
#: This will not necessarily be populated, particularly by
#: backends that reimplement large portions of
#: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`
self.deps = dict()
#: A dict of ``<package name>`` -> ``<list of provided
#: symbols>``. This will not necessarily be populated,
#: particularly by backends that reimplement large portions of
#: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`
self.provides = dict()
#: The file (or directory) used for this source's cache data
self.cachefile = os.path.join(self.basepath,
"cache-%s" % self.cachekey)
if not self.rawurl:
baseurl = self.url + "%(version)s/%(component)s/%(arch)s/"
else:
baseurl = self.rawurl
#: A list of dicts, each of which describes the URL to one
#: repository contained in this source. Each dict contains
#: the following keys:
#:
#: * ``version``: The version of the repo (``None`` for
#: ``rawurl`` repos)
#: * ``component``: The component use to form this URL
#: (``None`` for ``rawurl`` repos)
#: * ``arch``: The architecture of this repo
#: * ``baseurl``: Either the ``rawurl`` attribute, or the
#: format string built from the ``url`` attribute
#: * ``url``: The actual URL to the repository
self.url_map = []
for arch in self.arches:
if self.url:
usettings = [
dict(version=self.version, component=comp, arch=arch)
for comp in self.components
]
else: # rawurl given
usettings = [
dict(version=self.version, component=None, arch=arch)
]
for setting in usettings:
if not self.rawurl:
setting['baseurl'] = self.url
else:
setting['baseurl'] = self.rawurl
setting['url'] = baseurl % setting
self.url_map.extend(usettings)
