def list_sources(cfg_list, depends, pkg_list):
src_list = []
LOG.debug(
"Looking for data source in: %s,"
" via packages %s that matches dependencies %s",
cfg_list,
pkg_list,
depends,
)
for ds_name in cfg_list:
if not ds_name.startswith(DS_PREFIX):
ds_name = "%s%s" % (DS_PREFIX, ds_name)
m_locs, _looked_locs = importer.find_module(ds_name, pkg_list,
["get_datasource_list"])
if not m_locs:
LOG.error(
"Could not import %s. Does the DataSource exist and "
"is it importable?",
ds_name,
)
for m_loc in m_locs:
mod = importer.import_module(m_loc)
lister = getattr(mod, "get_datasource_list")
matches = lister(depends)
if matches:
src_list.extend(matches)
break
return src_list
def get_schema() -> dict:
"""Return jsonschema coalesced from all cc_* cloud-config modules."""
schema_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"cloud-init-schema.json")
full_schema = None
try:
full_schema = json.loads(load_file(schema_file))
except Exception as e:
LOG.warning("Cannot parse JSON schema file %s. %s", schema_file, e)
if not full_schema:
LOG.warning(
"No base JSON schema files found at %s."
" Setting default empty schema",
schema_file,
)
full_schema = {
"$defs": {},
"$schema": "http://json-schema.org/draft-04/schema#",
"allOf": [],
}
# TODO( Drop the get_modules loop when all legacy cc_* schema migrates )
# Supplement base_schema with any legacy modules which still contain a
# "schema" attribute. Legacy cc_* modules will be migrated to use the
# store module schema in the composite cloud-init-schema-<version>.json
# and will drop "schema" at that point.
for (_, mod_name) in get_modules().items():
# All cc_* modules need a "meta" attribute to represent schema defs
(mod_locs, _) = importer.find_module(mod_name, ["cloudinit.config"],
["schema"])
if mod_locs:
mod = importer.import_module(mod_locs[0])
full_schema["allOf"].append(mod.schema)
return full_schema
def fetch(name):
locs = importer.find_module(name, ['', __name__], ['Distro'])
if not locs:
raise ImportError("No distribution found for distro %s" % (name))
mod = importer.import_module(locs[0])
cls = getattr(mod, 'Distro')
return cls
def test_get_hostname_subclass_support(self):
"""Validate get_hostname signature on all subclasses of DataSource."""
base_args = inspect.getfullargspec(DataSource.get_hostname)
# Import all DataSource subclasses so we can inspect them.
modules = util.get_modules_from_dir(
os.path.dirname(os.path.dirname(__file__)))
for _loc, name in modules.items():
mod_locs, _ = importer.find_module(name, ["cloudinit.sources"], [])
if mod_locs:
importer.import_module(mod_locs[0])
for child in DataSource.__subclasses__():
if "Test" in child.dsname:
continue
self.assertEqual(
base_args,
inspect.getfullargspec(child.get_hostname),
"%s does not implement DataSource.get_hostname params" % child,
)
for grandchild in child.__subclasses__():
self.assertEqual(
base_args,
inspect.getfullargspec(grandchild.get_hostname),
"%s does not implement DataSource.get_hostname params" %
grandchild,
)
def _fixup_modules(self, raw_mods):
mostly_mods = []
for raw_mod in raw_mods:
raw_name = raw_mod["mod"]
freq = raw_mod.get("freq")
run_args = raw_mod.get("args") or []
mod_name = config.form_module_name(raw_name)
if not mod_name:
continue
if freq and freq not in FREQUENCIES:
LOG.warning(
"Config specified module %s has an unknown frequency %s",
raw_name,
freq,
)
# Reset it so when ran it will get set to a known value
freq = None
mod_locs, looked_locs = importer.find_module(
mod_name, ["", type_utils.obj_name(config)], ["handle"]
)
if not mod_locs:
LOG.warning(
"Could not find module named %s (searched %s)",
mod_name,
looked_locs,
)
continue
mod = config.fixup_module(importer.import_module(mod_locs[0]))
mostly_mods.append([mod, raw_name, freq, run_args])
return mostly_mods
def fetch(name):
locs = importer.find_module(name, ["", __name__], ["Distro"])
if not locs:
raise ImportError("No distribution found for distro %s" % (name))
mod = importer.import_module(locs[0])
cls = getattr(mod, "Distro")
return cls
def register_handlers_in_dir(path):
# Attempts to register any handler modules under the given path.
if not path or not os.path.isdir(path):
return
potential_handlers = util.find_modules(path)
for (fname, mod_name) in potential_handlers.items():
try:
mod_locs, looked_locs = importer.find_module(
mod_name, [""], ["list_types", "handle_part"]
)
if not mod_locs:
LOG.warning(
"Could not find a valid user-data handler"
" named %s in file %s (searched %s)",
mod_name,
fname,
looked_locs,
)
continue
mod = importer.import_module(mod_locs[0])
mod = handlers.fixup_handler(mod)
types = c_handlers.register(mod)
if types:
LOG.debug(
"Added custom handler for %s [%s] from %s",
types,
mod,
fname,
)
except Exception:
util.logexc(
LOG, "Failed to register handler from %s", fname
)
def test_get_hostname_subclass_support(self):
"""Validate get_hostname signature on all subclasses of DataSource."""
# Use inspect.getfullargspec when we drop py2.6 and py2.7
get_args = inspect.getargspec # pylint: disable=W1505
base_args = get_args(DataSource.get_hostname) # pylint: disable=W1505
# Import all DataSource subclasses so we can inspect them.
modules = util.find_modules(os.path.dirname(os.path.dirname(__file__)))
for _loc, name in modules.items():
mod_locs, _ = importer.find_module(name, ['cloudinit.sources'], [])
if mod_locs:
importer.import_module(mod_locs[0])
for child in DataSource.__subclasses__():
if 'Test' in child.dsname:
continue
self.assertEqual(
base_args,
get_args(child.get_hostname), # pylint: disable=W1505
'%s does not implement DataSource.get_hostname params'
% child)
for grandchild in child.__subclasses__():
self.assertEqual(
base_args,
get_args(grandchild.get_hostname), # pylint: disable=W1505
'%s does not implement DataSource.get_hostname params'
% grandchild)
def fetch(name):
locs, looked_locs = importer.find_module(name, ['', __name__], ['Distro'])
if not locs:
raise ImportError("No distribution found for distro %s (searched %s)"
% (name, looked_locs))
mod = importer.import_module(locs[0])
cls = getattr(mod, 'Distro')
return cls
def fetch(name: str) -> Type[Distro]:
locs, looked_locs = importer.find_module(name, ["", __name__], ["Distro"])
if not locs:
raise ImportError("No distribution found for distro %s (searched %s)" %
(name, looked_locs))
mod = importer.import_module(locs[0])
cls = getattr(mod, "Distro")
return cls
def get_meta() -> dict:
"""Return metadata coalesced from all cc_* cloud-config module."""
full_meta = dict()
for (_, mod_name) in get_modules().items():
mod_locs, _ = importer.find_module(mod_name, ["cloudinit.config"],
["meta"])
if mod_locs:
mod = importer.import_module(mod_locs[0])
full_meta[mod.meta["id"]] = mod.meta
return full_meta
def get_schema() -> dict:
"""Return jsonschema coalesced from all cc_* cloud-config module."""
full_schema = {
"$schema": "http://json-schema.org/draft-04/schema#",
"id": "cloud-config-schema",
"allOf": [],
}
for (_, mod_name) in get_modules().items():
(mod_locs, _) = importer.find_module(mod_name, ["cloudinit.config"],
["schema"])
if mod_locs:
mod = importer.import_module(mod_locs[0])
full_schema["allOf"].append(mod.schema)
return full_schema
def _fixup_modules(self, raw_mods) -> List[ModuleDetails]:
"""Convert list of returned from _read_modules() into new format.
Invalid modules and arguments are ingnored.
Also ensures that the module has the required meta fields.
"""
mostly_mods = []
for raw_mod in raw_mods:
raw_name = raw_mod["mod"]
freq = raw_mod.get("freq")
run_args = raw_mod.get("args") or []
mod_name = form_module_name(raw_name)
if not mod_name:
continue
if freq and freq not in FREQUENCIES:
LOG.warning(
"Config specified module %s has an unknown frequency %s",
raw_name,
freq,
)
# Misconfigured in /etc/cloud/cloud.cfg. Reset so cc_* module
# default meta attribute "frequency" value is used.
freq = None
mod_locs, looked_locs = importer.find_module(
mod_name, ["", type_utils.obj_name(config)], ["handle"])
if not mod_locs:
LOG.warning(
"Could not find module named %s (searched %s)",
mod_name,
looked_locs,
)
continue
mod = importer.import_module(mod_locs[0])
validate_module(mod, raw_name)
if freq is None:
# Use cc_* module default setting since no cloud.cfg overrides
freq = mod.meta["frequency"]
mostly_mods.append(
ModuleDetails(
module=mod,
name=raw_name,
frequency=freq,
run_args=run_args,
))
return mostly_mods
def list_sources(cfg_list, depends, pkg_list):
src_list = []
LOG.debug(("Looking for for data source in: %s,"
" via packages %s that matches dependencies %s"), cfg_list,
pkg_list, depends)
for ds_name in cfg_list:
if not ds_name.startswith(DS_PREFIX):
ds_name = '%s%s' % (DS_PREFIX, ds_name)
m_locs = importer.find_module(ds_name, pkg_list,
['get_datasource_list'])
for m_loc in m_locs:
mod = importer.import_module(m_loc)
lister = getattr(mod, "get_datasource_list")
matches = lister(depends)
if matches:
src_list.extend(matches)
break
return src_list
def get_schema():
"""Return jsonschema coalesced from all cc_* cloud-config module."""
global FULL_SCHEMA
if FULL_SCHEMA:
return FULL_SCHEMA
full_schema = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'id': 'cloud-config-schema', 'allOf': []}
configs_dir = os.path.dirname(os.path.abspath(__file__))
potential_handlers = find_modules(configs_dir)
for (fname, mod_name) in potential_handlers.items():
mod_locs, looked_locs = importer.find_module(
mod_name, ['cloudinit.config'], ['schema'])
if mod_locs:
mod = importer.import_module(mod_locs[0])
full_schema['allOf'].append(mod.schema)
FULL_SCHEMA = full_schema
return full_schema
def get_schema():
"""Return jsonschema coalesced from all cc_* cloud-config module."""
global FULL_SCHEMA
if FULL_SCHEMA:
return FULL_SCHEMA
full_schema = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'id': 'cloud-config-schema', 'allOf': []}
configs_dir = os.path.dirname(os.path.abspath(__file__))
potential_handlers = find_modules(configs_dir)
for (_fname, mod_name) in potential_handlers.items():
mod_locs, _looked_locs = importer.find_module(
mod_name, ['cloudinit.config'], ['schema'])
if mod_locs:
mod = importer.import_module(mod_locs[0])
full_schema['allOf'].append(mod.schema)
FULL_SCHEMA = full_schema
return full_schema
def list_sources(cfg_list, depends, pkg_list):
src_list = []
LOG.debug(("Looking for for data source in: %s,"
" via packages %s that matches dependencies %s"),
cfg_list, pkg_list, depends)
for ds_name in cfg_list:
if not ds_name.startswith(DS_PREFIX):
ds_name = '%s%s' % (DS_PREFIX, ds_name)
m_locs, _looked_locs = importer.find_module(ds_name,
pkg_list,
['get_datasource_list'])
for m_loc in m_locs:
mod = importer.import_module(m_loc)
lister = getattr(mod, "get_datasource_list")
matches = lister(depends)
if matches:
src_list.extend(matches)
break
return src_list
def construct(parsed_mergers):
mergers_to_be = []
for (m_name, m_ops) in parsed_mergers:
if not m_name.startswith(MERGER_PREFIX):
m_name = MERGER_PREFIX + str(m_name)
merger_locs = importer.find_module(m_name, [__name__], [MERGER_ATTR])
if not merger_locs:
msg = ("Could not find merger module named '%s' "
"with attribute '%s'") % (m_name, MERGER_ATTR)
raise ImportError(msg)
else:
mod = importer.import_module(merger_locs[0])
mod_attr = getattr(mod, MERGER_ATTR)
mergers_to_be.append((mod_attr, m_ops))
# Now form them...
mergers = []
root = LookupMerger(mergers)
for (attr, opts) in mergers_to_be:
mergers.append(attr(root, opts))
return root
def _fixup_modules(self, raw_mods):
mostly_mods = []
for raw_mod in raw_mods:
raw_name = raw_mod["mod"]
freq = raw_mod.get("freq")
run_args = raw_mod.get("args") or []
mod_name = config.form_module_name(raw_name)
if not mod_name:
continue
if freq and freq not in FREQUENCIES:
LOG.warn(("Config specified module %s" " has an unknown frequency %s"), raw_name, freq)
# Reset it so when ran it will get set to a known value
freq = None
mod_locs, looked_locs = importer.find_module(mod_name, ["", type_utils.obj_name(config)], ["handle"])
if not mod_locs:
LOG.warn("Could not find module named %s (searched %s)", mod_name, looked_locs)
continue
mod = config.fixup_module(importer.import_module(mod_locs[0]))
mostly_mods.append([mod, raw_name, freq, run_args])
return mostly_mods
def construct(parsed_mergers):
mergers_to_be = []
for (m_name, m_ops) in parsed_mergers:
if not m_name.startswith(MERGER_PREFIX):
m_name = MERGER_PREFIX + str(m_name)
merger_locs = importer.find_module(m_name,
[__name__],
[MERGER_ATTR])
if not merger_locs:
msg = ("Could not find merger module named '%s' "
"with attribute '%s'") % (m_name, MERGER_ATTR)
raise ImportError(msg)
else:
mod = importer.import_module(merger_locs[0])
mod_attr = getattr(mod, MERGER_ATTR)
mergers_to_be.append((mod_attr, m_ops))
# Now form them...
mergers = []
root = LookupMerger(mergers)
for (attr, opts) in mergers_to_be:
mergers.append(attr(root, opts))
return root
def load_doc(requested_modules: list) -> str:
"""Load module docstrings
Docstrings are generated on module load. Reduce, reuse, recycle.
"""
docs = ""
all_modules = list(get_modules().values()) + ["all"]
invalid_docs = set(requested_modules).difference(set(all_modules))
if invalid_docs:
error("Invalid --docs value {}. Must be one of: {}".format(
list(invalid_docs),
", ".join(all_modules),
))
for mod_name in all_modules:
if "all" in requested_modules or mod_name in requested_modules:
(mod_locs, _) = importer.find_module(mod_name,
["cloudinit.config"],
["meta"])
if mod_locs:
mod = importer.import_module(mod_locs[0])
docs += mod.__doc__ or ""
return docs
def _fixup_modules(self, raw_mods):
mostly_mods = []
for raw_mod in raw_mods:
raw_name = raw_mod['mod']
freq = raw_mod.get('freq')
run_args = raw_mod.get('args') or []
mod_name = config.form_module_name(raw_name)
if not mod_name:
continue
if freq and freq not in FREQUENCIES:
LOG.warn(("Config specified module %s"
" has an unknown frequency %s"), raw_name, freq)
# Reset it so when ran it will get set to a known value
freq = None
mod_locs = importer.find_module(
mod_name, ['', type_utils.obj_name(config)], ['handle'])
if not mod_locs:
LOG.warn("Could not find module named %s", mod_name)
continue
mod = config.fixup_module(importer.import_module(mod_locs[0]))
mostly_mods.append([mod, raw_name, freq, run_args])
return mostly_mods
def register_handlers_in_dir(path):
# Attempts to register any handler modules under the given path.
if not path or not os.path.isdir(path):
return
potential_handlers = util.find_modules(path)
for (fname, mod_name) in potential_handlers.items():
try:
mod_locs, looked_locs = importer.find_module(
mod_name, [''], ['list_types', 'handle_part'])
if not mod_locs:
LOG.warn("Could not find a valid user-data handler"
" named %s in file %s (searched %s)",
mod_name, fname, looked_locs)
continue
mod = importer.import_module(mod_locs[0])
mod = handlers.fixup_handler(mod)
types = c_handlers.register(mod)
if types:
LOG.debug("Added custom handler for %s [%s] from %s",
types, mod, fname)
except Exception:
util.logexc(LOG, "Failed to register handler from %s",
fname)
