def test_none_ds_runs_modules_which_do_not_define_distros(self):
"""Any modules which do not define a distros attribute are run."""
initer = stages.Init()
initer.read_cfg()
initer.initialize()
initer.fetch()
initer.instancify()
initer.update()
initer.cloudify().run(
"consume_data",
initer.consume_data,
args=[PER_INSTANCE],
freq=PER_INSTANCE,
)
mods = Modules(initer)
(which_ran, failures) = mods.run_section("cloud_init_modules")
self.assertTrue(len(failures) == 0)
self.assertTrue(os.path.exists("/etc/blah.ini"))
self.assertIn("write-files", which_ran)
contents = util.load_file("/etc/blah.ini")
self.assertEqual(contents, "blah")
self.assertNotIn(
"Skipping modules ['write-files'] because they are not verified on"
" distro 'ubuntu'",
self.logs.getvalue(),
)
def test_none_ds_forces_run_via_unverified_modules(self):
"""run_section forced skipped modules by using unverified_modules."""
# re-write cloud.cfg with unverified_modules override
cfg = copy.deepcopy(self.cfg)
cfg["unverified_modules"] = ["spacewalk"] # Would have skipped
cloud_cfg = safeyaml.dumps(cfg)
util.ensure_dir(os.path.join(self.new_root, "etc", "cloud"))
util.write_file(
os.path.join(self.new_root, "etc", "cloud", "cloud.cfg"),
cloud_cfg)
initer = stages.Init()
initer.read_cfg()
initer.initialize()
initer.fetch()
initer.instancify()
initer.update()
initer.cloudify().run(
"consume_data",
initer.consume_data,
args=[PER_INSTANCE],
freq=PER_INSTANCE,
)
mods = Modules(initer)
(which_ran, failures) = mods.run_section("cloud_init_modules")
self.assertTrue(len(failures) == 0)
self.assertIn("spacewalk", which_ran)
self.assertIn("running unverified_modules: 'spacewalk'",
self.logs.getvalue())
def test_none_ds_run_with_no_config_modules(self):
"""run_section will report no modules run when none are configured."""
# re-write cloud.cfg with unverified_modules override
cfg = copy.deepcopy(self.cfg)
# Represent empty configuration in /etc/cloud/cloud.cfg
cfg["cloud_init_modules"] = None
cloud_cfg = safeyaml.dumps(cfg)
util.ensure_dir(os.path.join(self.new_root, "etc", "cloud"))
util.write_file(
os.path.join(self.new_root, "etc", "cloud", "cloud.cfg"),
cloud_cfg)
initer = stages.Init()
initer.read_cfg()
initer.initialize()
initer.fetch()
initer.instancify()
initer.update()
initer.cloudify().run(
"consume_data",
initer.consume_data,
args=[PER_INSTANCE],
freq=PER_INSTANCE,
)
mods = Modules(initer)
(which_ran, failures) = mods.run_section("cloud_init_modules")
self.assertTrue(len(failures) == 0)
self.assertEqual([], which_ran)
def test_none_ds_runs_modules_which_distros_all(self):
"""Skip modules which define distros attribute as supporting 'all'.
This is done in the module with the declaration:
distros = [ALL_DISTROS]. runcmd is an example.
"""
initer = stages.Init()
initer.read_cfg()
initer.initialize()
initer.fetch()
initer.instancify()
initer.update()
initer.cloudify().run(
"consume_data",
initer.consume_data,
args=[PER_INSTANCE],
freq=PER_INSTANCE,
)
mods = Modules(initer)
(which_ran, failures) = mods.run_section("cloud_init_modules")
self.assertTrue(len(failures) == 0)
self.assertIn("runcmd", which_ran)
self.assertNotIn(
"Skipping modules 'runcmd' because they are not verified on"
" distro 'ubuntu'",
self.logs.getvalue(),
)
def test_simple_jsonp_vendor_and_vendor2_and_user(self):
# test that user-data wins over vendor
user_blob = """
#cloud-config-jsonp
[
{ "op": "add", "path": "/baz", "value": "qux" },
{ "op": "add", "path": "/bar", "value": "qux2" },
{ "op": "add", "path": "/foobar", "value": "qux3" }
]
"""
vendor_blob = """
#cloud-config-jsonp
[
{ "op": "add", "path": "/baz", "value": "quxA" },
{ "op": "add", "path": "/bar", "value": "quxB" },
{ "op": "add", "path": "/foo", "value": "quxC" },
{ "op": "add", "path": "/corge", "value": "quxEE" }
]
"""
vendor2_blob = """
#cloud-config-jsonp
[
{ "op": "add", "path": "/corge", "value": "quxD" },
{ "op": "add", "path": "/grault", "value": "quxFF" },
{ "op": "add", "path": "/foobar", "value": "quxGG" }
]
"""
self.reRoot()
initer = stages.Init()
initer.datasource = FakeDataSource(user_blob,
vendordata=vendor_blob,
vendordata2=vendor2_blob)
initer.read_cfg()
initer.initialize()
initer.fetch()
initer.instancify()
initer.update()
initer.cloudify().run(
"consume_data",
initer.consume_data,
args=[PER_INSTANCE],
freq=PER_INSTANCE,
)
mods = Modules(initer)
(_which_ran, _failures) = mods.run_section("cloud_init_modules")
cfg = mods.cfg
self.assertIn("vendor_data", cfg)
self.assertIn("vendor_data2", cfg)
# Confirm that vendordata2 overrides vendordata, and that
# userdata overrides both
self.assertEqual("qux", cfg["baz"])
self.assertEqual("qux2", cfg["bar"])
self.assertEqual("qux3", cfg["foobar"])
self.assertEqual("quxC", cfg["foo"])
self.assertEqual("quxD", cfg["corge"])
self.assertEqual("quxFF", cfg["grault"])
def main_modules(action_name, args):
name = args.mode
# Cloud-init 'modules' stages are broken up into the following sub-stages
# 1. Ensure that the init object fetches its config without errors
# 2. Get the datasource from the init object, if it does
# not exist then that means the main_init stage never
# worked, and thus this stage can not run.
# 3. Construct the modules object
# 4. Adjust any subsequent logging/output redirections using
# the modules objects configuration
# 5. Run the modules for the given stage name
# 6. Done!
w_msg = welcome_format("%s:%s" % (action_name, name))
init = stages.Init(ds_deps=[], reporter=args.reporter)
# Stage 1
init.read_cfg(extract_fns(args))
# Stage 2
try:
init.fetch(existing="trust")
except sources.DataSourceNotFoundException:
# There was no datasource found, theres nothing to do
msg = (
"Can not apply stage %s, no datasource found! Likely bad "
"things to come!" % name
)
util.logexc(LOG, msg)
print_exc(msg)
if not args.force:
return [(msg)]
_maybe_persist_instance_data(init)
# Stage 3
mods = Modules(init, extract_fns(args), reporter=args.reporter)
# Stage 4
try:
LOG.debug("Closing stdin")
util.close_stdin()
util.fixup_output(mods.cfg, name)
except Exception:
util.logexc(LOG, "Failed to setup output redirection!")
if args.debug:
# Reset so that all the debug handlers are closed out
LOG.debug(
"Logging being reset, this logger may no longer be active shortly"
)
logging.resetLogging()
logging.setupLogging(mods.cfg)
apply_reporting_cfg(init.cfg)
# now that logging is setup and stdout redirected, send welcome
welcome(name, msg=w_msg)
# Stage 5
return run_module_section(mods, name, name)
def test_none_ds(self):
new_root = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, new_root)
self.replicateTestRoot("simple_ubuntu", new_root)
cfg = {
"datasource_list": ["None"],
"cloud_init_modules": ["write-files"],
"system_info": {
"paths": {
"run_dir": new_root
}
},
}
ud = helpers.readResource("user_data.1.txt")
cloud_cfg = safeyaml.dumps(cfg)
util.ensure_dir(os.path.join(new_root, "etc", "cloud"))
util.write_file(os.path.join(new_root, "etc", "cloud", "cloud.cfg"),
cloud_cfg)
self._patchIn(new_root)
# Now start verifying whats created
initer = stages.Init()
initer.read_cfg()
initer.initialize()
initer.fetch()
initer.datasource.userdata_raw = ud
initer.instancify()
initer.update()
initer.cloudify().run(
"consume_data",
initer.consume_data,
args=[PER_INSTANCE],
freq=PER_INSTANCE,
)
mirrors = initer.distro.get_option("package_mirrors")
self.assertEqual(1, len(mirrors))
mirror = mirrors[0]
self.assertEqual(mirror["arches"], ["i386", "amd64", "blah"])
mods = Modules(initer)
(which_ran, failures) = mods.run_section("cloud_init_modules")
self.assertTrue(len(failures) == 0)
self.assertTrue(os.path.exists("/etc/blah.ini"))
self.assertIn("write-files", which_ran)
contents = util.load_file("/etc/blah.ini")
self.assertEqual(contents, "blah")
def test_vendor_user_yaml_cloud_config(self):
vendor_blob = """
#cloud-config
a: b
name: vendor
run:
- x
- y
"""
user_blob = """
#cloud-config
a: c
vendor_data:
enabled: true
prefix: /bin/true
name: user
run:
- z
"""
self.reRoot()
initer = stages.Init()
initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob)
initer.read_cfg()
initer.initialize()
initer.fetch()
initer.instancify()
initer.update()
initer.cloudify().run(
"consume_data",
initer.consume_data,
args=[PER_INSTANCE],
freq=PER_INSTANCE,
)
mods = Modules(initer)
(_which_ran, _failures) = mods.run_section("cloud_init_modules")
cfg = mods.cfg
self.assertIn("vendor_data", cfg)
self.assertEqual("c", cfg["a"])
self.assertEqual("user", cfg["name"])
self.assertNotIn("x", cfg["run"])
self.assertNotIn("y", cfg["run"])
self.assertIn("z", cfg["run"])
def test_dont_allow_user_data(self, mock_cfg):
mock_cfg.return_value = {"allow_userdata": False}
# test that user-data is ignored but vendor-data is kept
user_blob = """
#cloud-config-jsonp
[
{ "op": "add", "path": "/baz", "value": "qux" },
{ "op": "add", "path": "/bar", "value": "qux2" }
]
"""
vendor_blob = """
#cloud-config-jsonp
[
{ "op": "add", "path": "/baz", "value": "quxA" },
{ "op": "add", "path": "/bar", "value": "quxB" },
{ "op": "add", "path": "/foo", "value": "quxC" }
]
"""
self.reRoot()
initer = stages.Init()
initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob)
initer.read_cfg()
initer.initialize()
initer.fetch()
initer.instancify()
initer.update()
initer.cloudify().run(
"consume_data",
initer.consume_data,
args=[PER_INSTANCE],
freq=PER_INSTANCE,
)
mods = Modules(initer)
(_which_ran, _failures) = mods.run_section("cloud_init_modules")
cfg = mods.cfg
self.assertIn("vendor_data", cfg)
self.assertEqual("quxA", cfg["baz"])
self.assertEqual("quxB", cfg["bar"])
self.assertEqual("quxC", cfg["foo"])
def test_simple_jsonp_no_vendor_consumed(self):
# make sure that vendor data is not consumed
user_blob = """
#cloud-config-jsonp
[
{ "op": "add", "path": "/baz", "value": "qux" },
{ "op": "add", "path": "/bar", "value": "qux2" },
{ "op": "add", "path": "/vendor_data", "value": {"enabled": "false"}}
]
"""
vendor_blob = """
#cloud-config-jsonp
[
{ "op": "add", "path": "/baz", "value": "quxA" },
{ "op": "add", "path": "/bar", "value": "quxB" },
{ "op": "add", "path": "/foo", "value": "quxC" }
]
"""
self.reRoot()
initer = stages.Init()
initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob)
initer.read_cfg()
initer.initialize()
initer.fetch()
initer.instancify()
initer.update()
initer.cloudify().run(
"consume_data",
initer.consume_data,
args=[PER_INSTANCE],
freq=PER_INSTANCE,
)
mods = Modules(initer)
(_which_ran, _failures) = mods.run_section("cloud_init_modules")
cfg = mods.cfg
self.assertEqual("qux", cfg["baz"])
self.assertEqual("qux2", cfg["bar"])
self.assertNotIn("foo", cfg)
def test_vendordata_script(self):
vendor_blob = """
#!/bin/bash
echo "test"
"""
vendor2_blob = """
#!/bin/bash
echo "dynamic test"
"""
user_blob = """
#cloud-config
vendor_data:
enabled: true
prefix: /bin/true
"""
new_root = self.reRoot()
initer = stages.Init()
initer.datasource = FakeDataSource(user_blob,
vendordata=vendor_blob,
vendordata2=vendor2_blob)
initer.read_cfg()
initer.initialize()
initer.fetch()
initer.instancify()
initer.update()
initer.cloudify().run(
"consume_data",
initer.consume_data,
args=[PER_INSTANCE],
freq=PER_INSTANCE,
)
mods = Modules(initer)
(_which_ran, _failures) = mods.run_section("cloud_init_modules")
vendor_script = initer.paths.get_ipath_cur("vendor_scripts")
vendor_script_fns = "%s%s/part-001" % (new_root, vendor_script)
self.assertTrue(os.path.exists(vendor_script_fns))
def test_none_ds_skips_modules_which_define_unmatched_distros(self):
"""Skip modules which define distros which don't match the current."""
initer = stages.Init()
initer.read_cfg()
initer.initialize()
initer.fetch()
initer.instancify()
initer.update()
initer.cloudify().run(
"consume_data",
initer.consume_data,
args=[PER_INSTANCE],
freq=PER_INSTANCE,
)
mods = Modules(initer)
(which_ran, failures) = mods.run_section("cloud_init_modules")
self.assertTrue(len(failures) == 0)
self.assertIn(
"Skipping modules 'spacewalk' because they are not verified on"
" distro 'ubuntu'",
self.logs.getvalue(),
)
self.assertNotIn("spacewalk", which_ran)
def main_single(name, args):
# Cloud-init single stage is broken up into the following sub-stages
# 1. Ensure that the init object fetches its config without errors
# 2. Attempt to fetch the datasource (warn if it doesn't work)
# 3. Construct the modules object
# 4. Adjust any subsequent logging/output redirections using
# the modules objects configuration
# 5. Run the single module
# 6. Done!
mod_name = args.name
w_msg = welcome_format(name)
init = stages.Init(ds_deps=[], reporter=args.reporter)
# Stage 1
init.read_cfg(extract_fns(args))
# Stage 2
try:
init.fetch(existing="trust")
except sources.DataSourceNotFoundException:
# There was no datasource found,
# that might be bad (or ok) depending on
# the module being ran (so continue on)
util.logexc(
LOG, "Failed to fetch your datasource, likely bad things to come!"
)
print_exc(
"Failed to fetch your datasource, likely bad things to come!"
)
if not args.force:
return 1
_maybe_persist_instance_data(init)
# Stage 3
mods = Modules(init, extract_fns(args), reporter=args.reporter)
mod_args = args.module_args
if mod_args:
LOG.debug("Using passed in arguments %s", mod_args)
mod_freq = args.frequency
if mod_freq:
LOG.debug("Using passed in frequency %s", mod_freq)
mod_freq = FREQ_SHORT_NAMES.get(mod_freq)
# Stage 4
try:
LOG.debug("Closing stdin")
util.close_stdin()
util.fixup_output(mods.cfg, None)
except Exception:
util.logexc(LOG, "Failed to setup output redirection!")
if args.debug:
# Reset so that all the debug handlers are closed out
LOG.debug(
"Logging being reset, this logger may no longer be active shortly"
)
logging.resetLogging()
logging.setupLogging(mods.cfg)
apply_reporting_cfg(init.cfg)
# now that logging is setup and stdout redirected, send welcome
welcome(name, msg=w_msg)
# Stage 5
(which_ran, failures) = mods.run_single(mod_name, mod_args, mod_freq)
if failures:
LOG.warning("Ran %s but it failed!", mod_name)
return 1
elif not which_ran:
LOG.warning("Did not run %s, does it exist?", mod_name)
return 1
else:
# Guess it worked
return 0
def main_init(name, args):
deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK]
if args.local:
deps = [sources.DEP_FILESYSTEM]
early_logs = [
attempt_cmdline_url(
path=os.path.join(
"%s.d" % CLOUD_CONFIG, "91_kernel_cmdline_url.cfg"
),
network=not args.local,
)
]
# Cloud-init 'init' stage is broken up into the following sub-stages
# 1. Ensure that the init object fetches its config without errors
# 2. Setup logging/output redirections with resultant config (if any)
# 3. Initialize the cloud-init filesystem
# 4. Check if we can stop early by looking for various files
# 5. Fetch the datasource
# 6. Connect to the current instance location + update the cache
# 7. Consume the userdata (handlers get activated here)
# 8. Construct the modules object
# 9. Adjust any subsequent logging/output redirections using the modules
# objects config as it may be different from init object
# 10. Run the modules for the 'init' stage
# 11. Done!
if not args.local:
w_msg = welcome_format(name)
else:
w_msg = welcome_format("%s-local" % (name))
init = stages.Init(ds_deps=deps, reporter=args.reporter)
# Stage 1
init.read_cfg(extract_fns(args))
# Stage 2
outfmt = None
errfmt = None
try:
early_logs.append((logging.DEBUG, "Closing stdin."))
util.close_stdin()
(outfmt, errfmt) = util.fixup_output(init.cfg, name)
except Exception:
msg = "Failed to setup output redirection!"
util.logexc(LOG, msg)
print_exc(msg)
early_logs.append((logging.WARN, msg))
if args.debug:
# Reset so that all the debug handlers are closed out
LOG.debug(
"Logging being reset, this logger may no longer be active shortly"
)
logging.resetLogging()
logging.setupLogging(init.cfg)
apply_reporting_cfg(init.cfg)
# Any log usage prior to setupLogging above did not have local user log
# config applied. We send the welcome message now, as stderr/out have
# been redirected and log now configured.
welcome(name, msg=w_msg)
# re-play early log messages before logging was setup
for lvl, msg in early_logs:
LOG.log(lvl, msg)
# Stage 3
try:
init.initialize()
except Exception:
util.logexc(LOG, "Failed to initialize, likely bad things to come!")
# Stage 4
path_helper = init.paths
purge_cache_on_python_version_change(init)
mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK
if mode == sources.DSMODE_NETWORK:
existing = "trust"
sys.stderr.write("%s\n" % (netinfo.debug_info()))
else:
existing = "check"
mcfg = util.get_cfg_option_bool(init.cfg, "manual_cache_clean", False)
if mcfg:
LOG.debug("manual cache clean set from config")
existing = "trust"
else:
mfile = path_helper.get_ipath_cur("manual_clean_marker")
if os.path.exists(mfile):
LOG.debug("manual cache clean found from marker: %s", mfile)
existing = "trust"
init.purge_cache()
# Stage 5
bring_up_interfaces = _should_bring_up_interfaces(init, args)
try:
init.fetch(existing=existing)
# if in network mode, and the datasource is local
# then work was done at that stage.
if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode:
LOG.debug(
"[%s] Exiting. datasource %s in local mode",
mode,
init.datasource,
)
return (None, [])
except sources.DataSourceNotFoundException:
# In the case of 'cloud-init init' without '--local' it is a bit
# more likely that the user would consider it failure if nothing was
# found.
if mode == sources.DSMODE_LOCAL:
LOG.debug("No local datasource found")
else:
util.logexc(
LOG, "No instance datasource found! Likely bad things to come!"
)
if not args.force:
init.apply_network_config(bring_up=bring_up_interfaces)
LOG.debug("[%s] Exiting without datasource", mode)
if mode == sources.DSMODE_LOCAL:
return (None, [])
else:
return (None, ["No instance datasource found."])
else:
LOG.debug(
"[%s] barreling on in force mode without datasource", mode
)
_maybe_persist_instance_data(init)
# Stage 6
iid = init.instancify()
LOG.debug(
"[%s] %s will now be targeting instance id: %s. new=%s",
mode,
name,
iid,
init.is_new_instance(),
)
if mode == sources.DSMODE_LOCAL:
# Before network comes up, set any configured hostname to allow
# dhcp clients to advertize this hostname to any DDNS services
# LP: #1746455.
_maybe_set_hostname(init, stage="local", retry_stage="network")
init.apply_network_config(bring_up=bring_up_interfaces)
if mode == sources.DSMODE_LOCAL:
if init.datasource.dsmode != mode:
LOG.debug(
"[%s] Exiting. datasource %s not in local mode.",
mode,
init.datasource,
)
return (init.datasource, [])
else:
LOG.debug(
"[%s] %s is in local mode, will apply init modules now.",
mode,
init.datasource,
)
# Give the datasource a chance to use network resources.
# This is used on Azure to communicate with the fabric over network.
init.setup_datasource()
# update fully realizes user-data (pulling in #include if necessary)
init.update()
_maybe_set_hostname(init, stage="init-net", retry_stage="modules:config")
# Stage 7
try:
# Attempt to consume the data per instance.
# This may run user-data handlers and/or perform
# url downloads and such as needed.
(ran, _results) = init.cloudify().run(
"consume_data",
init.consume_data,
args=[PER_INSTANCE],
freq=PER_INSTANCE,
)
if not ran:
# Just consume anything that is set to run per-always
# if nothing ran in the per-instance code
#
# See: https://bugs.launchpad.net/bugs/819507 for a little
# reason behind this...
init.consume_data(PER_ALWAYS)
except Exception:
util.logexc(LOG, "Consuming user data failed!")
return (init.datasource, ["Consuming user data failed!"])
# Validate user-data adheres to schema definition
if os.path.exists(init.paths.get_ipath_cur("userdata_raw")):
validate_cloudconfig_schema(
config=init.cfg, strict=False, log_details=False
)
else:
LOG.debug("Skipping user-data validation. No user-data found.")
apply_reporting_cfg(init.cfg)
# Stage 8 - re-read and apply relevant cloud-config to include user-data
mods = Modules(init, extract_fns(args), reporter=args.reporter)
# Stage 9
try:
outfmt_orig = outfmt
errfmt_orig = errfmt
(outfmt, errfmt) = util.get_output_cfg(mods.cfg, name)
if outfmt_orig != outfmt or errfmt_orig != errfmt:
LOG.warning("Stdout, stderr changing to (%s, %s)", outfmt, errfmt)
(outfmt, errfmt) = util.fixup_output(mods.cfg, name)
except Exception:
util.logexc(LOG, "Failed to re-adjust output redirection!")
logging.setupLogging(mods.cfg)
# give the activated datasource a chance to adjust
init.activate_datasource()
di_report_warn(datasource=init.datasource, cfg=init.cfg)
# Stage 10
return (init.datasource, run_module_section(mods, name, name))