def test_none_ds_forces_run_via_unverified_modules(self): """run_section forced skipped modules by using unverified_modules.""" # re-write cloud.cfg with unverified_modules override cfg = copy.deepcopy(self.cfg) cfg["unverified_modules"] = ["spacewalk"] # Would have skipped cloud_cfg = safeyaml.dumps(cfg) util.ensure_dir(os.path.join(self.new_root, "etc", "cloud")) util.write_file( os.path.join(self.new_root, "etc", "cloud", "cloud.cfg"), cloud_cfg) initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") self.assertTrue(len(failures) == 0) self.assertIn("spacewalk", which_ran) self.assertIn("running unverified_modules: 'spacewalk'", self.logs.getvalue())
def test_none_ds_run_with_no_config_modules(self): """run_section will report no modules run when none are configured.""" # re-write cloud.cfg with unverified_modules override cfg = copy.deepcopy(self.cfg) # Represent empty configuration in /etc/cloud/cloud.cfg cfg["cloud_init_modules"] = None cloud_cfg = safeyaml.dumps(cfg) util.ensure_dir(os.path.join(self.new_root, "etc", "cloud")) util.write_file( os.path.join(self.new_root, "etc", "cloud", "cloud.cfg"), cloud_cfg) initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") self.assertTrue(len(failures) == 0) self.assertEqual([], which_ran)
def test_vendordata_script(self): vendor_blob = ''' #!/bin/bash echo "test" ''' user_blob = ''' #cloud-config vendor_data: enabled: True prefix: /bin/true ''' new_root = self.reRoot() initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run('consume_data', initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) mods = stages.Modules(initer) (_which_ran, _failures) = mods.run_section('cloud_init_modules') vendor_script = initer.paths.get_ipath_cur('vendor_scripts') vendor_script_fns = "%s%s/part-001" % (new_root, vendor_script) self.assertTrue(os.path.exists(vendor_script_fns))
def test_none_ds_runs_modules_which_distros_all(self): """Skip modules which define distros attribute as supporting 'all'. This is done in the module with the declaration: distros = [ALL_DISTROS]. runcmd is an example. """ initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") self.assertTrue(len(failures) == 0) self.assertIn("runcmd", which_ran) self.assertNotIn( "Skipping modules 'runcmd' because they are not verified on" " distro 'ubuntu'", self.logs.getvalue(), )
def test_none_ds_forces_run_via_unverified_modules(self): """run_section forced skipped modules by using unverified_modules.""" # re-write cloud.cfg with unverified_modules override self.cfg['unverified_modules'] = ['spacewalk'] # Would have skipped cloud_cfg = util.yaml_dumps(self.cfg) util.ensure_dir(os.path.join(self.new_root, 'etc', 'cloud')) util.write_file( os.path.join(self.new_root, 'etc', 'cloud', 'cloud.cfg'), cloud_cfg) initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run('consume_data', initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section('cloud_init_modules') self.assertTrue(len(failures) == 0) self.assertIn('spacewalk', which_ran) self.assertIn("running unverified_modules: 'spacewalk'", self.logs.getvalue())
def test_none_ds_runs_modules_which_do_not_define_distros(self): """Any modules which do not define a distros attribute are run.""" initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") self.assertTrue(len(failures) == 0) self.assertTrue(os.path.exists("/etc/blah.ini")) self.assertIn("write-files", which_ran) contents = util.load_file("/etc/blah.ini") self.assertEqual(contents, "blah") self.assertNotIn( "Skipping modules ['write-files'] because they are not verified on" " distro 'ubuntu'", self.logs.getvalue(), )
def test_simple_jsonp_vendor_and_vendor2_and_user(self): # test that user-data wins over vendor user_blob = """ #cloud-config-jsonp [ { "op": "add", "path": "/baz", "value": "qux" }, { "op": "add", "path": "/bar", "value": "qux2" }, { "op": "add", "path": "/foobar", "value": "qux3" } ] """ vendor_blob = """ #cloud-config-jsonp [ { "op": "add", "path": "/baz", "value": "quxA" }, { "op": "add", "path": "/bar", "value": "quxB" }, { "op": "add", "path": "/foo", "value": "quxC" }, { "op": "add", "path": "/corge", "value": "quxEE" } ] """ vendor2_blob = """ #cloud-config-jsonp [ { "op": "add", "path": "/corge", "value": "quxD" }, { "op": "add", "path": "/grault", "value": "quxFF" }, { "op": "add", "path": "/foobar", "value": "quxGG" } ] """ self.reRoot() initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob, vendordata2=vendor2_blob) initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mods = stages.Modules(initer) (_which_ran, _failures) = mods.run_section("cloud_init_modules") cfg = mods.cfg self.assertIn("vendor_data", cfg) self.assertIn("vendor_data2", cfg) # Confirm that vendordata2 overrides vendordata, and that # userdata overrides both self.assertEqual("qux", cfg["baz"]) self.assertEqual("qux2", cfg["bar"]) self.assertEqual("qux3", cfg["foobar"]) self.assertEqual("quxC", cfg["foo"]) self.assertEqual("quxD", cfg["corge"]) self.assertEqual("quxFF", cfg["grault"])
def main_modules(action_name, args): name = args.mode # Cloud-init 'modules' stages are broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors # 2. Get the datasource from the init object, if it does # not exist then that means the main_init stage never # worked, and thus this stage can not run. # 3. Construct the modules object # 4. Adjust any subsequent logging/output redirections using # the modules objects configuration # 5. Run the modules for the given stage name # 6. Done! w_msg = welcome_format("%s:%s" % (action_name, name)) init = stages.Init(ds_deps=[], reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 try: init.fetch(existing="trust") except sources.DataSourceNotFoundException: # There was no datasource found, theres nothing to do msg = ( "Can not apply stage %s, no datasource found! Likely bad " "things to come!" % name ) util.logexc(LOG, msg) print_exc(msg) if not args.force: return [(msg)] _maybe_persist_instance_data(init) # Stage 3 mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) # Stage 4 try: LOG.debug("Closing stdin") util.close_stdin() util.fixup_output(mods.cfg, name) except Exception: util.logexc(LOG, "Failed to setup output redirection!") if args.debug: # Reset so that all the debug handlers are closed out LOG.debug( "Logging being reset, this logger may no longer be active shortly" ) logging.resetLogging() logging.setupLogging(mods.cfg) apply_reporting_cfg(init.cfg) # now that logging is setup and stdout redirected, send welcome welcome(name, msg=w_msg) # Stage 5 return run_module_section(mods, name, name)
def test_none_ds(self): new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, new_root) self.replicateTestRoot('simple_ubuntu', new_root) cfg = { 'datasource_list': ['None'], 'write_files': [ { 'path': '/etc/blah.ini', 'content': 'blah', 'permissions': 0o755, }, ], 'cloud_init_modules': ['write-files'], } cloud_cfg = util.yaml_dumps(cfg) util.ensure_dir(os.path.join(new_root, 'etc', 'cloud')) util.write_file(os.path.join(new_root, 'etc', 'cloud', 'cloud.cfg'), cloud_cfg) self._patchIn(new_root) # Now start verifying whats created initer = stages.Init() initer.read_cfg() initer.initialize() self.assertTrue(os.path.exists("/var/lib/cloud")) for d in ['scripts', 'seed', 'instances', 'handlers', 'sem', 'data']: self.assertTrue(os.path.isdir(os.path.join("/var/lib/cloud", d))) initer.fetch() iid = initer.instancify() self.assertEqual(iid, 'iid-datasource-none') initer.update() self.assertTrue(os.path.islink("var/lib/cloud/instance")) initer.cloudify().run('consume_data', initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section('cloud_init_modules') self.assertTrue(len(failures) == 0) self.assertTrue(os.path.exists('/etc/blah.ini')) self.assertIn('write-files', which_ran) contents = util.load_file('/etc/blah.ini') self.assertEqual(contents, 'blah')
def test_none_ds(self): new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, new_root) self.replicateTestRoot("simple_ubuntu", new_root) cfg = { "datasource_list": ["None"], "cloud_init_modules": ["write-files"], "system_info": { "paths": { "run_dir": new_root } }, } ud = helpers.readResource("user_data.1.txt") cloud_cfg = safeyaml.dumps(cfg) util.ensure_dir(os.path.join(new_root, "etc", "cloud")) util.write_file(os.path.join(new_root, "etc", "cloud", "cloud.cfg"), cloud_cfg) self._patchIn(new_root) # Now start verifying whats created initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.datasource.userdata_raw = ud initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mirrors = initer.distro.get_option("package_mirrors") self.assertEqual(1, len(mirrors)) mirror = mirrors[0] self.assertEqual(mirror["arches"], ["i386", "amd64", "blah"]) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") self.assertTrue(len(failures) == 0) self.assertTrue(os.path.exists("/etc/blah.ini")) self.assertIn("write-files", which_ran) contents = util.load_file("/etc/blah.ini") self.assertEqual(contents, "blah")
def test_vendor_user_yaml_cloud_config(self): vendor_blob = """ #cloud-config a: b name: vendor run: - x - y """ user_blob = """ #cloud-config a: c vendor_data: enabled: true prefix: /bin/true name: user run: - z """ self.reRoot() initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mods = stages.Modules(initer) (_which_ran, _failures) = mods.run_section("cloud_init_modules") cfg = mods.cfg self.assertIn("vendor_data", cfg) self.assertEqual("c", cfg["a"]) self.assertEqual("user", cfg["name"]) self.assertNotIn("x", cfg["run"]) self.assertNotIn("y", cfg["run"]) self.assertIn("z", cfg["run"])
def test_vendor_user_yaml_cloud_config(self): vendor_blob = ''' #cloud-config a: b name: vendor run: - x - y ''' user_blob = ''' #cloud-config a: c vendor_data: enabled: True prefix: /bin/true name: user run: - z ''' new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, new_root) self._patchIn(new_root) initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run('consume_data', initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) mods = stages.Modules(initer) (_which_ran, _failures) = mods.run_section('cloud_init_modules') cfg = mods.cfg self.assertIn('vendor_data', cfg) self.assertEqual('c', cfg['a']) self.assertEqual('user', cfg['name']) self.assertNotIn('x', cfg['run']) self.assertNotIn('y', cfg['run']) self.assertIn('z', cfg['run'])
def test_none_ds(self): new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, new_root) self.replicateTestRoot('simple_ubuntu', new_root) cfg = { 'datasource_list': ['None'], 'cloud_init_modules': ['write-files'], 'system_info': { 'paths': { 'run_dir': new_root } } } ud = helpers.readResource('user_data.1.txt') cloud_cfg = safeyaml.dumps(cfg) util.ensure_dir(os.path.join(new_root, 'etc', 'cloud')) util.write_file(os.path.join(new_root, 'etc', 'cloud', 'cloud.cfg'), cloud_cfg) self._patchIn(new_root) # Now start verifying whats created initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.datasource.userdata_raw = ud initer.instancify() initer.update() initer.cloudify().run('consume_data', initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) mirrors = initer.distro.get_option('package_mirrors') self.assertEqual(1, len(mirrors)) mirror = mirrors[0] self.assertEqual(mirror['arches'], ['i386', 'amd64', 'blah']) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section('cloud_init_modules') self.assertTrue(len(failures) == 0) self.assertTrue(os.path.exists('/etc/blah.ini')) self.assertIn('write-files', which_ran) contents = util.load_file('/etc/blah.ini') self.assertEqual(contents, 'blah')
def test_none_ds_skips_modules_which_define_unmatched_distros(self): """Skip modules which define distros which don't match the current.""" initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run('consume_data', initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section('cloud_init_modules') self.assertTrue(len(failures) == 0) self.assertIn( "Skipping modules 'spacewalk' because they are not verified on" " distro 'ubuntu'", self.logs.getvalue()) self.assertNotIn('spacewalk', which_ran)
def test_dont_allow_user_data(self, mock_cfg): mock_cfg.return_value = {"allow_userdata": False} # test that user-data is ignored but vendor-data is kept user_blob = """ #cloud-config-jsonp [ { "op": "add", "path": "/baz", "value": "qux" }, { "op": "add", "path": "/bar", "value": "qux2" } ] """ vendor_blob = """ #cloud-config-jsonp [ { "op": "add", "path": "/baz", "value": "quxA" }, { "op": "add", "path": "/bar", "value": "quxB" }, { "op": "add", "path": "/foo", "value": "quxC" } ] """ self.reRoot() initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mods = stages.Modules(initer) (_which_ran, _failures) = mods.run_section("cloud_init_modules") cfg = mods.cfg self.assertIn("vendor_data", cfg) self.assertEqual("quxA", cfg["baz"]) self.assertEqual("quxB", cfg["bar"]) self.assertEqual("quxC", cfg["foo"])
def test_simple_jsonp_no_vendor_consumed(self): # make sure that vendor data is not consumed user_blob = ''' #cloud-config-jsonp [ { "op": "add", "path": "/baz", "value": "qux" }, { "op": "add", "path": "/bar", "value": "qux2" }, { "op": "add", "path": "/vendor_data", "value": {"enabled": "false"}} ] ''' vendor_blob = ''' #cloud-config-jsonp [ { "op": "add", "path": "/baz", "value": "quxA" }, { "op": "add", "path": "/bar", "value": "quxB" }, { "op": "add", "path": "/foo", "value": "quxC" } ] ''' new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, new_root) self._patchIn(new_root) initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run('consume_data', initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) mods = stages.Modules(initer) (_which_ran, _failures) = mods.run_section('cloud_init_modules') cfg = mods.cfg self.assertEqual('qux', cfg['baz']) self.assertEqual('qux2', cfg['bar']) self.assertNotIn('foo', cfg)
def test_simple_jsonp_no_vendor_consumed(self): # make sure that vendor data is not consumed user_blob = """ #cloud-config-jsonp [ { "op": "add", "path": "/baz", "value": "qux" }, { "op": "add", "path": "/bar", "value": "qux2" }, { "op": "add", "path": "/vendor_data", "value": {"enabled": "false"}} ] """ vendor_blob = """ #cloud-config-jsonp [ { "op": "add", "path": "/baz", "value": "quxA" }, { "op": "add", "path": "/bar", "value": "quxB" }, { "op": "add", "path": "/foo", "value": "quxC" } ] """ self.reRoot() initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mods = stages.Modules(initer) (_which_ran, _failures) = mods.run_section("cloud_init_modules") cfg = mods.cfg self.assertEqual("qux", cfg["baz"]) self.assertEqual("qux2", cfg["bar"]) self.assertNotIn("foo", cfg)
def test_simple_jsonp_vendor_and_user(self): # test that user-data wins over vendor user_blob = ''' #cloud-config-jsonp [ { "op": "add", "path": "/baz", "value": "qux" }, { "op": "add", "path": "/bar", "value": "qux2" } ] ''' vendor_blob = ''' #cloud-config-jsonp [ { "op": "add", "path": "/baz", "value": "quxA" }, { "op": "add", "path": "/bar", "value": "quxB" }, { "op": "add", "path": "/foo", "value": "quxC" } ] ''' new_root = self.makeDir() self._patchIn(new_root) initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) initer.read_cfg() initer.initialize() initer.fetch() _iid = initer.instancify() initer.update() initer.cloudify().run('consume_data', initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) mods = stages.Modules(initer) (_which_ran, _failures) = mods.run_section('cloud_init_modules') cfg = mods.cfg self.assertIn('vendor_data', cfg) self.assertEquals('qux', cfg['baz']) self.assertEquals('qux2', cfg['bar']) self.assertEquals('quxC', cfg['foo'])
def main_single(name, args): # Cloud-init single stage is broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors # 2. Attempt to fetch the datasource (warn if it doesn't work) # 3. Construct the modules object # 4. Adjust any subsequent logging/output redirections using # the modules objects configuration # 5. Run the single module # 6. Done! mod_name = args.name w_msg = welcome_format(name) init = stages.Init(ds_deps=[], reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 try: init.fetch(existing="trust") except sources.DataSourceNotFoundException: # There was no datasource found, # that might be bad (or ok) depending on # the module being ran (so continue on) util.logexc(LOG, ("Failed to fetch your datasource," " likely bad things to come!")) print_exc(("Failed to fetch your datasource," " likely bad things to come!")) if not args.force: return 1 # Stage 3 mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) mod_args = args.module_args if mod_args: LOG.debug("Using passed in arguments %s", mod_args) mod_freq = args.frequency if mod_freq: LOG.debug("Using passed in frequency %s", mod_freq) mod_freq = FREQ_SHORT_NAMES.get(mod_freq) # Stage 4 try: LOG.debug("Closing stdin") util.close_stdin() util.fixup_output(mods.cfg, None) except Exception: util.logexc(LOG, "Failed to setup output redirection!") if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" " longer be active shortly")) logging.resetLogging() logging.setupLogging(mods.cfg) apply_reporting_cfg(init.cfg) # now that logging is setup and stdout redirected, send welcome welcome(name, msg=w_msg) # Stage 5 (which_ran, failures) = mods.run_single(mod_name, mod_args, mod_freq) if failures: LOG.warn("Ran %s but it failed!", mod_name) return 1 elif not which_ran: LOG.warn("Did not run %s, does it exist?", mod_name) return 1 else: # Guess it worked return 0
def main_init(name, args): deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] if args.local: deps = [sources.DEP_FILESYSTEM] early_logs = [ attempt_cmdline_url( path=os.path.join( "%s.d" % CLOUD_CONFIG, "91_kernel_cmdline_url.cfg" ), network=not args.local, ) ] # Cloud-init 'init' stage is broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors # 2. Setup logging/output redirections with resultant config (if any) # 3. Initialize the cloud-init filesystem # 4. Check if we can stop early by looking for various files # 5. Fetch the datasource # 6. Connect to the current instance location + update the cache # 7. Consume the userdata (handlers get activated here) # 8. Construct the modules object # 9. Adjust any subsequent logging/output redirections using the modules # objects config as it may be different from init object # 10. Run the modules for the 'init' stage # 11. Done! if not args.local: w_msg = welcome_format(name) else: w_msg = welcome_format("%s-local" % (name)) init = stages.Init(ds_deps=deps, reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 outfmt = None errfmt = None try: early_logs.append((logging.DEBUG, "Closing stdin.")) util.close_stdin() (outfmt, errfmt) = util.fixup_output(init.cfg, name) except Exception: msg = "Failed to setup output redirection!" util.logexc(LOG, msg) print_exc(msg) early_logs.append((logging.WARN, msg)) if args.debug: # Reset so that all the debug handlers are closed out LOG.debug( "Logging being reset, this logger may no longer be active shortly" ) logging.resetLogging() logging.setupLogging(init.cfg) apply_reporting_cfg(init.cfg) # Any log usage prior to setupLogging above did not have local user log # config applied. We send the welcome message now, as stderr/out have # been redirected and log now configured. welcome(name, msg=w_msg) # re-play early log messages before logging was setup for lvl, msg in early_logs: LOG.log(lvl, msg) # Stage 3 try: init.initialize() except Exception: util.logexc(LOG, "Failed to initialize, likely bad things to come!") # Stage 4 path_helper = init.paths purge_cache_on_python_version_change(init) mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK if mode == sources.DSMODE_NETWORK: existing = "trust" sys.stderr.write("%s\n" % (netinfo.debug_info())) LOG.debug( "Checking to see if files that we need already" " exist from a previous run that would allow us" " to stop early." ) # no-net is written by upstart cloud-init-nonet when network failed # to come up stop_files = [ os.path.join(path_helper.get_cpath("data"), "no-net"), ] existing_files = [] for fn in stop_files: if os.path.isfile(fn): existing_files.append(fn) if existing_files: LOG.debug( "[%s] Exiting. stop file %s existed", mode, existing_files ) return (None, []) else: LOG.debug( "Execution continuing, no previous run detected that" " would allow us to stop early." ) else: existing = "check" mcfg = util.get_cfg_option_bool(init.cfg, "manual_cache_clean", False) if mcfg: LOG.debug("manual cache clean set from config") existing = "trust" else: mfile = path_helper.get_ipath_cur("manual_clean_marker") if os.path.exists(mfile): LOG.debug("manual cache clean found from marker: %s", mfile) existing = "trust" init.purge_cache() # Delete the no-net file as well util.del_file(os.path.join(path_helper.get_cpath("data"), "no-net")) # Stage 5 bring_up_interfaces = _should_bring_up_interfaces(init, args) try: init.fetch(existing=existing) # if in network mode, and the datasource is local # then work was done at that stage. if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: LOG.debug( "[%s] Exiting. datasource %s in local mode", mode, init.datasource, ) return (None, []) except sources.DataSourceNotFoundException: # In the case of 'cloud-init init' without '--local' it is a bit # more likely that the user would consider it failure if nothing was # found. When using upstart it will also mentions job failure # in console log if exit code is != 0. if mode == sources.DSMODE_LOCAL: LOG.debug("No local datasource found") else: util.logexc( LOG, "No instance datasource found! Likely bad things to come!" ) if not args.force: init.apply_network_config(bring_up=bring_up_interfaces) LOG.debug("[%s] Exiting without datasource", mode) if mode == sources.DSMODE_LOCAL: return (None, []) else: return (None, ["No instance datasource found."]) else: LOG.debug( "[%s] barreling on in force mode without datasource", mode ) _maybe_persist_instance_data(init) # Stage 6 iid = init.instancify() LOG.debug( "[%s] %s will now be targeting instance id: %s. new=%s", mode, name, iid, init.is_new_instance(), ) if mode == sources.DSMODE_LOCAL: # Before network comes up, set any configured hostname to allow # dhcp clients to advertize this hostname to any DDNS services # LP: #1746455. _maybe_set_hostname(init, stage="local", retry_stage="network") init.apply_network_config(bring_up=bring_up_interfaces) if mode == sources.DSMODE_LOCAL: if init.datasource.dsmode != mode: LOG.debug( "[%s] Exiting. datasource %s not in local mode.", mode, init.datasource, ) return (init.datasource, []) else: LOG.debug( "[%s] %s is in local mode, will apply init modules now.", mode, init.datasource, ) # Give the datasource a chance to use network resources. # This is used on Azure to communicate with the fabric over network. init.setup_datasource() # update fully realizes user-data (pulling in #include if necessary) init.update() _maybe_set_hostname(init, stage="init-net", retry_stage="modules:config") # Stage 7 try: # Attempt to consume the data per instance. # This may run user-data handlers and/or perform # url downloads and such as needed. (ran, _results) = init.cloudify().run( "consume_data", init.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) if not ran: # Just consume anything that is set to run per-always # if nothing ran in the per-instance code # # See: https://bugs.launchpad.net/bugs/819507 for a little # reason behind this... init.consume_data(PER_ALWAYS) except Exception: util.logexc(LOG, "Consuming user data failed!") return (init.datasource, ["Consuming user data failed!"]) apply_reporting_cfg(init.cfg) # Stage 8 - re-read and apply relevant cloud-config to include user-data mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) # Stage 9 try: outfmt_orig = outfmt errfmt_orig = errfmt (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name) if outfmt_orig != outfmt or errfmt_orig != errfmt: LOG.warning("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) (outfmt, errfmt) = util.fixup_output(mods.cfg, name) except Exception: util.logexc(LOG, "Failed to re-adjust output redirection!") logging.setupLogging(mods.cfg) # give the activated datasource a chance to adjust init.activate_datasource() di_report_warn(datasource=init.datasource, cfg=init.cfg) # Stage 10 return (init.datasource, run_module_section(mods, name, name))
def main_init(name, args): deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] if args.local: deps = [sources.DEP_FILESYSTEM] if not args.local: # See doc/kernel-cmdline.txt # # This is used in maas datasource, in "ephemeral" (read-only root) # environment where the instance netboots to iscsi ro root. # and the entity that controls the pxe config has to configure # the maas datasource. # # Could be used elsewhere, only works on network based (not local). root_name = "%s.d" % (CLOUD_CONFIG) target_fn = os.path.join(root_name, "91_kernel_cmdline_url.cfg") util.read_write_cmdline_url(target_fn) # Cloud-init 'init' stage is broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors # 2. Setup logging/output redirections with resultant config (if any) # 3. Initialize the cloud-init filesystem # 4. Check if we can stop early by looking for various files # 5. Fetch the datasource # 6. Connect to the current instance location + update the cache # 7. Consume the userdata (handlers get activated here) # 8. Construct the modules object # 9. Adjust any subsequent logging/output redirections using the modules # objects config as it may be different from init object # 10. Run the modules for the 'init' stage # 11. Done! if not args.local: w_msg = welcome_format(name) else: w_msg = welcome_format("%s-local" % (name)) init = stages.Init(ds_deps=deps, reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 outfmt = None errfmt = None try: LOG.debug("Closing stdin") util.close_stdin() (outfmt, errfmt) = util.fixup_output(init.cfg, name) except Exception: util.logexc(LOG, "Failed to setup output redirection!") print_exc("Failed to setup output redirection!") if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" " longer be active shortly")) logging.resetLogging() logging.setupLogging(init.cfg) apply_reporting_cfg(init.cfg) # Any log usage prior to setupLogging above did not have local user log # config applied. We send the welcome message now, as stderr/out have # been redirected and log now configured. welcome(name, msg=w_msg) # Stage 3 try: init.initialize() except Exception: util.logexc(LOG, "Failed to initialize, likely bad things to come!") # Stage 4 path_helper = init.paths mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK if mode == sources.DSMODE_NETWORK: existing = "trust" sys.stderr.write("%s\n" % (netinfo.debug_info())) LOG.debug(("Checking to see if files that we need already" " exist from a previous run that would allow us" " to stop early.")) # no-net is written by upstart cloud-init-nonet when network failed # to come up stop_files = [ os.path.join(path_helper.get_cpath("data"), "no-net"), ] existing_files = [] for fn in stop_files: if os.path.isfile(fn): existing_files.append(fn) if existing_files: LOG.debug("[%s] Exiting. stop file %s existed", mode, existing_files) return (None, []) else: LOG.debug("Execution continuing, no previous run detected that" " would allow us to stop early.") else: existing = "check" if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): existing = "trust" init.purge_cache() # Delete the non-net file as well util.del_file(os.path.join(path_helper.get_cpath("data"), "no-net")) # Stage 5 try: init.fetch(existing=existing) # if in network mode, and the datasource is local # then work was done at that stage. if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: LOG.debug("[%s] Exiting. datasource %s in local mode", mode, init.datasource) return (None, []) except sources.DataSourceNotFoundException: # In the case of 'cloud-init init' without '--local' it is a bit # more likely that the user would consider it failure if nothing was # found. When using upstart it will also mentions job failure # in console log if exit code is != 0. if mode == sources.DSMODE_LOCAL: LOG.debug("No local datasource found") else: util.logexc(LOG, ("No instance datasource found!" " Likely bad things to come!")) if not args.force: init.apply_network_config(bring_up=not args.local) LOG.debug("[%s] Exiting without datasource in local mode", mode) if mode == sources.DSMODE_LOCAL: return (None, []) else: return (None, ["No instance datasource found."]) else: LOG.debug("[%s] barreling on in force mode without datasource", mode) # Stage 6 iid = init.instancify() LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", mode, name, iid, init.is_new_instance()) init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) if mode == sources.DSMODE_LOCAL: if init.datasource.dsmode != mode: LOG.debug("[%s] Exiting. datasource %s not in local mode.", mode, init.datasource) return (init.datasource, []) else: LOG.debug("[%s] %s is in local mode, will apply init modules now.", mode, init.datasource) # update fully realizes user-data (pulling in #include if necessary) init.update() # Stage 7 try: # Attempt to consume the data per instance. # This may run user-data handlers and/or perform # url downloads and such as needed. (ran, _results) = init.cloudify().run('consume_data', init.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) if not ran: # Just consume anything that is set to run per-always # if nothing ran in the per-instance code # # See: https://bugs.launchpad.net/bugs/819507 for a little # reason behind this... init.consume_data(PER_ALWAYS) except Exception: util.logexc(LOG, "Consuming user data failed!") return (init.datasource, ["Consuming user data failed!"]) apply_reporting_cfg(init.cfg) # Stage 8 - re-read and apply relevant cloud-config to include user-data mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) # Stage 9 try: outfmt_orig = outfmt errfmt_orig = errfmt (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name) if outfmt_orig != outfmt or errfmt_orig != errfmt: LOG.warn("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) (outfmt, errfmt) = util.fixup_output(mods.cfg, name) except Exception: util.logexc(LOG, "Failed to re-adjust output redirection!") logging.setupLogging(mods.cfg) # Stage 10 return (init.datasource, run_module_section(mods, name, name))
util.write_file(os.path.join(new_root, 'etc', 'cloud', 'cloud.cfg'), cloud_cfg) self._patchIn(new_root) # Now start verifying whats created initer = stages.Init() initer.read_cfg() initer.initialize() self.assertTrue(os.path.exists("/var/lib/cloud")) for d in ['scripts', 'seed', 'instances', 'handlers', 'sem', 'data']: self.assertTrue(os.path.isdir(os.path.join("/var/lib/cloud", d))) initer.fetch() iid = initer.instancify() self.assertEquals(iid, 'iid-datasource-none') initer.update() self.assertTrue(os.path.islink("var/lib/cloud/instance")) initer.cloudify().run('consume_data', initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section('cloud_init_modules') self.assertTrue(len(failures) == 0) self.assertTrue(os.path.exists('/etc/blah.ini')) self.assertIn('write-files', which_ran) contents = util.load_file('/etc/blah.ini') self.assertEquals(contents, 'blah')