def test_mixed_cloud_config(self): blob_cc = """ #cloud-config a: b c: d """ message_cc = MIMEBase("text", "cloud-config") message_cc.set_payload(blob_cc) blob_jp = """ #cloud-config-jsonp [ { "op": "replace", "path": "/a", "value": "c" }, { "op": "remove", "path": "/c" } ] """ message_jp = MIMEBase("text", "cloud-config-jsonp") message_jp.set_payload(blob_jp) message = MIMEMultipart() message.attach(message_cc) message.attach(message_jp) self.reRoot() ci = stages.Init() ci.datasource = FakeDataSource(str(message)) ci.fetch() ci.consume_data() cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) cc = util.load_yaml(cc_contents) self.assertEqual(1, len(cc)) self.assertEqual("c", cc["a"])
async def wait_for_cloudinit(self): if self.opts.dry_run: self.cloud_init_ok = True return ci_start = time.time() status_coro = arun_command(["cloud-init", "status", "--wait"]) try: status_cp = await asyncio.wait_for(status_coro, 600) except asyncio.CancelledError: status_txt = '<timeout>' self.cloud_init_ok = False else: status_txt = status_cp.stdout self.cloud_init_ok = True log.debug("waited %ss for cloud-init", time.time() - ci_start) if "status: done" in status_txt: log.debug("loading cloud config") init = stages.Init() init.read_cfg() init.fetch(existing="trust") self.cloud = init.cloudify() autoinstall_path = '/autoinstall.yaml' if 'autoinstall' in self.cloud.cfg: if not os.path.exists(autoinstall_path): atomic_helper.write_file( autoinstall_path, safeyaml.dumps( self.cloud.cfg['autoinstall']).encode('utf-8'), mode=0o600) if os.path.exists(autoinstall_path): self.opts.autoinstall = autoinstall_path else: log.debug( "cloud-init status: %r, assumed disabled", status_txt)
def test_none_ds_forces_run_via_unverified_modules(self): """run_section forced skipped modules by using unverified_modules.""" # re-write cloud.cfg with unverified_modules override cfg = copy.deepcopy(self.cfg) cfg["unverified_modules"] = ["spacewalk"] # Would have skipped cloud_cfg = safeyaml.dumps(cfg) util.ensure_dir(os.path.join(self.new_root, "etc", "cloud")) util.write_file( os.path.join(self.new_root, "etc", "cloud", "cloud.cfg"), cloud_cfg) initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") self.assertTrue(len(failures) == 0) self.assertIn("spacewalk", which_ran) self.assertIn("running unverified_modules: 'spacewalk'", self.logs.getvalue())
def test_include_bad_url_no_fail(self, mock_sleep): """Test #include with a bad URL and failure disabled""" bad_url = 'http://bad/forbidden' bad_data = '#cloud-config\nbad: true\n' httpretty.register_uri(httpretty.GET, bad_url, bad_data, status=403) included_url = 'http://hostname/path' included_data = '#cloud-config\nincluded: true\n' httpretty.register_uri(httpretty.GET, included_url, included_data) blob = '#include\n%s\n%s' % (bad_url, included_url) self.reRoot() ci = stages.Init() ci.datasource = FakeDataSource(blob) log_file = self.capture_log(logging.WARNING) ci.fetch() ci.consume_data() self.assertIn("403 Client Error: Forbidden for url: %s" % bad_url, log_file.getvalue()) cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) cc = util.load_yaml(cc_contents) self.assertIsNone(cc.get('bad')) self.assertTrue(cc.get('included'))
def test_none_ds_forces_run_via_unverified_modules(self): """run_section forced skipped modules by using unverified_modules.""" # re-write cloud.cfg with unverified_modules override self.cfg['unverified_modules'] = ['spacewalk'] # Would have skipped cloud_cfg = util.yaml_dumps(self.cfg) util.ensure_dir(os.path.join(self.new_root, 'etc', 'cloud')) util.write_file( os.path.join(self.new_root, 'etc', 'cloud', 'cloud.cfg'), cloud_cfg) initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run('consume_data', initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section('cloud_init_modules') self.assertTrue(len(failures) == 0) self.assertIn('spacewalk', which_ran) self.assertIn("running unverified_modules: 'spacewalk'", self.logs.getvalue())
def test_none_ds_runs_modules_which_distros_all(self): """Skip modules which define distros attribute as supporting 'all'. This is done in the module with the declaration: distros = [ALL_DISTROS]. runcmd is an example. """ initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") self.assertTrue(len(failures) == 0) self.assertIn("runcmd", which_ran) self.assertNotIn( "Skipping modules 'runcmd' because they are not verified on" " distro 'ubuntu'", self.logs.getvalue(), )
def test_mixed_cloud_config(self): blob_cc = ''' #cloud-config a: b c: d ''' message_cc = MIMEBase("text", "cloud-config") message_cc.set_payload(blob_cc) blob_jp = ''' #cloud-config-jsonp [ { "op": "replace", "path": "/a", "value": "c" }, { "op": "remove", "path": "/c" } ] ''' message_jp = MIMEBase('text', "cloud-config-jsonp") message_jp.set_payload(blob_jp) message = MIMEMultipart() message.attach(message_cc) message.attach(message_jp) ci = stages.Init() ci.datasource = FakeDataSource(str(message)) new_root = self.makeDir() self.patchUtils(new_root) self.patchOS(new_root) ci.fetch() ci.consume_data() cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) cc = util.load_yaml(cc_contents) self.assertEquals(1, len(cc)) self.assertEquals('c', cc['a'])
def test_none_ds_runs_modules_which_do_not_define_distros(self): """Any modules which do not define a distros attribute are run.""" initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") self.assertTrue(len(failures) == 0) self.assertTrue(os.path.exists("/etc/blah.ini")) self.assertIn("write-files", which_ran) contents = util.load_file("/etc/blah.ini") self.assertEqual(contents, "blah") self.assertNotIn( "Skipping modules ['write-files'] because they are not verified on" " distro 'ubuntu'", self.logs.getvalue(), )
def test_cloud_config_as_x_shell_script(self): blob_cc = ''' #cloud-config a: b c: d ''' message_cc = MIMEBase("text", "x-shellscript") message_cc.set_payload(blob_cc) blob_jp = ''' #cloud-config-jsonp [ { "op": "replace", "path": "/a", "value": "c" }, { "op": "remove", "path": "/c" } ] ''' message_jp = MIMEBase('text', "cloud-config-jsonp") message_jp.set_payload(blob_jp) message = MIMEMultipart() message.attach(message_cc) message.attach(message_jp) self.reRoot() ci = stages.Init() ci.datasource = FakeDataSource(str(message)) ci.fetch() ci.consume_data() cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) cc = util.load_yaml(cc_contents) self.assertEqual(1, len(cc)) self.assertEqual('c', cc['a'])
def test_vendordata_script(self): vendor_blob = ''' #!/bin/bash echo "test" ''' user_blob = ''' #cloud-config vendor_data: enabled: True prefix: /bin/true ''' new_root = self.reRoot() initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run('consume_data', initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) mods = stages.Modules(initer) (_which_ran, _failures) = mods.run_section('cloud_init_modules') vendor_script = initer.paths.get_ipath_cur('vendor_scripts') vendor_script_fns = "%s%s/part-001" % (new_root, vendor_script) self.assertTrue(os.path.exists(vendor_script_fns))
def test_mime_gzip_compressed(self): """Tests that individual message gzip encoding works.""" def gzip_part(text): return MIMEApplication(gzip_text(text), 'gzip') base_content1 = ''' #cloud-config a: 2 ''' base_content2 = ''' #cloud-config b: 3 c: 4 ''' message = MIMEMultipart('test') message.attach(gzip_part(base_content1)) message.attach(gzip_part(base_content2)) ci = stages.Init() ci.datasource = FakeDataSource(str(message)) self.reRoot() ci.fetch() ci.consume_data() contents = util.load_file(ci.paths.get_ipath("cloud_config")) contents = util.load_yaml(contents) self.assertTrue(isinstance(contents, dict)) self.assertEqual(3, len(contents)) self.assertEqual(2, contents['a']) self.assertEqual(3, contents['b']) self.assertEqual(4, contents['c'])
def test_cloud_config_archive(self): non_decodable = b'\x11\xc9\xb4gTH\xee\x12' data = [{ 'content': '#cloud-config\npassword: gocubs\n' }, { 'content': '#cloud-config\nlocale: chicago\n' }, { 'content': non_decodable }] message = b'#cloud-config-archive\n' + safeyaml.dumps(data).encode() self.reRoot() ci = stages.Init() ci.datasource = FakeDataSource(message) fs = {} def fsstore(filename, content, mode=0o0644, omode="wb"): fs[filename] = content # consuming the user-data provided should write 'cloud_config' file # which will have our yaml in it. with mock.patch('cloudinit.util.write_file') as mockobj: mockobj.side_effect = fsstore ci.fetch() ci.consume_data() cfg = util.load_yaml(fs[ci.paths.get_ipath("cloud_config")]) self.assertEqual(cfg.get('password'), 'gocubs') self.assertEqual(cfg.get('locale'), 'chicago')
def test_none_ds_run_with_no_config_modules(self): """run_section will report no modules run when none are configured.""" # re-write cloud.cfg with unverified_modules override cfg = copy.deepcopy(self.cfg) # Represent empty configuration in /etc/cloud/cloud.cfg cfg["cloud_init_modules"] = None cloud_cfg = safeyaml.dumps(cfg) util.ensure_dir(os.path.join(self.new_root, "etc", "cloud")) util.write_file( os.path.join(self.new_root, "etc", "cloud", "cloud.cfg"), cloud_cfg) initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") self.assertTrue(len(failures) == 0) self.assertEqual([], which_ran)
def setUp(self): super(TestInit, self).setUp() self.tmpdir = self.tmp_dir() self.init = stages.Init() # Setup fake Paths for Init to reference self.init._cfg = {'system_info': { 'distro': 'ubuntu', 'paths': {'cloud_dir': self.tmpdir, 'run_dir': self.tmpdir}}} self.init.datasource = FakeDataSource(paths=self.init.paths)
def test_simple_jsonp_vendor_and_vendor2_and_user(self): # test that user-data wins over vendor user_blob = """ #cloud-config-jsonp [ { "op": "add", "path": "/baz", "value": "qux" }, { "op": "add", "path": "/bar", "value": "qux2" }, { "op": "add", "path": "/foobar", "value": "qux3" } ] """ vendor_blob = """ #cloud-config-jsonp [ { "op": "add", "path": "/baz", "value": "quxA" }, { "op": "add", "path": "/bar", "value": "quxB" }, { "op": "add", "path": "/foo", "value": "quxC" }, { "op": "add", "path": "/corge", "value": "quxEE" } ] """ vendor2_blob = """ #cloud-config-jsonp [ { "op": "add", "path": "/corge", "value": "quxD" }, { "op": "add", "path": "/grault", "value": "quxFF" }, { "op": "add", "path": "/foobar", "value": "quxGG" } ] """ self.reRoot() initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob, vendordata2=vendor2_blob) initer.read_cfg() initer.initialize() initer.fetch() initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mods = stages.Modules(initer) (_which_ran, _failures) = mods.run_section("cloud_init_modules") cfg = mods.cfg self.assertIn("vendor_data", cfg) self.assertIn("vendor_data2", cfg) # Confirm that vendordata2 overrides vendordata, and that # userdata overrides both self.assertEqual("qux", cfg["baz"]) self.assertEqual("qux2", cfg["bar"]) self.assertEqual("qux3", cfg["foobar"]) self.assertEqual("quxC", cfg["foo"]) self.assertEqual("quxD", cfg["corge"]) self.assertEqual("quxFF", cfg["grault"])
def main(): log.setupLogging() init = stages.Init() s = DataSourceVmwareGuestinfo(init.cfg, init.distro, init.paths) if s.get_data(): print("Found data") else: print("Didn't find data") print("userdata_raw: %r" % s.userdata_raw) print("metadata: %r" % s.metadata)
def init(self, paths): """A fixture which yields a stages.Init instance with paths and cfg set As it is replaced with a mock, consumers of this fixture can set `init._cfg` if the default empty dict configuration is not appropriate. """ with mock.patch("cloudinit.stages.util.ensure_dirs"): init = stages.Init() init._cfg = {} init._paths = paths yield init
def main_modules(action_name, args): name = args.mode # Cloud-init 'modules' stages are broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors # 2. Get the datasource from the init object, if it does # not exist then that means the main_init stage never # worked, and thus this stage can not run. # 3. Construct the modules object # 4. Adjust any subsequent logging/output redirections using # the modules objects configuration # 5. Run the modules for the given stage name # 6. Done! w_msg = welcome_format("%s:%s" % (action_name, name)) init = stages.Init(ds_deps=[], reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 try: init.fetch(existing="trust") except sources.DataSourceNotFoundException: # There was no datasource found, theres nothing to do msg = ( "Can not apply stage %s, no datasource found! Likely bad " "things to come!" % name ) util.logexc(LOG, msg) print_exc(msg) if not args.force: return [(msg)] _maybe_persist_instance_data(init) # Stage 3 mods = Modules(init, extract_fns(args), reporter=args.reporter) # Stage 4 try: LOG.debug("Closing stdin") util.close_stdin() util.fixup_output(mods.cfg, name) except Exception: util.logexc(LOG, "Failed to setup output redirection!") if args.debug: # Reset so that all the debug handlers are closed out LOG.debug( "Logging being reset, this logger may no longer be active shortly" ) logging.resetLogging() logging.setupLogging(mods.cfg) apply_reporting_cfg(init.cfg) # now that logging is setup and stdout redirected, send welcome welcome(name, msg=w_msg) # Stage 5 return run_module_section(mods, name, name)
def test_none_ds_populates_var_lib_cloud(self): """Init and run_section default behavior creates appropriate dirs.""" # Now start verifying whats created initer = stages.Init() initer.read_cfg() initer.initialize() self.assertTrue(os.path.exists("/var/lib/cloud")) for d in ["scripts", "seed", "instances", "handlers", "sem", "data"]: self.assertTrue(os.path.isdir(os.path.join("/var/lib/cloud", d))) initer.fetch() iid = initer.instancify() self.assertEqual(iid, "iid-datasource-none") initer.update() self.assertTrue(os.path.islink("var/lib/cloud/instance"))
def test_mime_text_plain(self): """Mime message of type text/plain is ignored but shows warning.""" ci = stages.Init() message = MIMEBase("text", "plain") message.set_payload("Just text") ci.datasource = FakeDataSource(message.as_string().encode()) with mock.patch('cloudinit.util.write_file') as mockobj: log_file = self.capture_log(logging.WARNING) ci.fetch() ci.consume_data() self.assertIn("Unhandled unknown content-type (text/plain)", log_file.getvalue()) mockobj.assert_called_once_with(ci.paths.get_ipath("cloud_config"), "", 0o600)
def setup(self, tmpdir): self.tmpdir = tmpdir self.init = stages.Init() self.init._cfg = { "system_info": { "distro": "ubuntu", "paths": { "cloud_dir": self.tmpdir, "run_dir": self.tmpdir }, } } self.init.datasource = FakeDataSource(paths=self.init.paths) self._real_is_new_instance = self.init.is_new_instance self.init.is_new_instance = mock.Mock(return_value=True)
def test_include(self, mock_sleep): """Test #include.""" included_url = 'http://hostname/path' included_data = '#cloud-config\nincluded: true\n' httpretty.register_uri(httpretty.GET, included_url, included_data) blob = '#include\n%s\n' % included_url self.reRoot() ci = stages.Init() ci.datasource = FakeDataSource(blob) ci.fetch() ci.consume_data() cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) cc = util.load_yaml(cc_contents) self.assertTrue(cc.get('included'))
def test_unhandled_type_warning(self): """Raw text without magic is ignored but shows warning.""" ci = stages.Init() data = "arbitrary text\n" ci.datasource = FakeDataSource(data) with mock.patch('cloudinit.util.write_file') as mockobj: log_file = self.capture_log(logging.WARNING) ci.fetch() ci.consume_data() self.assertIn( "Unhandled non-multipart (text/x-not-multipart) userdata:", log_file.getvalue()) mockobj.assert_called_once_with(ci.paths.get_ipath("cloud_config"), "", 0o600)
def test_none_ds(self): new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, new_root) self.replicateTestRoot('simple_ubuntu', new_root) cfg = { 'datasource_list': ['None'], 'write_files': [ { 'path': '/etc/blah.ini', 'content': 'blah', 'permissions': 0o755, }, ], 'cloud_init_modules': ['write-files'], } cloud_cfg = util.yaml_dumps(cfg) util.ensure_dir(os.path.join(new_root, 'etc', 'cloud')) util.write_file(os.path.join(new_root, 'etc', 'cloud', 'cloud.cfg'), cloud_cfg) self._patchIn(new_root) # Now start verifying whats created initer = stages.Init() initer.read_cfg() initer.initialize() self.assertTrue(os.path.exists("/var/lib/cloud")) for d in ['scripts', 'seed', 'instances', 'handlers', 'sem', 'data']: self.assertTrue(os.path.isdir(os.path.join("/var/lib/cloud", d))) initer.fetch() iid = initer.instancify() self.assertEqual(iid, 'iid-datasource-none') initer.update() self.assertTrue(os.path.islink("var/lib/cloud/instance")) initer.cloudify().run('consume_data', initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section('cloud_init_modules') self.assertTrue(len(failures) == 0) self.assertTrue(os.path.exists('/etc/blah.ini')) self.assertIn('write-files', which_ran) contents = util.load_file('/etc/blah.ini') self.assertEqual(contents, 'blah')
def test_mime_text_plain(self): """Mime message of type text/plain is ignored but shows warning.""" ci = stages.Init() message = MIMEBase("text", "plain") message.set_payload("Just text") ci.datasource = FakeDataSource(message.as_string()) mock_write = self.mocker.replace("cloudinit.util.write_file", passthrough=False) mock_write(ci.paths.get_ipath("cloud_config"), "", 0600) self.mocker.replay() log_file = self.capture_log(logging.WARNING) ci.fetch() ci.consume_userdata() self.assertIn("Unhandled unknown content-type (text/plain)", log_file.getvalue())
def test_mime_application_octet_stream(self): """Mime type application/octet-stream is ignored but shows warning.""" ci = stages.Init() message = MIMEBase("application", "octet-stream") message.set_payload(b'\xbf\xe6\xb2\xc3\xd3\xba\x13\xa4\xd8\xa1\xcc') encoders.encode_base64(message) ci.datasource = FakeDataSource(message.as_string().encode()) with mock.patch('cloudinit.util.write_file') as mockobj: log_file = self.capture_log(logging.WARNING) ci.fetch() ci.consume_data() self.assertIn( "Unhandled unknown content-type (application/octet-stream)", log_file.getvalue()) mockobj.assert_called_once_with(ci.paths.get_ipath("cloud_config"), "", 0o600)
def setUp(self): super(TestInit, self).setUp() self.tmpdir = self.tmp_dir() self.init = stages.Init() # Setup fake Paths for Init to reference self.init._cfg = { "system_info": { "distro": "ubuntu", "paths": { "cloud_dir": self.tmpdir, "run_dir": self.tmpdir }, } } self.init.datasource = FakeDataSource(paths=self.init.paths) self._real_is_new_instance = self.init.is_new_instance self.init.is_new_instance = mock.Mock(return_value=True)
def test_unhandled_type_warning(self): """Raw text without magic is ignored but shows warning.""" ci = stages.Init() data = "arbitrary text\n" ci.datasource = FakeDataSource(data) mock_write = self.mocker.replace("cloudinit.util.write_file", passthrough=False) mock_write(ci.paths.get_ipath("cloud_config"), "", 0600) self.mocker.replay() log_file = self.capture_log(logging.WARNING) ci.fetch() ci.consume_data() self.assertIn( "Unhandled non-multipart (text/x-not-multipart) userdata:", log_file.getvalue())
def test_shellscript(self): """Raw text starting #!/bin/sh is treated as script.""" ci = stages.Init() script = "#!/bin/sh\necho hello\n" ci.datasource = FakeDataSource(script) outpath = os.path.join(ci.paths.get_ipath_cur("scripts"), "part-001") mock_write = self.mocker.replace("cloudinit.util.write_file", passthrough=False) mock_write(ci.paths.get_ipath("cloud_config"), "", 0600) mock_write(outpath, script, 0700) self.mocker.replay() log_file = self.capture_log(logging.WARNING) ci.fetch() ci.consume_data() self.assertEqual("", log_file.getvalue())
def test_none_ds(self): new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, new_root) self.replicateTestRoot("simple_ubuntu", new_root) cfg = { "datasource_list": ["None"], "cloud_init_modules": ["write-files"], "system_info": { "paths": { "run_dir": new_root } }, } ud = helpers.readResource("user_data.1.txt") cloud_cfg = safeyaml.dumps(cfg) util.ensure_dir(os.path.join(new_root, "etc", "cloud")) util.write_file(os.path.join(new_root, "etc", "cloud", "cloud.cfg"), cloud_cfg) self._patchIn(new_root) # Now start verifying whats created initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() initer.datasource.userdata_raw = ud initer.instancify() initer.update() initer.cloudify().run( "consume_data", initer.consume_data, args=[PER_INSTANCE], freq=PER_INSTANCE, ) mirrors = initer.distro.get_option("package_mirrors") self.assertEqual(1, len(mirrors)) mirror = mirrors[0] self.assertEqual(mirror["arches"], ["i386", "amd64", "blah"]) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") self.assertTrue(len(failures) == 0) self.assertTrue(os.path.exists("/etc/blah.ini")) self.assertIn("write-files", which_ran) contents = util.load_file("/etc/blah.ini") self.assertEqual(contents, "blah")