예제 #1
0
def load_json_or_yaml(data):
    if not data:
        return {}
    try:
        return util.load_json(data)
    except (json.JSONDecodeError, TypeError):
        return util.load_yaml(data)
예제 #2
0
    def test_cloud_config_archive(self):
        non_decodable = b'\x11\xc9\xb4gTH\xee\x12'
        data = [{'content': '#cloud-config\npassword: gocubs\n'},
                {'content': '#cloud-config\nlocale: chicago\n'},
                {'content': non_decodable}]
        message = b'#cloud-config-archive\n' + util.yaml_dumps(data).encode()

        ci = stages.Init()
        ci.datasource = FakeDataSource(message)

        fs = {}

        def fsstore(filename, content, mode=0o0644, omode="wb"):
            fs[filename] = content

        # consuming the user-data provided should write 'cloud_config' file
        # which will have our yaml in it.
        with mock.patch('cloudinit.util.write_file') as mockobj:
            mockobj.side_effect = fsstore
            ci.fetch()
            ci.consume_data()

        cfg = util.load_yaml(fs[ci.paths.get_ipath("cloud_config")])
        self.assertEqual(cfg.get('password'), 'gocubs')
        self.assertEqual(cfg.get('locale'), 'chicago')
예제 #3
0
 def cloud_config(self):
     """
     get the cloud-config used by the test
     """
     if not self._cloud_config:
         self._cloud_config = c_util.load_yaml(self.conf)
     return self._cloud_config
예제 #4
0
 def test_none_returns_default(self):
     """If yaml.load returns None, then default should be returned."""
     blobs = ("", " ", "# foo\n", "#")
     mdef = self.mydefault
     self.assertEqual([(b, self.mydefault) for b in blobs],
                      [(b, util.load_yaml(blob=b, default=mdef))
                       for b in blobs])
예제 #5
0
 def check_instance_id(self, sys_cfg) -> str:
     """Return True if instance_id unchanged."""
     response = read_metadata(metadata_only=True)
     md = response.get("meta-data", {})
     if not isinstance(md, dict):
         md = util.load_yaml(md)
     return md.get("instance-id") == self.metadata.get("instance-id")
예제 #6
0
    def test_merging_cloud_config(self):
        blob = '''
#cloud-config
a: b
e: f
run:
 - b
 - c
'''
        message1 = MIMEBase("text", "cloud-config")
        message1.set_payload(blob)

        blob2 = '''
#cloud-config
a: e
e: g
run:
 - stuff
 - morestuff
'''
        message2 = MIMEBase("text", "cloud-config")
        message2['X-Merge-Type'] = ('dict(recurse_array,'
                                    'recurse_str)+list(append)+str(append)')
        message2.set_payload(blob2)

        blob3 = '''
#cloud-config
e:
 - 1
 - 2
 - 3
p: 1
'''
        message3 = MIMEBase("text", "cloud-config")
        message3.set_payload(blob3)

        messages = [message1, message2, message3]

        paths = c_helpers.Paths({}, ds=FakeDataSource(''))
        cloud_cfg = handlers.cloud_config.CloudConfigPartHandler(paths)

        new_root = self.makeDir()
        self.patchUtils(new_root)
        self.patchOS(new_root)
        cloud_cfg.handle_part(None, handlers.CONTENT_START, None, None, None,
                              None)
        for i, m in enumerate(messages):
            headers = dict(m)
            fn = "part-%s" % (i + 1)
            payload = m.get_payload(decode=True)
            cloud_cfg.handle_part(None, headers['Content-Type'],
                                  fn, payload, None, headers)
        cloud_cfg.handle_part(None, handlers.CONTENT_END, None, None, None,
                              None)
        contents = util.load_file(paths.get_ipath('cloud_config'))
        contents = util.load_yaml(contents)
        self.assertEquals(contents['run'], ['b', 'c', 'stuff', 'morestuff'])
        self.assertEquals(contents['a'], 'be')
        self.assertEquals(contents['e'], [1, 2, 3])
        self.assertEquals(contents['p'], 1)
예제 #7
0
def _raw_instance_data_to_dict(metadata_type: str, metadata_value) -> dict:
    """Convert raw instance data from str, bytes, YAML to dict

    :param metadata_type: string, one of as: meta-data, vendor-data, user-data
        network-config

    :param metadata_value: str, bytes or dict representing or instance-data.

    :raises: InvalidMetaDataError on invalid instance-data content.
    """
    if isinstance(metadata_value, dict):
        return metadata_value
    if metadata_value is None:
        return {}
    try:
        parsed_metadata = util.load_yaml(metadata_value)
    except AttributeError as exc:  # not str or bytes
        raise sources.InvalidMetaDataException(
            "Invalid {md_type}. Expected str, bytes or dict but found:"
            " {value}".format(md_type=metadata_type,
                              value=metadata_value)) from exc
    if parsed_metadata is None:
        raise sources.InvalidMetaDataException(
            "Invalid {md_type} format. Expected YAML but found:"
            " {value}".format(md_type=metadata_type, value=metadata_value))
    return parsed_metadata
예제 #8
0
    def _get_data(self) -> bool:
        """Crawl LXD socket API instance data and return True on success"""
        if not self._is_platform_viable():
            LOG.debug("Not an LXD datasource: No LXD socket found.")
            return False

        self._crawled_metadata = util.log_time(logfunc=LOG.debug,
                                               msg='Crawl of metadata service',
                                               func=read_metadata)
        self.metadata = _raw_instance_data_to_dict(
            "meta-data", self._crawled_metadata.get("meta-data"))
        config = self._crawled_metadata.get("config", {})
        user_metadata = config.get("user.meta-data", {})
        if user_metadata:
            user_metadata = _raw_instance_data_to_dict("user.meta-data",
                                                       user_metadata)
        if not isinstance(self.metadata, dict):
            self.metadata = util.mergemanydict(
                [util.load_yaml(self.metadata), user_metadata])
        if "user-data" in self._crawled_metadata:
            self.userdata_raw = self._crawled_metadata["user-data"]
        if "network-config" in self._crawled_metadata:
            self._network_config = _maybe_remove_top_network(
                _raw_instance_data_to_dict(
                    "network-config",
                    self._crawled_metadata["network-config"]))
        if "vendor-data" in self._crawled_metadata:
            self.vendordata_raw = self._crawled_metadata["vendor-data"]
        return True
예제 #9
0
    def test_variant_sets_distro_in_cloud_cfg_subp(self, tmpdir):
        outfile = tmpdir.join("outcfg").strpath

        subp.subp(self.cmd + ["--variant", "ubuntu", self.tmpl_path, outfile])
        with open(outfile) as stream:
            system_cfg = util.load_yaml(stream.read())
        assert system_cfg["system_info"]["distro"] == "ubuntu"
예제 #10
0
def _merge_new_seed(cur, seeded):
    ret = cur.copy()

    newmd = seeded.get('meta-data', {})
    if not isinstance(seeded['meta-data'], dict):
        newmd = util.load_yaml(seeded['meta-data'])
    ret['meta-data'] = util.mergemanydict([cur['meta-data'], newmd])

    if seeded.get('network-config'):
        ret['network-config'] = util.load_yaml(seeded['network-config'])

    if 'user-data' in seeded:
        ret['user-data'] = seeded['user-data']
    if 'vendor-data' in seeded:
        ret['vendor-data'] = seeded['vendor-data']
    return ret
예제 #11
0
def _has_proper_console_support():
    stdout, _ = subp(['lxc', 'info'])
    info = load_yaml(stdout)
    reason = None
    if 'console' not in info.get('api_extensions', []):
        reason = "LXD server does not support console api extension"
    else:
        dver = info.get('environment', {}).get('driver_version', "")
        if dver.startswith("2.") or dver.startwith("1."):
            reason = "LXD Driver version not 3.x+ (%s)" % dver
        else:
            try:
                stdout, stderr = subp(['lxc', 'console', '--help'],
                                      decode=False)
                if not (b'console' in stdout and b'log' in stdout):
                    reason = "no '--log' in lxc console --help"
            except ProcessExecutionError as e:
                reason = "no 'console' command in lxc client"

    if reason:
        LOG.debug("no console-support: %s", reason)
        return False
    else:
        LOG.debug("console-support looks good")
        return True
예제 #12
0
def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):
    if cmdline is None:
        cmdline = util.get_cmdline()

    if files is None:
        files = _get_klibc_net_cfg_files()

    if 'network-config=' in cmdline:
        data64 = None
        for tok in cmdline.split():
            if tok.startswith("network-config="):
                data64 = tok.split("=", 1)[1]
        if data64:
            return util.load_yaml(_b64dgz(data64))

    if not _is_initramfs_netconfig(files, cmdline):
        return None

    if mac_addrs is None:
        mac_addrs = {}
        for k in get_devicelist():
            mac_addr = read_sys_net_safe(k, 'address')
            if mac_addr:
                mac_addrs[k] = mac_addr

    return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs)
예제 #13
0
    def test_mime_gzip_compressed(self):
        """Tests that individual message gzip encoding works."""
        def gzip_part(text):
            return MIMEApplication(gzip_text(text), 'gzip')

        base_content1 = '''
#cloud-config
a: 2
'''

        base_content2 = '''
#cloud-config
b: 3
c: 4
'''

        message = MIMEMultipart('test')
        message.attach(gzip_part(base_content1))
        message.attach(gzip_part(base_content2))
        ci = stages.Init()
        ci.datasource = FakeDataSource(str(message))
        new_root = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, new_root)
        self.patchUtils(new_root)
        self.patchOS(new_root)
        ci.fetch()
        ci.consume_data()
        contents = util.load_file(ci.paths.get_ipath("cloud_config"))
        contents = util.load_yaml(contents)
        self.assertTrue(isinstance(contents, dict))
        self.assertEquals(3, len(contents))
        self.assertEquals(2, contents['a'])
        self.assertEquals(3, contents['b'])
        self.assertEquals(4, contents['c'])
예제 #14
0
    def test_include_bad_url(self, mock_sleep):
        """Test #include with a bad URL."""
        bad_url = 'http://bad/forbidden'
        bad_data = '#cloud-config\nbad: true\n'
        httpretty.register_uri(httpretty.GET, bad_url, bad_data, status=403)

        included_url = 'http://hostname/path'
        included_data = '#cloud-config\nincluded: true\n'
        httpretty.register_uri(httpretty.GET, included_url, included_data)

        blob = '#include\n%s\n%s' % (bad_url, included_url)

        self.reRoot()
        ci = stages.Init()
        ci.datasource = FakeDataSource(blob)
        log_file = self.capture_log(logging.WARNING)
        ci.fetch()
        ci.consume_data()

        self.assertIn("403 Client Error: Forbidden for url: %s" % bad_url,
                      log_file.getvalue())

        cc_contents = util.load_file(ci.paths.get_ipath("cloud_config"))
        cc = util.load_yaml(cc_contents)
        self.assertIsNone(cc.get('bad'))
        self.assertTrue(cc.get('included'))
def _merge_new_seed(cur, seeded):
    ret = cur.copy()

    newmd = seeded.get('meta-data', {})
    if not isinstance(seeded['meta-data'], dict):
        newmd = util.load_yaml(seeded['meta-data'])
    ret['meta-data'] = util.mergemanydict([cur['meta-data'], newmd])

    if seeded.get('network-config'):
        ret['network-config'] = util.load_yaml(seeded['network-config'])

    if 'user-data' in seeded:
        ret['user-data'] = seeded['user-data']
    if 'vendor-data' in seeded:
        ret['vendor-data'] = seeded['vendor-data']
    return ret
예제 #16
0
 def test_nonallowed_returns_default(self):
     # for now, anything not in the allowed list just returns the default.
     myyaml = yaml.dump({'1': "one"})
     self.assertEqual(util.load_yaml(blob=myyaml,
                                     default=self.mydefault,
                                     allowed=(str,)),
                      self.mydefault)
예제 #17
0
 def test_python_unicode(self):
     # complex type of python/unicde is explicitly allowed
     myobj = {'1': unicode("FOOBAR")}
     safe_yaml = yaml.dump(myobj)
     self.assertEqual(util.load_yaml(blob=safe_yaml,
                                     default=self.mydefault),
                      myobj)
예제 #18
0
    def test_cloud_config_archive(self):
        non_decodable = b'\x11\xc9\xb4gTH\xee\x12'
        data = [{
            'content': '#cloud-config\npassword: gocubs\n'
        }, {
            'content': '#cloud-config\nlocale: chicago\n'
        }, {
            'content': non_decodable
        }]
        message = b'#cloud-config-archive\n' + util.yaml_dumps(data).encode()

        ci = stages.Init()
        ci.datasource = FakeDataSource(message)

        fs = {}

        def fsstore(filename, content, mode=0o0644, omode="wb"):
            fs[filename] = content

        # consuming the user-data provided should write 'cloud_config' file
        # which will have our yaml in it.
        with mock.patch('cloudinit.util.write_file') as mockobj:
            mockobj.side_effect = fsstore
            ci.fetch()
            ci.consume_data()

        cfg = util.load_yaml(fs[ci.paths.get_ipath("cloud_config")])
        self.assertEqual(cfg.get('password'), 'gocubs')
        self.assertEqual(cfg.get('locale'), 'chicago')
예제 #19
0
 def _load_merge_files(self):
     merge_root = self.resourceLocation('merge_sources')
     tests = []
     source_ids = collections.defaultdict(list)
     expected_files = {}
     for fn in glob.glob(os.path.join(merge_root, SOURCE_PAT)):
         base_fn = os.path.basename(fn)
         file_id = re.match(r"source(\d+)\-(\d+)[.]yaml", base_fn)
         if not file_id:
             raise IOError("File %s does not have a numeric identifier" %
                           (fn))
         file_id = int(file_id.group(1))
         source_ids[file_id].append(fn)
         expected_fn = os.path.join(merge_root, EXPECTED_PAT % (file_id))
         if not os.path.isfile(expected_fn):
             raise IOError("No expected file found at %s" % (expected_fn))
         expected_files[file_id] = expected_fn
     for i in sorted(source_ids.keys()):
         source_file_contents = []
         for fn in sorted(source_ids[i]):
             source_file_contents.append([fn, util.load_file(fn)])
         expected = util.load_yaml(util.load_file(expected_files[i]))
         entry = [source_file_contents, [expected, expected_files[i]]]
         tests.append(entry)
     return tests
예제 #20
0
    def test_cloud_config_archive(self):
        non_decodable = b"\x11\xc9\xb4gTH\xee\x12"
        data = [
            {
                "content": "#cloud-config\npassword: gocubs\n"
            },
            {
                "content": "#cloud-config\nlocale: chicago\n"
            },
            {
                "content": non_decodable
            },
        ]
        message = b"#cloud-config-archive\n" + safeyaml.dumps(data).encode()

        self.reRoot()
        ci = stages.Init()
        ci.datasource = FakeDataSource(message)

        fs = {}

        def fsstore(filename, content, mode=0o0644, omode="wb"):
            fs[filename] = content

        # consuming the user-data provided should write 'cloud_config' file
        # which will have our yaml in it.
        with mock.patch("cloudinit.util.write_file") as mockobj:
            mockobj.side_effect = fsstore
            ci.fetch()
            ci.consume_data()

        cfg = util.load_yaml(fs[ci.paths.get_ipath("cloud_config")])
        self.assertEqual(cfg.get("password"), "gocubs")
        self.assertEqual(cfg.get("locale"), "chicago")
예제 #21
0
    def test_include_bad_url_no_fail(self, mock_sleep):
        """Test #include with a bad URL and failure disabled"""
        bad_url = "http://bad/forbidden"
        bad_data = "#cloud-config\nbad: true\n"
        httpretty.register_uri(httpretty.GET, bad_url, bad_data, status=403)

        included_url = "http://hostname/path"
        included_data = "#cloud-config\nincluded: true\n"
        httpretty.register_uri(httpretty.GET, included_url, included_data)

        blob = "#include\n%s\n%s" % (bad_url, included_url)

        self.reRoot()
        ci = stages.Init()
        ci.datasource = FakeDataSource(blob)
        log_file = self.capture_log(logging.WARNING)
        ci.fetch()
        ci.consume_data()

        self.assertIn(
            "403 Client Error: Forbidden for url: %s" % bad_url,
            log_file.getvalue(),
        )

        cc_contents = util.load_file(ci.paths.get_ipath("cloud_config"))
        cc = util.load_yaml(cc_contents)
        self.assertIsNone(cc.get("bad"))
        self.assertTrue(cc.get("included"))
예제 #22
0
    def test_mime_gzip_compressed(self):
        """Tests that individual message gzip encoding works."""
        def gzip_part(text):
            return MIMEApplication(gzip_text(text), "gzip")

        base_content1 = """
#cloud-config
a: 2
"""

        base_content2 = """
#cloud-config
b: 3
c: 4
"""

        message = MIMEMultipart("test")
        message.attach(gzip_part(base_content1))
        message.attach(gzip_part(base_content2))
        ci = stages.Init()
        ci.datasource = FakeDataSource(str(message))
        self.reRoot()
        ci.fetch()
        ci.consume_data()
        contents = util.load_file(ci.paths.get_ipath("cloud_config"))
        contents = util.load_yaml(contents)
        self.assertTrue(isinstance(contents, dict))
        self.assertEqual(3, len(contents))
        self.assertEqual(2, contents["a"])
        self.assertEqual(3, contents["b"])
        self.assertEqual(4, contents["c"])
예제 #23
0
    def test_merging_cloud_config(self):
        blob = '''
#cloud-config
a: b
e: f
run:
 - b
 - c
'''
        message1 = MIMEBase("text", "cloud-config")
        message1.set_payload(blob)

        blob2 = '''
#cloud-config
a: e
e: g
run:
 - stuff
 - morestuff
'''
        message2 = MIMEBase("text", "cloud-config")
        message2['X-Merge-Type'] = ('dict(recurse_array,'
                                    'recurse_str)+list(append)+str(append)')
        message2.set_payload(blob2)

        blob3 = '''
#cloud-config
e:
 - 1
 - 2
 - 3
p: 1
'''
        message3 = MIMEBase("text", "cloud-config")
        message3.set_payload(blob3)

        messages = [message1, message2, message3]

        paths = c_helpers.Paths({}, ds=FakeDataSource(''))
        cloud_cfg = handlers.cloud_config.CloudConfigPartHandler(paths)

        new_root = self.makeDir()
        self.patchUtils(new_root)
        self.patchOS(new_root)
        cloud_cfg.handle_part(None, handlers.CONTENT_START, None, None, None,
                              None)
        for i, m in enumerate(messages):
            headers = dict(m)
            fn = "part-%s" % (i + 1)
            payload = m.get_payload(decode=True)
            cloud_cfg.handle_part(None, headers['Content-Type'],
                                  fn, payload, None, headers)
        cloud_cfg.handle_part(None, handlers.CONTENT_END, None, None, None,
                              None)
        contents = util.load_file(paths.get_ipath('cloud_config'))
        contents = util.load_yaml(contents)
        self.assertEquals(contents['run'], ['b', 'c', 'stuff', 'morestuff'])
        self.assertEquals(contents['a'], 'be')
        self.assertEquals(contents['e'], [1, 2, 3])
        self.assertEquals(contents['p'], 1)
예제 #24
0
    def test_cloud_config_as_x_shell_script(self):
        blob_cc = """
#cloud-config
a: b
c: d
"""
        message_cc = MIMEBase("text", "x-shellscript")
        message_cc.set_payload(blob_cc)

        blob_jp = """
#cloud-config-jsonp
[
     { "op": "replace", "path": "/a", "value": "c" },
     { "op": "remove", "path": "/c" }
]
"""

        message_jp = MIMEBase("text", "cloud-config-jsonp")
        message_jp.set_payload(blob_jp)

        message = MIMEMultipart()
        message.attach(message_cc)
        message.attach(message_jp)

        self.reRoot()
        ci = stages.Init()
        ci.datasource = FakeDataSource(str(message))
        ci.fetch()
        ci.consume_data()
        cc_contents = util.load_file(ci.paths.get_ipath("cloud_config"))
        cc = util.load_yaml(cc_contents)
        self.assertEqual(1, len(cc))
        self.assertEqual("c", cc["a"])
예제 #25
0
    def test_mixed_cloud_config(self):
        blob_cc = """
#cloud-config
a: b
c: d
"""
        message_cc = MIMEBase("text", "cloud-config")
        message_cc.set_payload(blob_cc)

        blob_jp = """
#cloud-config-jsonp
[
     { "op": "replace", "path": "/a", "value": "c" },
     { "op": "remove", "path": "/c" }
]
"""

        message_jp = MIMEBase("text", "cloud-config-jsonp")
        message_jp.set_payload(blob_jp)

        message = MIMEMultipart()
        message.attach(message_cc)
        message.attach(message_jp)

        ci = stages.Init()
        ci.datasource = FakeDataSource(str(message))
        new_root = self.makeDir()
        self.patchUtils(new_root)
        self.patchOS(new_root)
        ci.fetch()
        ci.consume_userdata()
        cc_contents = util.load_file(ci.paths.get_ipath("cloud_config"))
        cc = util.load_yaml(cc_contents)
        self.assertEquals(1, len(cc))
        self.assertEquals("c", cc["a"])
예제 #26
0
 def _load_merge_files(self):
     merge_root = self.resourceLocation('merge_sources')
     tests = []
     source_ids = collections.defaultdict(list)
     expected_files = {}
     for fn in glob.glob(os.path.join(merge_root, SOURCE_PAT)):
         base_fn = os.path.basename(fn)
         file_id = re.match(r"source(\d+)\-(\d+)[.]yaml", base_fn)
         if not file_id:
             raise IOError("File %s does not have a numeric identifier"
                           % (fn))
         file_id = int(file_id.group(1))
         source_ids[file_id].append(fn)
         expected_fn = os.path.join(merge_root, EXPECTED_PAT % (file_id))
         if not os.path.isfile(expected_fn):
             raise IOError("No expected file found at %s" % (expected_fn))
         expected_files[file_id] = expected_fn
     for i in sorted(source_ids.keys()):
         source_file_contents = []
         for fn in sorted(source_ids[i]):
             source_file_contents.append([fn, util.load_file(fn)])
         expected = util.load_yaml(util.load_file(expected_files[i]))
         entry = [source_file_contents, [expected, expected_files[i]]]
         tests.append(entry)
     return tests
예제 #27
0
    def test_mixed_cloud_config(self):
        blob_cc = '''
#cloud-config
a: b
c: d
'''
        message_cc = MIMEBase("text", "cloud-config")
        message_cc.set_payload(blob_cc)

        blob_jp = '''
#cloud-config-jsonp
[
     { "op": "replace", "path": "/a", "value": "c" },
     { "op": "remove", "path": "/c" }
]
'''

        message_jp = MIMEBase('text', "cloud-config-jsonp")
        message_jp.set_payload(blob_jp)

        message = MIMEMultipart()
        message.attach(message_cc)
        message.attach(message_jp)

        self.reRoot()
        ci = stages.Init()
        ci.datasource = FakeDataSource(str(message))
        ci.fetch()
        ci.consume_data()
        cc_contents = util.load_file(ci.paths.get_ipath("cloud_config"))
        cc = util.load_yaml(cc_contents)
        self.assertEqual(1, len(cc))
        self.assertEqual('c', cc['a'])
예제 #28
0
 def test_nonallowed_returns_default(self):
     # for now, anything not in the allowed list just returns the default.
     myyaml = yaml.dump({'1': "one"})
     self.assertEqual(util.load_yaml(blob=myyaml,
                                     default=self.mydefault,
                                     allowed=(str,)),
                      self.mydefault)
예제 #29
0
 def test_none_returns_default(self):
     """If yaml.load returns None, then default should be returned."""
     blobs = ("", " ", "# foo\n", "#")
     mdef = self.mydefault
     self.assertEqual(
         [(b, self.mydefault) for b in blobs],
         [(b, util.load_yaml(blob=b, default=mdef)) for b in blobs])
예제 #30
0
    def test_mixed_cloud_config(self):
        blob_cc = '''
#cloud-config
a: b
c: d
'''
        message_cc = MIMEBase("text", "cloud-config")
        message_cc.set_payload(blob_cc)

        blob_jp = '''
#cloud-config-jsonp
[
     { "op": "replace", "path": "/a", "value": "c" },
     { "op": "remove", "path": "/c" }
]
'''

        message_jp = MIMEBase('text', "cloud-config-jsonp")
        message_jp.set_payload(blob_jp)

        message = MIMEMultipart()
        message.attach(message_cc)
        message.attach(message_jp)

        self.reRoot()
        ci = stages.Init()
        ci.datasource = FakeDataSource(str(message))
        ci.fetch()
        ci.consume_data()
        cc_contents = util.load_file(ci.paths.get_ipath("cloud_config"))
        cc = util.load_yaml(cc_contents)
        self.assertEqual(1, len(cc))
        self.assertEqual('c', cc['a'])
예제 #31
0
 def test_python_unicode(self):
     # complex type of python/unicde is explicitly allowed
     myobj = {'1': unicode("FOOBAR")}
     safe_yaml = yaml.dump(myobj)
     self.assertEqual(util.load_yaml(blob=safe_yaml,
                                     default=self.mydefault),
                      myobj)
예제 #32
0
파일: util.py 프로젝트: vultr/cloud-init
def merge_results(data, path):
    """Handle merging results from collect phase and verify phase."""
    current = {}
    if os.path.exists(path):
        with open(path, 'r') as fp:
            current = c_util.load_yaml(fp.read())
    current.update(data)
    yaml_dump(current, path)
예제 #33
0
    def handle_part(self, _data, ctype, filename, payload, frequency):
        if ctype == TAUPAGE_AMI_CONFIG_MIME_TYPE:
            LOG.info("Got Taupage AMI configuration; merging with {config}".format(config=TAUPAGE_CONFIG))

            LOG.debug("Parsing given input...")
            config_new = util.load_yaml(payload)

            LOG.debug("Loading existing configuration...")
            config_yaml = util.read_file_or_url(TAUPAGE_CONFIG)
            config_old = util.load_yaml(config_yaml)

            LOG.debug("Merging configurations...")
            config_merged = dict(config_old.items() + config_new.items())

            LOG.debug("Storing merged configuration...")
            config_yaml = util.yaml_dumps(config_merged)
            util.write_file(TAUPAGE_CONFIG, config_yaml, 0o444)
def _merge_new_seed(cur, seeded):
    ret = cur.copy()
    ret['meta-data'] = util.mergemanydict([cur['meta-data'],
                                          util.load_yaml(seeded['meta-data'])])
    ret['user-data'] = seeded['user-data']
    if 'vendor-data' in seeded:
        ret['vendor-data'] = seeded['vendor-data']
    return ret
 def test_variant_sets_distro_in_cloud_cfg(self, variant, tmpdir):
     outfile = tmpdir.join('outcfg').strpath
     subp.subp(self.cmd + ['--variant', variant, self.tmpl_path, outfile])
     with open(outfile) as stream:
         system_cfg = util.load_yaml(stream.read())
     if variant == 'unknown':
         variant = 'ubuntu'  # Unknown is defaulted to ubuntu
     assert system_cfg['system_info']['distro'] == variant
예제 #36
0
def _merge_new_seed(cur, seeded):
    ret = cur.copy()

    newmd = seeded.get("meta-data", {})
    if not isinstance(seeded["meta-data"], dict):
        newmd = util.load_yaml(seeded["meta-data"])
    ret["meta-data"] = util.mergemanydict([cur["meta-data"], newmd])

    if seeded.get("network-config"):
        ret["network-config"] = _maybe_remove_top_network(
            util.load_yaml(seeded.get("network-config")))

    if "user-data" in seeded:
        ret["user-data"] = seeded["user-data"]
    if "vendor-data" in seeded:
        ret["vendor-data"] = seeded["vendor-data"]
    return ret
예제 #37
0
 def test_variant_sets_distro_in_cloud_cfg(self, variant, tmpdir):
     outfile = tmpdir.join("outcfg").strpath
     subp.subp(self.cmd + ["--variant", variant, self.tmpl_path, outfile])
     with open(outfile) as stream:
         system_cfg = util.load_yaml(stream.read())
     if variant == "unknown":
         variant = "ubuntu"  # Unknown is defaulted to ubuntu
     assert system_cfg["system_info"]["distro"] == variant
예제 #38
0
def _merge_new_seed(cur, seeded):
    ret = cur.copy()
    ret['meta-data'] = util.mergemanydict([cur['meta-data'],
                                          util.load_yaml(seeded['meta-data'])])
    ret['user-data'] = seeded['user-data']
    if 'vendor-data' in seeded:
        ret['vendor-data'] = seeded['vendor-data']
    return ret
예제 #39
0
 def test_bogus_scan_error_returns_default(self):
     '''On Yaml scan error, load_yaml returns the default and logs issue.'''
     badyaml = "1\n 2:"
     self.assertEqual(util.load_yaml(blob=badyaml, default=self.mydefault),
                      self.mydefault)
     self.assertIn(
         'Failed loading yaml blob. Invalid format at line 2 column 3:'
         ' "mapping values are not allowed here', self.logs.getvalue())
    def test_variant_sets_network_renderer_priority_in_cloud_cfg(
            self, variant, renderers, tmpdir):
        outfile = tmpdir.join('outcfg').strpath
        subp.subp(self.cmd + ['--variant', variant, self.tmpl_path, outfile])
        with open(outfile) as stream:
            system_cfg = util.load_yaml(stream.read())

        assert renderers == system_cfg['system_info']['network']['renderers']
예제 #41
0
def read_metadata(url, timeout=2, sec_between=2, retries=30):
    response = url_helper.readurl(url,
                                  timeout=timeout,
                                  sec_between=sec_between,
                                  retries=retries)
    if not response.ok():
        raise RuntimeError("unable to read metadata at %s" % url)
    return util.load_yaml(response.contents.decode())
예제 #42
0
    def test_merging_cloud_config(self):
        blob = """
#cloud-config
a: b
e: f
run:
 - b
 - c
"""
        message1 = MIMEBase("text", "cloud-config")
        message1.set_payload(blob)

        blob2 = """
#cloud-config
a: e
e: g
run:
 - stuff
 - morestuff
"""
        message2 = MIMEBase("text", "cloud-config")
        message2[
            "X-Merge-Type"] = "dict(recurse_array,recurse_str)+list(append)+str(append)"
        message2.set_payload(blob2)

        blob3 = """
#cloud-config
e:
 - 1
 - 2
 - 3
p: 1
"""
        message3 = MIMEBase("text", "cloud-config")
        message3.set_payload(blob3)

        messages = [message1, message2, message3]

        paths = c_helpers.Paths({}, ds=FakeDataSource(""))
        cloud_cfg = handlers.cloud_config.CloudConfigPartHandler(paths)

        self.reRoot()
        cloud_cfg.handle_part(None, handlers.CONTENT_START, None, None, None,
                              None)
        for i, m in enumerate(messages):
            headers = dict(m)
            fn = "part-%s" % (i + 1)
            payload = m.get_payload(decode=True)
            cloud_cfg.handle_part(None, headers["Content-Type"], fn, payload,
                                  None, headers)
        cloud_cfg.handle_part(None, handlers.CONTENT_END, None, None, None,
                              None)
        contents = util.load_file(paths.get_ipath("cloud_config"))
        contents = util.load_yaml(contents)
        self.assertEqual(contents["run"], ["b", "c", "stuff", "morestuff"])
        self.assertEqual(contents["a"], "be")
        self.assertEqual(contents["e"], [1, 2, 3])
        self.assertEqual(contents["p"], 1)
예제 #43
0
 def test_bogus_parse_error_returns_default(self):
     '''On Yaml parse error, load_yaml returns default and logs issue.'''
     badyaml = "{}}"
     self.assertEqual(util.load_yaml(blob=badyaml, default=self.mydefault),
                      self.mydefault)
     self.assertIn(
         'Failed loading yaml blob. Invalid format at line 1 column 3:'
         " \"expected \'<document start>\', but found \'}\'",
         self.logs.getvalue())
예제 #44
0
    def test_merging_cloud_config(self):
        blob = """
#cloud-config
a: b
e: f
run:
 - b
 - c
"""
        message1 = MIMEBase("text", "cloud-config")
        message1.set_payload(blob)

        blob2 = """
#cloud-config
a: e
e: g
run:
 - stuff
 - morestuff
"""
        message2 = MIMEBase("text", "cloud-config")
        message2["X-Merge-Type"] = "dict(recurse_array," "recurse_str)+list(append)+str(append)"
        message2.set_payload(blob2)

        blob3 = """
#cloud-config
e:
 - 1
 - 2
 - 3
p: 1
"""
        message3 = MIMEBase("text", "cloud-config")
        message3.set_payload(blob3)

        messages = [message1, message2, message3]

        paths = c_helpers.Paths({}, ds=FakeDataSource(""))
        cloud_cfg = handlers.cloud_config.CloudConfigPartHandler(paths)

        new_root = self.makeDir()
        self.patchUtils(new_root)
        self.patchOS(new_root)
        cloud_cfg.handle_part(None, handlers.CONTENT_START, None, None, None, None)
        for i, m in enumerate(messages):
            headers = dict(m)
            fn = "part-%s" % (i + 1)
            payload = m.get_payload(decode=True)
            cloud_cfg.handle_part(None, headers["Content-Type"], fn, payload, None, headers)
        cloud_cfg.handle_part(None, handlers.CONTENT_END, None, None, None, None)
        contents = util.load_file(paths.get_ipath("cloud_config"))
        contents = util.load_yaml(contents)
        self.assertEquals(contents["run"], ["b", "c", "stuff", "morestuff"])
        self.assertEquals(contents["a"], "be")
        self.assertEquals(contents["e"], [1, 2, 3])
        self.assertEquals(contents["p"], 1)
예제 #45
0
 def test_bogus_parse_error_returns_default(self):
     '''On Yaml parse error, load_yaml returns default and logs issue.'''
     badyaml = "{}}"
     self.assertEqual(util.load_yaml(blob=badyaml,
                                     default=self.mydefault),
                      self.mydefault)
     self.assertIn(
         'Failed loading yaml blob. Invalid format at line 1 column 3:'
         " \"expected \'<document start>\', but found \'}\'",
         self.logs.getvalue())
예제 #46
0
 def test_bogus_scan_error_returns_default(self):
     '''On Yaml scan error, load_yaml returns the default and logs issue.'''
     badyaml = "1\n 2:"
     self.assertEqual(util.load_yaml(blob=badyaml,
                                     default=self.mydefault),
                      self.mydefault)
     self.assertIn(
         'Failed loading yaml blob. Invalid format at line 2 column 3:'
         ' "mapping values are not allowed here',
         self.logs.getvalue())
예제 #47
0
def handle_part(data,ctype,filename,payload):

  # data: the cloudinit object
  # ctype: '__begin__', '__end__', or the specific mime-type of the part
  # filename: the filename for the part, or dynamically generated part if
  #           no filename is given attribute is present
  # payload: the content of the part (empty for begin or end)
  
  if ctype == "__begin__":
     #print "my handler is beginning"
     return
  if ctype == "__end__":
     #print "my handler is ending"
     return

  logger.info('==== received ctype=%s filename=%s ====' % (ctype,filename))

  # Payload should be interpreted as yaml since configuration is given in cloud-config format
  cfg = util.load_yaml(payload)
  
  # If there isn't a se_config reference in the configuration don't do anything
  if 'se_config' not in cfg:
     logger.error('se_config configuration was not found!')
     return 
 
  logger.info('ready to configure SE')
  global se_config_cfg 
  se_config_cfg = cfg['se_config'] 
  
  if 'name' in se_config_cfg:
     name = se_config_cfg['name']
     logger.info('configuring SE: '+name+'')
  else:
     logger.error('SE name not specified!')
     return 

  logger.info('linking to proper mysql-connector-java.jar')
  if 'javamc' in se_config_cfg:
     javamc = se_config_cfg['javamc']
  else:
     javamc = 'mysql-connector-java-5.1.17.jar'
  try:
    cmd = ('rm -f /usr/share/java/storm-backend-server/mysql-connector-java-*.jar')  
    DPopen(cmd, 'True')
  except:
    logger.error('could not remove /usr/share/java/storm-backend-server/mysql-connector-java-*.jar')
    return
  try:
    cmd = ('ln -s /usr/share/java/'+javamc+' /usr/share/java/storm-backend-server/'+javamc+'') 
    DPopen(cmd, 'True')
  except:
    logger.error('could not link /usr/share/java/'+javamc+' to /usr/share/java/storm-backend-server/!')
    return

  logger.info('==== end ctype=%s filename=%s' % (ctype, filename))	       
예제 #48
0
 def test_nonallowed_returns_default(self):
     '''Any unallowed types result in returning default; log the issue.'''
     # for now, anything not in the allowed list just returns the default.
     myyaml = yaml.dump({'1': "one"})
     self.assertEqual(util.load_yaml(blob=myyaml,
                                     default=self.mydefault,
                                     allowed=(str,)),
                      self.mydefault)
     regex = re.compile(
         r'Yaml load allows \(<(class|type) \'str\'>,\) root types, but'
         r' got dict')
     self.assertTrue(regex.search(self.logs.getvalue()),
                     msg='Missing expected yaml load error')
예제 #49
0
    def _explode_archive(self, archive, append_msg):
        entries = util.load_yaml(archive, default=[], allowed=(list, set))
        for ent in entries:
            # ent can be one of:
            #  dict { 'filename' : 'value', 'content' :
            #       'value', 'type' : 'value' }
            #    filename and type not be present
            # or
            #  scalar(payload)
            if isinstance(ent, six.string_types):
                ent = {"content": ent}
            if not isinstance(ent, (dict)):
                # TODO(harlowja) raise?
                continue

            content = ent.get("content", "")
            mtype = ent.get("type")
            if not mtype:
                default = ARCHIVE_UNDEF_TYPE
                if isinstance(content, six.binary_type):
                    default = ARCHIVE_UNDEF_BINARY_TYPE
                mtype = handlers.type_from_starts_with(content, default)

            maintype, subtype = mtype.split("/", 1)
            if maintype == "text":
                if isinstance(content, six.binary_type):
                    content = content.decode()
                msg = MIMEText(content, _subtype=subtype)
            else:
                msg = MIMEBase(maintype, subtype)
                msg.set_payload(content)

            if "filename" in ent:
                _set_filename(msg, ent["filename"])
            if "launch-index" in ent:
                msg.add_header("Launch-Index", str(ent["launch-index"]))

            for header in list(ent.keys()):
                if header.lower() in (
                    "content",
                    "filename",
                    "type",
                    "launch-index",
                    "content-disposition",
                    ATTACHMENT_FIELD.lower(),
                    CONTENT_TYPE.lower(),
                ):
                    continue
                msg.add_header(header, ent[header])

            self._attach_part(append_msg, msg)
예제 #50
0
def handle_part(data,ctype,filename,payload):

  # data: the cloudinit object
  # ctype: '__begin__', '__end__', or the specific mime-type of the part
  # filename: the filename for the part, or dynamically generated part if
  #           no filename is given attribute is present
  # payload: the content of the part (empty for begin or end)
  
  if ctype == "__begin__":
     #print "my handler is beginning"
     return
  if ctype == "__end__":
     #print "my handler is ending"
     return

  logger.info('==== received ctype=%s filename=%s ====' % (ctype,filename))

  # Payload should be interpreted as yaml since configuration is given in cloud-config format
  cfg = util.load_yaml(payload)

  # If there isn't a zabbix reference in the configuration don't do anything
  if 'zabbix' not in cfg:
    logger.error('zabbix configuration was not found!')
    return
  else:
    install = cfg['zabbix']
  
  if install == False:
    logger.info('Im NOT going to install zabbix!')
    return
  
  logger.info('Installing Zabbix...')
  try:
    cmd = ('yum -y --enablerepo=epel install zabbix zabbix-agent')
    DPopen(shlex.split(cmd),'False')  
  except:
    logger.error('could not install Zabbix!')

  logger.info('starting Zabbix agent...')
  try:
    cmd = ('chkconfig zabbix-agent on')
    DPopen(cmd, 'True')
    cmd = ('service zabbix-agent restart')  
    DPopen(cmd, 'True')
  except:
    logger.error('could not start Zabbix agent!')
    return
  
  
  logger.info('==== end ctype=%s filename=%s' % (ctype, filename))	       
def handle_part(data,ctype,filename,payload):

  # data: the cloudinit object
  # ctype: '__begin__', '__end__', or the specific mime-type of the part
  # filename: the filename for the part, or dynamically generated part if
  #           no filename is given attribute is present
  # payload: the content of the part (empty for begin or end)
  
  if ctype == "__begin__":
     return
  if ctype == "__end__":
     return

  logger.info('==== received ctype=%s filename=%s ====' % (ctype,filename))

  # Payload should be interpreted as yaml since configuration is given in cloud-config format
  cfg = util.load_yaml(payload)
  
  # If there isn't a myproxy_config reference in the configuration don't do anything
  if 'myproxy_config' not in cfg:
     logger.error('myproxy_config configuration was not found!')
     return 
 
  logger.info('ready to configure PX')
  global myproxy_config_cfg 
  myproxy_config_cfg = cfg['myproxy_config'] 
  
  logger.info('Copying certificate to /etc/grid-security/myproxy...')
  try:
     md = ('mkdir /etc/grid-security/myproxy/')
     DPopen(cmd, 'True')
  except:
     logger.error('could not create /etc/grid-security/myproxy!')
     return
  for item in ['hostcert.pem','hostkey.pem']:
     try:
        cmd = ('cp /etc/grid-security/'+item+' /etc/grid-security/myproxy/')
        DPopen(cmd, 'True')
     except:
        logger.error('could not copy '+item+'!')
        return
     try:
        cmd = ('chown myproxy:myproxy /etc/grid-security/myproxy/'+item+'')
        DPopen(cmd, 'True')
     except:
        logger.error('could change ownership of '+item+'!')
        return


  logger.info('==== end ctype=%s filename=%s' % (ctype, filename))	       
예제 #52
0
    def handle_part(self, _data, ctype, filename, payload, frequency):
        if ctype == TAUPAGE_AMI_CONFIG_MIME_TYPE:
            LOG.info("Got Taupage AMI configuration; merging with {config}".format(config=TAUPAGE_CONFIG))

            LOG.debug("Parsing given input...")
            config_new = util.load_yaml(payload)

            LOG.debug("Loading existing configuration...")
            config_yaml = util.read_file_or_url(TAUPAGE_CONFIG)
            config_old = util.load_yaml(config_yaml)

            LOG.debug("Merging configurations...")
            config_merged = dict(config_old.items() + config_new.items())

            LOG.debug("Storing merged configuration...")
            config_yaml = util.yaml_dumps(config_merged)
            util.write_file(TMP_TAUPAGE_CONFIG, config_yaml, 0o444)

            LOG.debug("Comparing current configuration with the old one...")
            subprocess.call(['diff', '-u0', TAUPAGE_CONFIG, TMP_TAUPAGE_CONFIG])

            LOG.debug("Moving the new configuration into place...")
            shutil.move(TMP_TAUPAGE_CONFIG, TAUPAGE_CONFIG)
예제 #53
0
    def test_include(self, mock_sleep):
        """Test #include."""
        included_url = 'http://hostname/path'
        included_data = '#cloud-config\nincluded: true\n'
        httpretty.register_uri(httpretty.GET, included_url, included_data)

        blob = '#include\n%s\n' % included_url

        self.reRoot()
        ci = stages.Init()
        ci.datasource = FakeDataSource(blob)
        ci.fetch()
        ci.consume_data()
        cc_contents = util.load_file(ci.paths.get_ipath("cloud_config"))
        cc = util.load_yaml(cc_contents)
        self.assertTrue(cc.get('included'))
    def test_apt_v1_srcl_custom(self):
        """Test rendering from a custom source.list template"""
        cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL)
        mycloud = self._get_cloud('ubuntu')

        # the second mock restores the original subp
        with mock.patch.object(util, 'write_file') as mockwrite:
            with mock.patch.object(util, 'subp', self.subp):
                with mock.patch.object(Distro, 'get_primary_arch',
                                       return_value='amd64'):
                    cc_apt_configure.handle("notimportant", cfg, mycloud,
                                            LOG, None)

        mockwrite.assert_called_once_with(
            '/etc/apt/sources.list',
            EXPECTED_CONVERTED_CONTENT,
            mode=420)
예제 #55
0
def handle_part(data,ctype,filename,payload):

  # data: the cloudinit object
  # ctype: '__begin__', '__end__', or the specific mime-type of the part
  # filename: the filename for the part, or dynamically generated part if
  #           no filename is given attribute is present
  # payload: the content of the part (empty for begin or end)
  
  if ctype == "__begin__":
     #print "my handler is beginning"
     return
  if ctype == "__end__":
     #print "my handler is ending"
     return

  logger.info('==== received ctype=%s filename=%s ====' % (ctype,filename))

  # Payload should be interpreted as yaml since configuration is given in cloud-config format
  cfg = util.load_yaml(payload)
  
  # If there isn't an apel reference in the configuration don't do anything
  if 'apel' not in cfg:
     logger.error('apel configuration was not found!')
     return 
 
  logger.info('ready to configure APEL')
  global apel_cfg 
  apel_cfg = cfg['apel'] 
  
  if 'parser.cfg' in apel_cfg:
     os.mkdir('/etc/apel') 
     try:
         val = apel_cfg['parser.cfg']
         get_embedded('parser.cfg', val, '/etc/apel/')
     except:
        logger.error('could not write configuration file!')
        return
  try:
     cmd=('echo "00 19 * * * root /usr/bin/apelparser" > /etc/cron.d/apel')
     DPopen(cmd, 'True')
  except:
     logger.error('could not add apelparser to crontab!')
     return
 
  
  logger.info('==== end ctype=%s filename=%s' % (ctype, filename))	       
예제 #56
0
def fetch_base_config():
    base_cfgs = []
    default_cfg = util.get_builtin_cfg()
    kern_contents = util.read_cc_from_cmdline()

    # Kernel/cmdline parameters override system config
    if kern_contents:
        base_cfgs.append(util.load_yaml(kern_contents, default={}))

    # Anything in your conf.d location??
    # or the 'default' cloud.cfg location???
    base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG))

    # And finally the default gets to play
    if default_cfg:
        base_cfgs.append(default_cfg)

    return util.mergemanydict(base_cfgs)
예제 #57
0
def handle_part(data,ctype,filename,payload):

  # data: the cloudinit object
  # ctype: '__begin__', '__end__', or the specific mime-type of the part
  # filename: the filename for the part, or dynamically generated part if
  #           no filename is given attribute is present
  # payload: the content of the part (empty for begin or end)
  
  if ctype == "__begin__":
     #print "my handler is beginning"
     return
  if ctype == "__end__":
     #print "my handler is ending"
     return

  logger.info('==== received ctype=%s filename=%s ====' % (ctype,filename))

  # Payload should be interpreted as yaml since configuration is given in cloud-config format
  cfg = util.load_yaml(payload)

  # If there isn't a addons reference in the configuration don't do anything
  if 'addons' not in cfg:
    logger.error('addons configuration was not found!')
    return
  else:
    repos = cfg['addons']
    for repo in repos:
      logger.info('installing packages from repo '+repo+':')
      string = ''
      pkgs = repos[repo]
      for pkg in pkgs:
        string += (''+pkg+'  ')
      logger.info(string)
      try:
        if repo != 'none':
          cmd = ('yum -y install '+string+'')
        else:
          cmd = ('yum -y --enablerepo='+repo+' install '+string+'') 
        DPopen(cmd, 'True')
      except:
        logger.error('error: could not install addons!')

  
  logger.info('==== end ctype=%s filename=%s' % (ctype, filename))	       
예제 #58
0
 def _extract_mergers(self, payload, headers):
     merge_header_headers = ''
     for h in [MERGE_HEADER, 'X-%s' % (MERGE_HEADER)]:
         tmp_h = headers.get(h, '')
         if tmp_h:
             merge_header_headers = tmp_h
             break
     # Select either the merge-type from the content
     # or the merge type from the headers or default to our own set
     # if neither exists (or is empty) from the later.
     payload_yaml = util.load_yaml(payload)
     mergers_yaml = mergers.dict_extract_mergers(payload_yaml)
     mergers_header = mergers.string_extract_mergers(merge_header_headers)
     all_mergers = []
     all_mergers.extend(mergers_yaml)
     all_mergers.extend(mergers_header)
     if not all_mergers:
         all_mergers = DEF_MERGERS
     return (payload_yaml, all_mergers)
예제 #59
0
    def test_simple_jsonp(self):
        blob = '''
#cloud-config-jsonp
[
     { "op": "add", "path": "/baz", "value": "qux" },
     { "op": "add", "path": "/bar", "value": "qux2" }
]
'''

        ci = stages.Init()
        ci.datasource = FakeDataSource(blob)
        self.reRoot()
        ci.fetch()
        ci.consume_data()
        cc_contents = util.load_file(ci.paths.get_ipath("cloud_config"))
        cc = util.load_yaml(cc_contents)
        self.assertEqual(2, len(cc))
        self.assertEqual('qux', cc['baz'])
        self.assertEqual('qux2', cc['bar'])
예제 #60
-1
    def test_mime_gzip_compressed(self):
        """Tests that individual message gzip encoding works."""

        def gzip_part(text):
            return MIMEApplication(gzip_text(text), 'gzip')

        base_content1 = '''
#cloud-config
a: 2
'''

        base_content2 = '''
#cloud-config
b: 3
c: 4
'''

        message = MIMEMultipart('test')
        message.attach(gzip_part(base_content1))
        message.attach(gzip_part(base_content2))
        ci = stages.Init()
        ci.datasource = FakeDataSource(str(message))
        new_root = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, new_root)
        self.patchUtils(new_root)
        self.patchOS(new_root)
        ci.fetch()
        ci.consume_data()
        contents = util.load_file(ci.paths.get_ipath("cloud_config"))
        contents = util.load_yaml(contents)
        self.assertTrue(isinstance(contents, dict))
        self.assertEquals(3, len(contents))
        self.assertEquals(2, contents['a'])
        self.assertEquals(3, contents['b'])
        self.assertEquals(4, contents['c'])