def test_replace_namenode_name(self):
        conf_dir = environ.get("CONF_DIR")
        environ["HDFS_NAMENODE_NAME"] = "hdfs://namenode.magi.io:9000"

        xml = read_xml(conf_dir, self.core_site_file)
        output = process_core_site(xml)

        assert (verify_prop_values(output, "fs.default.name",
                                   environ["HDFS_NAMENODE_NAME"]))
Example #2
0
    def test_overwrite_core_site(self):
        environ["HDFS_NAMENODE_ADDRESS"] = "hdfs://namenode.magi.io:9000"
        environ["HDFS_NAMENODE_NAME"] = "hdfs://namenode.magi.io:9000"
        conf_dir = environ.get("CONF_DIR")
        original = read_xml(conf_dir, self.filename)
        processed = process_core_site(original)
        overwrite_file(conf_dir, self.filename, processed)

        updated = read_xml(conf_dir, self.filename)
        assert (verify_prop_values(updated, "fs.defaultFS",
                                   environ["HDFS_NAMENODE_ADDRESS"]))
        assert (verify_prop_values(updated, "fs.default.name",
                                   environ["HDFS_NAMENODE_NAME"]))
Example #3
0
#!/usr/bin/env python3

from os import environ

from common.helpers import read_xml, overwrite_file
from hdfs.helpers import process_core_site, process_hdfs_site

if __name__ == '__main__':
    conf_dir = environ.get("CONF_DIR") if environ.get(
        "CONF_DIR") else "/opt/hadoop/etc/hadoop"
    core_site_file = "core-site.xml"
    hdfs_site_file = "hdfs-site.xml"
    print(f"using configuration: {conf_dir}/{core_site_file}")
    print(f"using configuration: {conf_dir}/{hdfs_site_file}")
    core_site = read_xml(conf_dir, core_site_file)
    hdfs_site = read_xml(conf_dir, hdfs_site_file)

    processed_core_site = process_core_site(core_site)
    processed_hdfs_site = process_hdfs_site(hdfs_site)

    if processed_core_site is not None:
        overwrite_file(conf_dir, core_site_file, processed_core_site)
    if processed_hdfs_site is not None:
        overwrite_file(conf_dir, hdfs_site_file, processed_hdfs_site)
    else:
        print(f"using default {core_site_file} from docker images")
        print(
            "to learn more about the HDFS configs please visit the GitHub repo: https://github.com/magi-platform/magi"
        )