def test_overwrite_core_site(self):
environ["HDFS_NAMENODE_ADDRESS"] = "hdfs://namenode.magi.io:9000"
environ["HDFS_NAMENODE_NAME"] = "hdfs://namenode.magi.io:9000"
conf_dir = environ.get("CONF_DIR")
original = read_xml(conf_dir, self.filename)
processed = process_core_site(original)
overwrite_file(conf_dir, self.filename, processed)
updated = read_xml(conf_dir, self.filename)
assert (verify_prop_values(updated, "fs.defaultFS",
environ["HDFS_NAMENODE_ADDRESS"]))
assert (verify_prop_values(updated, "fs.default.name",
environ["HDFS_NAMENODE_NAME"]))
def test_replace_all_supported_properties(self):
conf_dir = environ.get("CONF_DIR")
environ["HDFS_NAMENODE_HTTP_PORT"] = "1337"
environ["HDFS_NAMENODE_HTTPS_PORT"] = "1338"
environ["HDFS_DATANODE_HTTP_PORT"] = "1339"
environ["HDFS_DATANODE_HTTPS_PORT"] = "1340"
environ["HDFS_DATANODE_ADDRESS"] = "1341"
environ["HDFS_DATANODE_IPC_PORT"] = "1342"
xml = read_xml(conf_dir, self.hdfs_site_file)
output = process_hdfs_site(xml)
assert (verify_prop_values(
output, "dfs.http.address",
f"0.0.0.0:{environ[ 'HDFS_NAMENODE_HTTP_PORT' ]}"))
assert (verify_prop_values(
output, "dfs.https.address",
f"0.0.0.0:{environ[ 'HDFS_NAMENODE_HTTPS_PORT' ]}"))
assert (verify_prop_values(
output, "dfs.datanode.http.address",
f"0.0.0.0:{environ[ 'HDFS_DATANODE_HTTP_PORT' ]}"))
assert (verify_prop_values(
output, "dfs.datanode.https.address",
f"0.0.0.0:{environ[ 'HDFS_DATANODE_HTTPS_PORT' ]}"))
assert (verify_prop_values(
output, "dfs.datanode.address",
f"0.0.0.0:{environ[ 'HDFS_DATANODE_ADDRESS' ]}"))
assert (verify_prop_values(
output, "dfs.datanode.ipc.address",
f"0.0.0.0:{environ[ 'HDFS_DATANODE_IPC_PORT' ]}"))
def test_overwrite_hbase_site(self):
conf_dir = environ.get("CONF_DIR")
environ["HBASE_ROOT_DIR"] = "hdfs://namenode.magi.io:9000/hbase"
environ["HBASE_ZOOKEEPER_HOST"] = "zk-test"
environ["HBASE_ZOOKEEPER_PORT"] = "6308"
original = read_xml(conf_dir, self.filename)
processed = process(original)
overwrite_file(conf_dir, self.filename, processed)
updated = read_xml(conf_dir, self.filename)
assert (verify_prop_values(updated, "hbase.rootdir",
environ["HBASE_ROOT_DIR"]))
assert (verify_prop_values(updated, "hbase.zookeeper.quorum",
environ["HBASE_ZOOKEEPER_HOST"]))
assert (verify_prop_values(updated,
"hbase.zookeeper.property.clientPort",
environ["HBASE_ZOOKEEPER_PORT"]))
def test_replace_zookeeper_host(self):
conf_dir = environ.get("CONF_DIR")
environ["HBASE_ZOOKEEPER_HOST"] = "zk-test"
xml = read_xml(conf_dir, self.hbase_site_file)
output = process(xml)
assert (verify_prop_values(output, "hbase.zookeeper.quorum",
environ["HBASE_ZOOKEEPER_HOST"]))
def test_replace_hbase_root_dir(self):
conf_dir = environ.get("CONF_DIR")
environ["HBASE_ROOT_DIR"] = "hdfs://namenode.magi.io:9000/hbase"
xml = read_xml(conf_dir, self.hbase_site_file)
output = process(xml)
assert (verify_prop_values(output, "hbase.rootdir",
environ["HBASE_ROOT_DIR"]))
def test_replace_namenode_name(self):
conf_dir = environ.get("CONF_DIR")
environ["HDFS_NAMENODE_NAME"] = "hdfs://namenode.magi.io:9000"
xml = read_xml(conf_dir, self.core_site_file)
output = process_core_site(xml)
assert (verify_prop_values(output, "fs.default.name",
environ["HDFS_NAMENODE_NAME"]))
def test_replace_zookeeper_port(self):
conf_dir = environ.get("CONF_DIR")
environ["HBASE_ZOOKEEPER_PORT"] = "6308"
xml = read_xml(conf_dir, self.hbase_site_file)
output = process(xml)
assert (verify_prop_values(output,
"hbase.zookeeper.property.clientPort",
environ["HBASE_ZOOKEEPER_PORT"]))
def test_overwrite_core_site(self):
environ["HDFS_NAMENODE_HTTP_PORT"] = "1337"
environ["HDFS_NAMENODE_HTTPS_PORT"] = "1338"
environ["HDFS_DATANODE_HTTP_PORT"] = "1339"
environ["HDFS_DATANODE_HTTPS_PORT"] = "1340"
environ["HDFS_DATANODE_ADDRESS"] = "1341"
environ["HDFS_DATANODE_IPC_PORT"] = "1342"
conf_dir = environ.get("CONF_DIR")
original = read_xml(conf_dir, self.filename)
processed = process_hdfs_site(original)
overwrite_file(conf_dir, self.filename, processed)
updated = read_xml(conf_dir, self.filename)
assert (verify_prop_values(
updated, "dfs.http.address",
f"0.0.0.0:{environ[ 'HDFS_NAMENODE_HTTP_PORT' ]}"))
assert (verify_prop_values(
updated, "dfs.https.address",
f"0.0.0.0:{environ[ 'HDFS_NAMENODE_HTTPS_PORT' ]}"))
assert (verify_prop_values(
updated, "dfs.datanode.http.address",
f"0.0.0.0:{environ[ 'HDFS_DATANODE_HTTP_PORT' ]}"))
assert (verify_prop_values(
updated, "dfs.datanode.https.address",
f"0.0.0.0:{environ[ 'HDFS_DATANODE_HTTPS_PORT' ]}"))
assert (verify_prop_values(
updated, "dfs.datanode.address",
f"0.0.0.0:{environ[ 'HDFS_DATANODE_ADDRESS' ]}"))
assert (verify_prop_values(
updated, "dfs.datanode.ipc.address",
f"0.0.0.0:{environ[ 'HDFS_DATANODE_IPC_PORT' ]}"))
if __name__ == '__main__':
unittest.main()
def test_replace_all_supported_properties(self):
conf_dir = environ.get("CONF_DIR")
environ["HBASE_ROOT_DIR"] = "hdfs://namenode.magi.io:9000/hbase"
environ["HBASE_ZOOKEEPER_HOST"] = "zk-test"
environ["HBASE_ZOOKEEPER_PORT"] = "6308"
xml = read_xml(conf_dir, self.hbase_site_file)
output = process(xml)
assert (verify_prop_values(output, "hbase.rootdir",
environ["HBASE_ROOT_DIR"]))
assert (verify_prop_values(output, "hbase.zookeeper.quorum",
environ["HBASE_ZOOKEEPER_HOST"]))
assert (verify_prop_values(output,
"hbase.zookeeper.property.clientPort",
environ["HBASE_ZOOKEEPER_PORT"]))
#!/usr/bin/env python3
from os import environ
from common.helpers import read_xml, overwrite_file
from hbase.helpers import process
if __name__ == '__main__':
conf_dir = environ.get("CONF_DIR") if environ.get(
"CONF_DIR") else "/opt/hbase/conf"
filename = "hbase-site.xml"
print(f"using configuration: {conf_dir}/{filename}")
xml = read_xml(conf_dir, filename)
processed_core_site = process(xml)
if processed_core_site is not None:
overwrite_file(conf_dir, filename, processed_core_site)
else:
print(f"using default {filename} from docker images")
print(
"to learn more about the HDFS configs please visit the GitHub repo: https://github.com/magi-platform/magi"
)
#!/usr/bin/env python3
from os import environ
from common.helpers import read_xml, overwrite_file
from hdfs.helpers import process_core_site, process_hdfs_site
if __name__ == '__main__':
conf_dir = environ.get("CONF_DIR") if environ.get(
"CONF_DIR") else "/opt/hadoop/etc/hadoop"
core_site_file = "core-site.xml"
hdfs_site_file = "hdfs-site.xml"
print(f"using configuration: {conf_dir}/{core_site_file}")
print(f"using configuration: {conf_dir}/{hdfs_site_file}")
core_site = read_xml(conf_dir, core_site_file)
hdfs_site = read_xml(conf_dir, hdfs_site_file)
processed_core_site = process_core_site(core_site)
processed_hdfs_site = process_hdfs_site(hdfs_site)
if processed_core_site is not None:
overwrite_file(conf_dir, core_site_file, processed_core_site)
if processed_hdfs_site is not None:
overwrite_file(conf_dir, hdfs_site_file, processed_hdfs_site)
else:
print(f"using default {core_site_file} from docker images")
print(
"to learn more about the HDFS configs please visit the GitHub repo: https://github.com/magi-platform/magi"
)