def test_tricky_confparse():
"""
We found (experimentally) that dealing with a file
sometimes triggered the wrong results here.
"""
cp_data = confparse.ConfParse(open(os.path.join(os.path.dirname(__file__),
"test_data",
"sample_conf.xml"), 'rb'))
assert_equal("org.apache.hadoop.examples.SleepJob", cp_data["mapred.mapper.class"])
def get_jobconf(jt, jobid):
"""
Returns a dict representation of the jobconf for the job corresponding
to jobid. filter_keys is an optional list of configuration keys to filter on.
"""
jid = jt.thriftjobid_from_string(jobid)
# This will throw if the the jobconf can't be found
xml_data = jt.get_job_xml(jid)
return confparse.ConfParse(xml_data)
def test_confparse():
data = """
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://localhost:8020</value>
</property>
<property>
<name>with_description</name>
<value>bar</value>
<description>A base for other temporary directories.</description>
</property>
<property>
<name>boolean_true</name>
<value>true</value>
</property>
<property>
<name>boolean_false</name>
<value>false</value>
</property>
</configuration>
"""
cp_data = confparse.ConfParse(data)
if not isinstance(data, bytes):
data = data.encode()
cp_file = confparse.ConfParse(string_io(data))
for cp in (cp_data, cp_file):
assert_equal(cp['fs.default.name'], 'hdfs://localhost:8020')
assert_equal(cp.get('with_description'), 'bar')
assert_equal(cp.get('not_in_xml', 'abc'), 'abc')
assert_equal(cp.getbool('boolean_true'), True)
assert_equal(cp.getbool('boolean_false'), False)
assert_equal(cp.getbool('not_in_xml', True), True)
try:
cp['bogus']
assert_true(False, 'Should not get here')
except KeyError as kerr:
ex = kerr
cp_empty = confparse.ConfParse("")
assert_equal(cp_empty.get('whatever', 'yes'), 'yes')
def _parse_site(site_path):
try:
data = file(site_path, 'r').read()
except IOError as err:
if err.errno != errno.ENOENT:
LOG.error('Cannot read from "%s": %s' % (site_path, err))
return
data = ""
return confparse.ConfParse(data)
def test_confparse():
"""Test configuration parsing"""
data = """
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://localhost:8020</value>
</property>
<property>
<name>with_description</name>
<value>bar</value>
<description>A base for other temporary directories.</description>
</property>
<property>
<name>boolean_true</name>
<value>true</value>
</property>
<property>
<name>boolean_false</name>
<value>false</value>
</property>
</configuration>
"""
cp_data = confparse.ConfParse(data)
cp_file = confparse.ConfParse(cStringIO.StringIO(data))
for cp in (cp_data, cp_file):
assert_equal(cp['fs.default.name'], 'hdfs://localhost:8020')
assert_equal(cp.get('with_description'), 'bar')
assert_equal(cp.get('not_in_xml', 'abc'), 'abc')
assert_equal(cp.getbool('boolean_true'), True)
assert_equal(cp.getbool('boolean_false'), False)
assert_equal(cp.getbool('not_in_xml', True), True)
try:
cp['bogus']
assert_true(False, 'Should not get here')
except KeyError, kerr:
ex = kerr
def _parse_site():
global SITE_DICT
global SITE_PATH
#Avoid circular import
from hbase.conf import HBASE_CONF_DIR
SITE_PATH = os.path.join(HBASE_CONF_DIR.get(), 'hbase-site.xml')
try:
data = file(SITE_PATH, 'r').read()
except IOError as err:
if err.errno != errno.ENOENT:
LOG.error('Cannot read from "%s": %s' % (SITE_PATH, err))
return
data = ""
SITE_DICT = confparse.ConfParse(data)
def _parse_core_site():
"""
Parse core-site.xml and store in _CORE_SITE_DICT
"""
global _CORE_SITE_DICT
global _CORE_SITE_PATH
try:
_CORE_SITE_PATH = get_config_root_hadoop('core-site.xml')
data = open_file(_CORE_SITE_PATH, 'r').read()
except IOError as err:
if err.errno != errno.ENOENT:
LOG.error('Cannot read from "%s": %s' % (_CORE_SITE_PATH, err))
return
# Keep going and make an empty ConfParse
data = ""
_CORE_SITE_DICT = confparse.ConfParse(data)
def _parse_hive_site():
"""
Parse hive-site.xml and store in _HIVE_SITE_DICT
"""
global _HIVE_SITE_DICT
global _HIVE_SITE_PATH
_HIVE_SITE_PATH = os.path.join(beeswax.conf.HIVE_CONF_DIR.get(), 'hive-site.xml')
try:
data = open_file(_HIVE_SITE_PATH, 'r').read()
except IOError as err:
if err.errno != errno.ENOENT:
LOG.error('Cannot read from "%s": %s' % (_HIVE_SITE_PATH, err))
return
# Keep going and make an empty ConfParse
data = ""
_HIVE_SITE_DICT = confparse.ConfParse(data)
def _parse_ssl_client_site():
global _SSL_SITE_DICT
global _SSL_SITE_PATH
for indentifier in conf.HDFS_CLUSTERS.get():
try:
_SSL_SITE_PATH = os.path.join(conf.HDFS_CLUSTERS[indentifier].HADOOP_CONF_DIR.get(), 'ssl-client.xml')
data = file(_SSL_SITE_PATH, 'r').read()
break
except KeyError:
data = ""
except IOError as err:
if err.errno != errno.ENOENT:
LOG.error('Cannot read from "%s": %s' % (_SSL_SITE_PATH, err))
return
# Keep going and make an empty ConfParse
data = ""
_SSL_SITE_DICT = confparse.ConfParse(data)
def get_hiveserver2_authentication():
return get_conf().get(_CNF_HIVESERVER2_AUTHENTICATION,
'NONE').upper() # NONE == PLAIN SASL
def hiveserver2_impersonation_enabled():
return get_conf().get(_CNF_HIVESERVER2_IMPERSONATION,
'FALSE').upper() == 'TRUE'
def _parse_hive_site():
"""
Parse hive-site.xml and store in _HIVE_SITE_DICT
"""
global _HIVE_SITE_DICT
global _HIVE_SITE_PATH
_HIVE_SITE_PATH = os.path.join(beeswax.conf.HIVE_CONF_DIR.get(),
'hive-site.xml')
try:
data = file(_HIVE_SITE_PATH, 'r').read()
except IOError, err:
if err.errno != errno.ENOENT:
LOG.error('Cannot read from "%s": %s' % (_HIVE_SITE_PATH, err))
return
# Keep going and make an empty ConfParse
data = ""
_HIVE_SITE_DICT = confparse.ConfParse(data)
def _parse_sites():
global _SITE_DICT
_SITE_DICT ={}
paths = [
('sentry', os.path.join(SENTRY_CONF_DIR.get(), 'sentry-site.xml')),
]
try:
from beeswax.conf import HIVE_CONF_DIR
paths.append(('hive', os.path.join(HIVE_CONF_DIR.get(), 'sentry-site.xml')))
except Exception, e:
LOG.error('Cannot read Hive sentry site: %s' % e)
for name, path in paths:
_SITE_DICT[name] = _parse_site(path)
def _parse_site(site_path):
try:
data = file(site_path, 'r').read()
except IOError, err:
if err.errno != errno.ENOENT:
LOG.error('Cannot read from "%s": %s' % (site_path, err))
return
data = ""
return confparse.ConfParse(data)
for cp in (cp_data, cp_file):
assert_equal(cp['fs.default.name'], 'hdfs://localhost:8020')
assert_equal(cp.get('with_description'), 'bar')
assert_equal(cp.get('not_in_xml', 'abc'), 'abc')
assert_equal(cp.getbool('boolean_true'), True)
assert_equal(cp.getbool('boolean_false'), False)
assert_equal(cp.getbool('not_in_xml', True), True)
try:
cp['bogus']
assert_true(False, 'Should not get here')
except KeyError, kerr:
ex = kerr
cp_empty = confparse.ConfParse("")
assert_equal(cp_empty.get('whatever', 'yes'), 'yes')
def test_tricky_confparse():
"""
We found (experimentally) that dealing with a file
sometimes triggered the wrong results here.
"""
cp_data = confparse.ConfParse(
file(
os.path.join(os.path.dirname(__file__), "test_data",
"sample_conf.xml")))
assert_equal("org.apache.hadoop.examples.SleepJob",
cp_data["mapred.mapper.class"])
def reset():
global _SENTRY_SITE_DICT
_SENTRY_SITE_DICT = None
def get_conf():
if _SENTRY_SITE_DICT is None:
_parse_site()
return _SENTRY_SITE_DICT
def get_hive_sentry_provider():
return get_conf().get(_CONF_HIVE_PROVIDER, 'default')
def _parse_site():
global _SENTRY_SITE_DICT
global _SENTRY_SITE_PATH
_SENTRY_SITE_PATH = os.path.join(SENTRY_CONF_DIR.get(), 'sentry-site.xml')
try:
data = file(_SENTRY_SITE_PATH, 'r').read()
except IOError, err:
if err.errno != errno.ENOENT:
LOG.error('Cannot read from "%s": %s' % (_SENTRY_SITE_PATH, err))
return
data = ""
_SENTRY_SITE_DICT = confparse.ConfParse(data)
