def test_tricky_confparse(): """ We found (experimentally) that dealing with a file sometimes triggered the wrong results here. """ cp_data = confparse.ConfParse(open(os.path.join(os.path.dirname(__file__), "test_data", "sample_conf.xml"), 'rb')) assert_equal("org.apache.hadoop.examples.SleepJob", cp_data["mapred.mapper.class"])
def get_jobconf(jt, jobid): """ Returns a dict representation of the jobconf for the job corresponding to jobid. filter_keys is an optional list of configuration keys to filter on. """ jid = jt.thriftjobid_from_string(jobid) # This will throw if the the jobconf can't be found xml_data = jt.get_job_xml(jid) return confparse.ConfParse(xml_data)
def test_confparse(): data = """ <configuration> <property> <name>fs.default.name</name> <value>hdfs://localhost:8020</value> </property> <property> <name>with_description</name> <value>bar</value> <description>A base for other temporary directories.</description> </property> <property> <name>boolean_true</name> <value>true</value> </property> <property> <name>boolean_false</name> <value>false</value> </property> </configuration> """ cp_data = confparse.ConfParse(data) if not isinstance(data, bytes): data = data.encode() cp_file = confparse.ConfParse(string_io(data)) for cp in (cp_data, cp_file): assert_equal(cp['fs.default.name'], 'hdfs://localhost:8020') assert_equal(cp.get('with_description'), 'bar') assert_equal(cp.get('not_in_xml', 'abc'), 'abc') assert_equal(cp.getbool('boolean_true'), True) assert_equal(cp.getbool('boolean_false'), False) assert_equal(cp.getbool('not_in_xml', True), True) try: cp['bogus'] assert_true(False, 'Should not get here') except KeyError as kerr: ex = kerr cp_empty = confparse.ConfParse("") assert_equal(cp_empty.get('whatever', 'yes'), 'yes')
def _parse_site(site_path): try: data = file(site_path, 'r').read() except IOError as err: if err.errno != errno.ENOENT: LOG.error('Cannot read from "%s": %s' % (site_path, err)) return data = "" return confparse.ConfParse(data)
def test_confparse(): """Test configuration parsing""" data = """ <configuration> <property> <name>fs.default.name</name> <value>hdfs://localhost:8020</value> </property> <property> <name>with_description</name> <value>bar</value> <description>A base for other temporary directories.</description> </property> <property> <name>boolean_true</name> <value>true</value> </property> <property> <name>boolean_false</name> <value>false</value> </property> </configuration> """ cp_data = confparse.ConfParse(data) cp_file = confparse.ConfParse(cStringIO.StringIO(data)) for cp in (cp_data, cp_file): assert_equal(cp['fs.default.name'], 'hdfs://localhost:8020') assert_equal(cp.get('with_description'), 'bar') assert_equal(cp.get('not_in_xml', 'abc'), 'abc') assert_equal(cp.getbool('boolean_true'), True) assert_equal(cp.getbool('boolean_false'), False) assert_equal(cp.getbool('not_in_xml', True), True) try: cp['bogus'] assert_true(False, 'Should not get here') except KeyError, kerr: ex = kerr
def _parse_site(): global SITE_DICT global SITE_PATH #Avoid circular import from hbase.conf import HBASE_CONF_DIR SITE_PATH = os.path.join(HBASE_CONF_DIR.get(), 'hbase-site.xml') try: data = file(SITE_PATH, 'r').read() except IOError as err: if err.errno != errno.ENOENT: LOG.error('Cannot read from "%s": %s' % (SITE_PATH, err)) return data = "" SITE_DICT = confparse.ConfParse(data)
def _parse_core_site(): """ Parse core-site.xml and store in _CORE_SITE_DICT """ global _CORE_SITE_DICT global _CORE_SITE_PATH try: _CORE_SITE_PATH = get_config_root_hadoop('core-site.xml') data = open_file(_CORE_SITE_PATH, 'r').read() except IOError as err: if err.errno != errno.ENOENT: LOG.error('Cannot read from "%s": %s' % (_CORE_SITE_PATH, err)) return # Keep going and make an empty ConfParse data = "" _CORE_SITE_DICT = confparse.ConfParse(data)
def _parse_hive_site(): """ Parse hive-site.xml and store in _HIVE_SITE_DICT """ global _HIVE_SITE_DICT global _HIVE_SITE_PATH _HIVE_SITE_PATH = os.path.join(beeswax.conf.HIVE_CONF_DIR.get(), 'hive-site.xml') try: data = open_file(_HIVE_SITE_PATH, 'r').read() except IOError as err: if err.errno != errno.ENOENT: LOG.error('Cannot read from "%s": %s' % (_HIVE_SITE_PATH, err)) return # Keep going and make an empty ConfParse data = "" _HIVE_SITE_DICT = confparse.ConfParse(data)
def _parse_ssl_client_site(): global _SSL_SITE_DICT global _SSL_SITE_PATH for indentifier in conf.HDFS_CLUSTERS.get(): try: _SSL_SITE_PATH = os.path.join(conf.HDFS_CLUSTERS[indentifier].HADOOP_CONF_DIR.get(), 'ssl-client.xml') data = file(_SSL_SITE_PATH, 'r').read() break except KeyError: data = "" except IOError as err: if err.errno != errno.ENOENT: LOG.error('Cannot read from "%s": %s' % (_SSL_SITE_PATH, err)) return # Keep going and make an empty ConfParse data = "" _SSL_SITE_DICT = confparse.ConfParse(data)
def get_hiveserver2_authentication(): return get_conf().get(_CNF_HIVESERVER2_AUTHENTICATION, 'NONE').upper() # NONE == PLAIN SASL def hiveserver2_impersonation_enabled(): return get_conf().get(_CNF_HIVESERVER2_IMPERSONATION, 'FALSE').upper() == 'TRUE' def _parse_hive_site(): """ Parse hive-site.xml and store in _HIVE_SITE_DICT """ global _HIVE_SITE_DICT global _HIVE_SITE_PATH _HIVE_SITE_PATH = os.path.join(beeswax.conf.HIVE_CONF_DIR.get(), 'hive-site.xml') try: data = file(_HIVE_SITE_PATH, 'r').read() except IOError, err: if err.errno != errno.ENOENT: LOG.error('Cannot read from "%s": %s' % (_HIVE_SITE_PATH, err)) return # Keep going and make an empty ConfParse data = "" _HIVE_SITE_DICT = confparse.ConfParse(data)
def _parse_sites(): global _SITE_DICT _SITE_DICT ={} paths = [ ('sentry', os.path.join(SENTRY_CONF_DIR.get(), 'sentry-site.xml')), ] try: from beeswax.conf import HIVE_CONF_DIR paths.append(('hive', os.path.join(HIVE_CONF_DIR.get(), 'sentry-site.xml'))) except Exception, e: LOG.error('Cannot read Hive sentry site: %s' % e) for name, path in paths: _SITE_DICT[name] = _parse_site(path) def _parse_site(site_path): try: data = file(site_path, 'r').read() except IOError, err: if err.errno != errno.ENOENT: LOG.error('Cannot read from "%s": %s' % (site_path, err)) return data = "" return confparse.ConfParse(data)
for cp in (cp_data, cp_file): assert_equal(cp['fs.default.name'], 'hdfs://localhost:8020') assert_equal(cp.get('with_description'), 'bar') assert_equal(cp.get('not_in_xml', 'abc'), 'abc') assert_equal(cp.getbool('boolean_true'), True) assert_equal(cp.getbool('boolean_false'), False) assert_equal(cp.getbool('not_in_xml', True), True) try: cp['bogus'] assert_true(False, 'Should not get here') except KeyError, kerr: ex = kerr cp_empty = confparse.ConfParse("") assert_equal(cp_empty.get('whatever', 'yes'), 'yes') def test_tricky_confparse(): """ We found (experimentally) that dealing with a file sometimes triggered the wrong results here. """ cp_data = confparse.ConfParse( file( os.path.join(os.path.dirname(__file__), "test_data", "sample_conf.xml"))) assert_equal("org.apache.hadoop.examples.SleepJob", cp_data["mapred.mapper.class"])
def reset(): global _SENTRY_SITE_DICT _SENTRY_SITE_DICT = None def get_conf(): if _SENTRY_SITE_DICT is None: _parse_site() return _SENTRY_SITE_DICT def get_hive_sentry_provider(): return get_conf().get(_CONF_HIVE_PROVIDER, 'default') def _parse_site(): global _SENTRY_SITE_DICT global _SENTRY_SITE_PATH _SENTRY_SITE_PATH = os.path.join(SENTRY_CONF_DIR.get(), 'sentry-site.xml') try: data = file(_SENTRY_SITE_PATH, 'r').read() except IOError, err: if err.errno != errno.ENOENT: LOG.error('Cannot read from "%s": %s' % (_SENTRY_SITE_PATH, err)) return data = "" _SENTRY_SITE_DICT = confparse.ConfParse(data)