def test_read_core_config_ha(self):
core_site_path = self.get_config_path('ha-core-site.xml')
config = HDFSConfig.read_core_config(core_site_path)
self.assertEquals(len(config), 1)
self.assertEquals('testha', config[0]['namenode'])
self.assertEquals(8020, config[0]['port'])
self.assertFalse(HDFSConfig.use_trash)
def test_ha_without_ports(self, environ_get):
environ_get.return_value = False
HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),)
HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-hdfs-site.xml'),)
config = HDFSConfig.get_external_config()
self._verify_hdfs_noport_settings(config)
def test_read_core_config_emr(self):
core_site_path = self.get_config_path('emr-core-site.xml')
config = HDFSConfig.read_core_config(core_site_path)
self.assertEquals(len(config), 1)
self.assertEquals('testha', config[0]['namenode'])
self.assertEquals(8020, config[0]['port'])
self.assertFalse(HDFSConfig.use_trash)
def test_read_core_config_emr(self):
core_site_path = self.get_config_path('emr-core-site.xml')
config = HDFSConfig.read_core_config(core_site_path)
namenodes = config['namenodes']
self.assertEquals(len(namenodes), 1)
self.assertEquals('testha', namenodes[0]['namenode'])
self.assertEquals(8020, namenodes[0]['port'])
def test_read_core_config_emr(self):
core_site_path = self.get_config_path('emr-core-site.xml')
config = HDFSConfig.read_core_config(core_site_path)
namenodes = config['namenodes']
self.assertEqual(len(namenodes), 1)
self.assertEqual('testha', namenodes[0]['namenode'])
self.assertEqual(8020, namenodes[0]['port'])
def test_ha_without_ports(self, environ_get):
environ_get.return_value = False
HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),)
HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-hdfs-site.xml'),)
config = HDFSConfig.get_external_config()
self._verify_hdfs_noport_settings(config)
def test_ha_config_trash_in_hdfs(self, environ_get):
environ_get.return_value = False
HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),)
HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-trash-hdfs-site.xml'),)
config = HDFSConfig.get_external_config()
self._verify_hdfs_noport_settings(config)
self.assertTrue(HDFSConfig.use_trash)
def test_retry_configs(self):
conf_path = self.get_config_path('ha-retry-hdfs-site.xml')
config = HDFSConfig.read_hdfs_config(conf_path)
self.assertEquals(config['client_retries'], 5)
self.assertEquals(config['client_sleep_base_millis'], 400)
self.assertEquals(config['client_sleep_max_millis'], 14000)
self.assertEquals(config['socket_timeout_millis'], 25000)
self.assertEquals(config['failover_max_attempts'], 7)
def test_retry_configs(self):
conf_path = self.get_config_path('ha-retry-hdfs-site.xml')
config = HDFSConfig.read_hdfs_config(conf_path)
self.assertEquals(config['client_retries'], 5)
self.assertEquals(config['client_sleep_base_millis'], 400)
self.assertEquals(config['client_sleep_max_millis'], 14000)
self.assertEquals(config['socket_timeout_millis'], 25000)
self.assertEquals(config['failover_max_attempts'], 7)
def test_ha_config_trash_in_core(self, environ_get):
environ_get.return_value = False
HDFSConfig.core_try_paths = (self.get_config_path('core-with-trash.xml'),)
HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-hdfs-site.xml'),)
config = HDFSConfig.get_external_config()
self._verify_hdfs_noport_settings(config)
self.assertTrue(config['use_trash'])
def test_read_config_non_ha_with_ports(self, environ_get):
environ_get.return_value = False
HDFSConfig.core_try_paths = (self.get_config_path('non-ha-port-core-site.xml'),)
HDFSConfig.hdfs_try_paths = ()
config = HDFSConfig.get_external_config()
self.assertEquals(len(config), 1)
self.assertEquals(config[0]['namenode'], 'testhost.net')
self.assertEquals(config[0]['port'], 8888)
self.assertFalse(HDFSConfig.use_trash)
def test_read_config_non_ha_with_ports(self, environ_get):
environ_get.return_value = False
HDFSConfig.core_try_paths = (self.get_config_path('non-ha-port-core-site.xml'),)
HDFSConfig.hdfs_try_paths = ()
config = HDFSConfig.get_external_config()
self.assertEquals(len(config), 1)
self.assertEquals(config[0]['namenode'], 'testhost.net')
self.assertEquals(config[0]['port'], 8888)
self.assertFalse(HDFSConfig.use_trash)
def read_config(self):
self.configs = HDFSConfig.get_external_config()
# Try to retrieve namenode config from within CL arguments
if self._read_config_cl():
return
config_file = os.path.join(os.path.expanduser('~'), '.snakebiterc')
if os.path.exists(config_file):
#if ~/.snakebiterc exists - read config from it
self._read_config_snakebiterc()
elif os.path.exists('/etc/snakebiterc'):
self._read_config_snakebiterc('/etc/snakebiterc')
else:
# if configs from HDFS config files exist and contain something
if self.configs:
for config in self.configs['namenodes']:
nn = Namenode(config['namenode'],
self.__use_cl_port_first(config['port']))
self.namenodes.append(nn)
if self.__usetrash_unset():
self.args.usetrash = self.configs['use_trash']
self.use_sasl = self.configs['use_sasl']
if len(self.namenodes):
return
else:
print(
"No ~/.snakebiterc found, no HADOOP_HOME set and no -n and -p provided"
)
print("Tried to find core-site.xml in:")
for core_conf_path in HDFSConfig.core_try_paths:
print(" - %s" % core_conf_path)
print("Tried to find hdfs-site.xml in:")
for hdfs_conf_path in HDFSConfig.hdfs_try_paths:
print(" - %s" % hdfs_conf_path)
print(
"\nYou can manually create ~/.snakebiterc with the following content:"
)
print('{')
print(' "config_version": 2,')
print(' "use_trash": true,')
print(' "namenodes": [')
print(' {"host": "namenode-ha1", "port": %d, "version": %d},' %
(Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION))
print(' {"host": "namenode-ha2", "port": %d, "version": %d}' %
(Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION))
print(' ]')
print('}')
sys.exit(1)
def test_read_hdfs_config_ha(self):
hdfs_site_path = self.get_config_path('ha-port-hdfs-site.xml')
config = HDFSConfig.read_hdfs_config(hdfs_site_path)
self._verify_hdfs_settings(config)
def test_read_hdfs_port_config_ha(self):
hdfs_core_path = self.get_config_path('ha-port-hdfs-site.xml')
conf = HDFSConfig.read_hadoop_config(hdfs_core_path)
config = HDFSConfig.read_hdfs_config('', conf, 'testha', ['namenode1-mydomain', 'namenode2-mydomain'])
self._verify_hdfs_port_settings(config)
def test_read_hdfs_config_ha(self):
hdfs_core_path = self.get_config_path('ha-port-hdfs-site.xml')
conf = HDFSConfig.read_hadoop_config(hdfs_core_path)
config = HDFSConfig.read_hdfs_config('', conf, '', [])
self._verify_hdfs_settings_all(config)
def test_read_hdfs_config_ha(self):
hdfs_site_path = self.get_config_path('ha-port-hdfs-site.xml')
config = HDFSConfig.read_hdfs_config(hdfs_site_path)
self._verify_hdfs_settings(config)
def test_use_datanode_hostname_configs(self):
conf_path = self.get_config_path('use-datanode-hostname-hdfs-site.xml')
config = HDFSConfig.read_hdfs_config(conf_path)
self.assertTrue(config['use_datanode_hostname'])
def test_use_datanode_hostname_configs(self):
conf_path = self.get_config_path('use-datanode-hostname-hdfs-site.xml')
config = HDFSConfig.read_hdfs_config(conf_path)
self.assertTrue(config['use_datanode_hostname'])
def create_client():
configs = HDFSConfig.get_external_config()
namenodes = []
config = configs[0]
namenode, port = config['namenode'], config['port']
return client.Client(namenode, port=port)
def create_client():
configs = HDFSConfig.get_external_config()
namenodes = []
config = configs[0]
namenode, port = config['namenode'], config['port']
return client.Client(namenode, port=port)
