def test_ha_without_ports(self, environ_get):
environ_get.return_value = False
HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),)
HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-hdfs-site.xml'),)
config = HDFSConfig.get_external_config()
self._verify_hdfs_noport_settings(config)
def test_ha_without_ports(self, environ_get):
environ_get.return_value = False
HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),)
HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-hdfs-site.xml'),)
config = HDFSConfig.get_external_config()
self._verify_hdfs_noport_settings(config)
def test_ha_config_trash_in_core(self, environ_get):
environ_get.return_value = False
HDFSConfig.core_try_paths = (self.get_config_path('core-with-trash.xml'),)
HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-hdfs-site.xml'),)
config = HDFSConfig.get_external_config()
self._verify_hdfs_noport_settings(config)
self.assertTrue(config['use_trash'])
def test_ha_config_trash_in_hdfs(self, environ_get):
environ_get.return_value = False
HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),)
HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-trash-hdfs-site.xml'),)
config = HDFSConfig.get_external_config()
self._verify_hdfs_noport_settings(config)
self.assertTrue(HDFSConfig.use_trash)
def test_read_config_non_ha_with_ports(self, environ_get):
environ_get.return_value = False
HDFSConfig.core_try_paths = (self.get_config_path('non-ha-port-core-site.xml'),)
HDFSConfig.hdfs_try_paths = ()
config = HDFSConfig.get_external_config()
self.assertEquals(len(config), 1)
self.assertEquals(config[0]['namenode'], 'testhost.net')
self.assertEquals(config[0]['port'], 8888)
self.assertFalse(HDFSConfig.use_trash)
def test_read_config_non_ha_with_ports(self, environ_get):
environ_get.return_value = False
HDFSConfig.core_try_paths = (self.get_config_path('non-ha-port-core-site.xml'),)
HDFSConfig.hdfs_try_paths = ()
config = HDFSConfig.get_external_config()
self.assertEquals(len(config), 1)
self.assertEquals(config[0]['namenode'], 'testhost.net')
self.assertEquals(config[0]['port'], 8888)
self.assertFalse(HDFSConfig.use_trash)
def read_config(self):
self.configs = HDFSConfig.get_external_config()
# Try to retrieve namenode config from within CL arguments
if self._read_config_cl():
return
config_file = os.path.join(os.path.expanduser('~'), '.snakebiterc')
if os.path.exists(config_file):
#if ~/.snakebiterc exists - read config from it
self._read_config_snakebiterc()
elif os.path.exists('/etc/snakebiterc'):
self._read_config_snakebiterc('/etc/snakebiterc')
else:
# if configs from HDFS config files exist and contain something
if self.configs:
for config in self.configs['namenodes']:
nn = Namenode(config['namenode'],
self.__use_cl_port_first(config['port']))
self.namenodes.append(nn)
if self.__usetrash_unset():
self.args.usetrash = self.configs['use_trash']
self.use_sasl = self.configs['use_sasl']
if len(self.namenodes):
return
else:
print(
"No ~/.snakebiterc found, no HADOOP_HOME set and no -n and -p provided"
)
print("Tried to find core-site.xml in:")
for core_conf_path in HDFSConfig.core_try_paths:
print(" - %s" % core_conf_path)
print("Tried to find hdfs-site.xml in:")
for hdfs_conf_path in HDFSConfig.hdfs_try_paths:
print(" - %s" % hdfs_conf_path)
print(
"\nYou can manually create ~/.snakebiterc with the following content:"
)
print('{')
print(' "config_version": 2,')
print(' "use_trash": true,')
print(' "namenodes": [')
print(' {"host": "namenode-ha1", "port": %d, "version": %d},' %
(Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION))
print(' {"host": "namenode-ha2", "port": %d, "version": %d}' %
(Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION))
print(' ]')
print('}')
sys.exit(1)
def create_client():
configs = HDFSConfig.get_external_config()
namenodes = []
config = configs[0]
namenode, port = config['namenode'], config['port']
return client.Client(namenode, port=port)
def create_client():
configs = HDFSConfig.get_external_config()
namenodes = []
config = configs[0]
namenode, port = config['namenode'], config['port']
return client.Client(namenode, port=port)
