def test_read_core_config_ha(self): core_site_path = self.get_config_path('ha-core-site.xml') config = HDFSConfig.read_core_config(core_site_path) self.assertEquals(len(config), 1) self.assertEquals('testha', config[0]['namenode']) self.assertEquals(8020, config[0]['port']) self.assertFalse(HDFSConfig.use_trash)
def test_ha_without_ports(self, environ_get): environ_get.return_value = False HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),) HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-hdfs-site.xml'),) config = HDFSConfig.get_external_config() self._verify_hdfs_noport_settings(config)
def test_read_core_config_emr(self): core_site_path = self.get_config_path('emr-core-site.xml') config = HDFSConfig.read_core_config(core_site_path) self.assertEquals(len(config), 1) self.assertEquals('testha', config[0]['namenode']) self.assertEquals(8020, config[0]['port']) self.assertFalse(HDFSConfig.use_trash)
def test_read_core_config_emr(self): core_site_path = self.get_config_path('emr-core-site.xml') config = HDFSConfig.read_core_config(core_site_path) namenodes = config['namenodes'] self.assertEquals(len(namenodes), 1) self.assertEquals('testha', namenodes[0]['namenode']) self.assertEquals(8020, namenodes[0]['port'])
def test_read_core_config_emr(self): core_site_path = self.get_config_path('emr-core-site.xml') config = HDFSConfig.read_core_config(core_site_path) namenodes = config['namenodes'] self.assertEqual(len(namenodes), 1) self.assertEqual('testha', namenodes[0]['namenode']) self.assertEqual(8020, namenodes[0]['port'])
def test_ha_config_trash_in_hdfs(self, environ_get): environ_get.return_value = False HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),) HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-trash-hdfs-site.xml'),) config = HDFSConfig.get_external_config() self._verify_hdfs_noport_settings(config) self.assertTrue(HDFSConfig.use_trash)
def test_retry_configs(self): conf_path = self.get_config_path('ha-retry-hdfs-site.xml') config = HDFSConfig.read_hdfs_config(conf_path) self.assertEquals(config['client_retries'], 5) self.assertEquals(config['client_sleep_base_millis'], 400) self.assertEquals(config['client_sleep_max_millis'], 14000) self.assertEquals(config['socket_timeout_millis'], 25000) self.assertEquals(config['failover_max_attempts'], 7)
def test_ha_config_trash_in_core(self, environ_get): environ_get.return_value = False HDFSConfig.core_try_paths = (self.get_config_path('core-with-trash.xml'),) HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-hdfs-site.xml'),) config = HDFSConfig.get_external_config() self._verify_hdfs_noport_settings(config) self.assertTrue(config['use_trash'])
def test_read_config_non_ha_with_ports(self, environ_get): environ_get.return_value = False HDFSConfig.core_try_paths = (self.get_config_path('non-ha-port-core-site.xml'),) HDFSConfig.hdfs_try_paths = () config = HDFSConfig.get_external_config() self.assertEquals(len(config), 1) self.assertEquals(config[0]['namenode'], 'testhost.net') self.assertEquals(config[0]['port'], 8888) self.assertFalse(HDFSConfig.use_trash)
def read_config(self): self.configs = HDFSConfig.get_external_config() # Try to retrieve namenode config from within CL arguments if self._read_config_cl(): return config_file = os.path.join(os.path.expanduser('~'), '.snakebiterc') if os.path.exists(config_file): #if ~/.snakebiterc exists - read config from it self._read_config_snakebiterc() elif os.path.exists('/etc/snakebiterc'): self._read_config_snakebiterc('/etc/snakebiterc') else: # if configs from HDFS config files exist and contain something if self.configs: for config in self.configs['namenodes']: nn = Namenode(config['namenode'], self.__use_cl_port_first(config['port'])) self.namenodes.append(nn) if self.__usetrash_unset(): self.args.usetrash = self.configs['use_trash'] self.use_sasl = self.configs['use_sasl'] if len(self.namenodes): return else: print( "No ~/.snakebiterc found, no HADOOP_HOME set and no -n and -p provided" ) print("Tried to find core-site.xml in:") for core_conf_path in HDFSConfig.core_try_paths: print(" - %s" % core_conf_path) print("Tried to find hdfs-site.xml in:") for hdfs_conf_path in HDFSConfig.hdfs_try_paths: print(" - %s" % hdfs_conf_path) print( "\nYou can manually create ~/.snakebiterc with the following content:" ) print('{') print(' "config_version": 2,') print(' "use_trash": true,') print(' "namenodes": [') print(' {"host": "namenode-ha1", "port": %d, "version": %d},' % (Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION)) print(' {"host": "namenode-ha2", "port": %d, "version": %d}' % (Namenode.DEFAULT_PORT, Namenode.DEFAULT_VERSION)) print(' ]') print('}') sys.exit(1)
def test_read_hdfs_config_ha(self): hdfs_site_path = self.get_config_path('ha-port-hdfs-site.xml') config = HDFSConfig.read_hdfs_config(hdfs_site_path) self._verify_hdfs_settings(config)
def test_read_hdfs_port_config_ha(self): hdfs_core_path = self.get_config_path('ha-port-hdfs-site.xml') conf = HDFSConfig.read_hadoop_config(hdfs_core_path) config = HDFSConfig.read_hdfs_config('', conf, 'testha', ['namenode1-mydomain', 'namenode2-mydomain']) self._verify_hdfs_port_settings(config)
def test_read_hdfs_config_ha(self): hdfs_core_path = self.get_config_path('ha-port-hdfs-site.xml') conf = HDFSConfig.read_hadoop_config(hdfs_core_path) config = HDFSConfig.read_hdfs_config('', conf, '', []) self._verify_hdfs_settings_all(config)
def test_use_datanode_hostname_configs(self): conf_path = self.get_config_path('use-datanode-hostname-hdfs-site.xml') config = HDFSConfig.read_hdfs_config(conf_path) self.assertTrue(config['use_datanode_hostname'])
def create_client(): configs = HDFSConfig.get_external_config() namenodes = [] config = configs[0] namenode, port = config['namenode'], config['port'] return client.Client(namenode, port=port)