def test_fs_configuration(fs_config, hadoop_bin_conf): """Test FS configuration. Returns list of (confvar, error).""" TEST_FILE = '/tmp/.hue_config_test.%s' % (random.randint(0, 9999999999)) res = [ ] res.extend(validate_port(fs_config.NN_THRIFT_PORT)) res.extend(validate_port(fs_config.NN_HDFS_PORT)) if res: return res # Check thrift plugin try: fs = HadoopFileSystem(fs_config.NN_HOST.get(), fs_config.NN_THRIFT_PORT.get(), fs_config.NN_HDFS_PORT.get(), hadoop_bin_conf.get()) fs.setuser(fs.superuser) ls = fs.listdir('/') except TTransport.TTransportException: msg = 'Failed to contact Namenode plugin at %s:%s.' % \ (fs_config.NN_HOST.get(), fs_config.NN_THRIFT_PORT.get()) LOG.exception(msg) res.append((fs_config, msg)) return res except (IOError, IOException): msg = 'Failed to see HDFS root directory at %s. Please check HDFS configuration.' % (fs.uri,) LOG.exception(msg) res.append((fs_config, msg)) return res if 'tmp' not in ls: return res # Check nn port (via upload) try: w_file = fs.open(TEST_FILE, 'w') except OSError, ex: msg = 'Failed to execute Hadoop (%s)' % (hadoop_bin_conf.get(),) LOG.exception(msg) res.append((hadoop_bin_conf, msg)) return res
def test_fs_configuration(fs_config, hadoop_bin_conf): """Test FS configuration. Returns list of (confvar, error).""" TEST_FILE = '/tmp/.hue_config_test.%s' % (random.randint(0, 9999999999)) res = [] res.extend(validate_port(fs_config.NN_THRIFT_PORT)) res.extend(validate_port(fs_config.NN_HDFS_PORT)) if res: return res # Check thrift plugin try: fs = HadoopFileSystem.from_config( fs_config, hadoop_bin_path=hadoop_bin_conf.get()) fs.setuser(fs.superuser) ls = fs.listdir('/') except TTransport.TTransportException: msg = 'Failed to contact Namenode plugin at %s:%s.' % \ (fs_config.NN_HOST.get(), fs_config.NN_THRIFT_PORT.get()) LOG.exception(msg) res.append((fs_config, msg)) return res except (IOError, IOException): msg = 'Failed to see HDFS root directory at %s. Please check HDFS configuration.' % ( fs.uri, ) LOG.exception(msg) res.append((fs_config, msg)) return res if 'tmp' not in ls: return res # Check nn port (via upload) try: w_file = fs.open(TEST_FILE, 'w') except OSError, ex: msg = 'Failed to execute Hadoop (%s)' % (hadoop_bin_conf.get(), ) LOG.exception(msg) res.append((hadoop_bin_conf, msg)) return res
def test_jt_configuration(cluster): """Test FS configuration. Returns list of (confvar, error).""" err = validate_port(cluster.JT_THRIFT_PORT) if err: return err try: jt = LiveJobTracker(cluster.JT_HOST.get(), cluster.JT_THRIFT_PORT.get()) jt.runtime_info() except TTransport.TTransportException: msg = "Failed to contact JobTracker plugin at %s:%s." % (cluster.JT_HOST.get(), cluster.JT_THRIFT_PORT.get()) return [(cluster, msg)] return []
def test_jt_configuration(cluster): """Test FS configuration. Returns list of (confvar, error).""" err = validate_port(cluster.JT_THRIFT_PORT) if err: return err try: jt = LiveJobTracker.from_conf(cluster) jt.runtime_info() except TTransport.TTransportException: msg = 'Failed to contact JobTracker plugin at %s:%s.' % \ (cluster.JT_HOST.get(), cluster.JT_THRIFT_PORT.get()) return [(cluster, msg)] return []