# check if running on Linux cluster or local Mac
_ON_LINUX_CLUSTER: bool = sys.platform.startswith('linux')
# detect & set up HDFS client
if _HADOOP_HOME:
os.environ['ARROW_LIBHDFS_DIR'] = \
str(Path(_HADOOP_HOME).resolve(strict=True) / 'lib' / 'native')
try:
HDFS_CLIENT = HadoopFileSystem()
try:
_LOGGER.debug(msg=(msg := 'Testing HDFS...'))
if HDFS_CLIENT.isdir(path='/'):
_ON_LINUX_CLUSTER_WITH_HDFS: bool = True
_LOGGER.debug(msg=f'{msg} done!')
else:
_ON_LINUX_CLUSTER_WITH_HDFS: bool = False
_LOGGER.debug(msg=f'{msg} UNAVAILABLE')
except Exception: # pylint: disable=broad-except
HDFS_CLIENT = None
_ON_LINUX_CLUSTER_WITH_HDFS: bool = False
_LOGGER.debug(msg=f'{msg} UNAVAILABLE')
except Exception: # pylint: disable=broad-except
HDFS_CLIENT = None
_ON_LINUX_CLUSTER_WITH_HDFS: bool = False
# detect & set up HDFS client
if _HADOOP_HOME:
os.environ['ARROW_LIBHDFS_DIR'] = \
os.path.join(
_HADOOP_HOME,
'lib',
'native')
try:
hdfs_client = HadoopFileSystem()
try:
print('Testing HDFS... ', end='')
if hdfs_client.isdir('/'):
_ON_LINUX_CLUSTER_WITH_HDFS = True
print('done!')
else:
_ON_LINUX_CLUSTER_WITH_HDFS = False
print('UNAVAILABLE')
except:
hdfs_client = None
_ON_LINUX_CLUSTER_WITH_HDFS = False
print('UNAVAILABLE')
except:
hdfs_client = None
_ON_LINUX_CLUSTER_WITH_HDFS = False