def get_bite(self): """ If Luigi has forked, we have a different PID, and need to reconnect. """ config = hdfs_config.hdfs() if self.pid != os.getpid() or not self._bite: client_kwargs = dict( filter( lambda k_v: k_v[1] is not None and k_v[1] != '', six.iteritems({ 'hadoop_version': config.client_version, 'effective_user': config.effective_user, }))) if config.snakebite_autoconfig: """ This is fully backwards compatible with the vanilla Client and can be used for a non HA cluster as well. This client tries to read ``${HADOOP_PATH}/conf/hdfs-site.xml`` to get the address of the namenode. The behaviour is the same as Client. """ from snakebite.client import AutoConfigClient self._bite = AutoConfigClient(**client_kwargs) else: from snakebite.client import Client self._bite = Client(config.namenode_host, config.namenode_port, **client_kwargs) return self._bite
def get_bite(self): """ If Luigi has forked, we have a different PID, and need to reconnect. """ config = hdfs_config.hdfs() if self.pid != os.getpid() or not self._bite: client_kwargs = dict(filter( lambda k_v: k_v[1] is not None and k_v[1] != '', six.iteritems({ 'hadoop_version': config.client_version, 'effective_user': config.effective_user, }) )) if config.snakebite_autoconfig: """ This is fully backwards compatible with the vanilla Client and can be used for a non HA cluster as well. This client tries to read ``${HADOOP_PATH}/conf/hdfs-site.xml`` to get the address of the namenode. The behaviour is the same as Client. """ from snakebite.client import AutoConfigClient self._bite = AutoConfigClient(**client_kwargs) else: from snakebite.client import Client self._bite = Client(config.namenode_host, config.namenode_port, **client_kwargs) return self._bite
def __init__(self, host=None, port=None, user=None): self.host = host or hdfs_config.hdfs().namenode_host self.port = port or webhdfs().port self.user = user or webhdfs().user or os.environ['USER']