class HDFSSErvice:
namenode_host = "localhost"
namenode_port = "9870"
root_folder = "/"
chunck_size = 100000
def __init__(self):
self._client = HdfsClient(hosts=self.namenode_host + ":" + self.namenode_port, user_name="root")
def get(self, hdfs_path: str):
file_size = self.get_file_size(hdfs_path)
for i in range(0, file_size, self.chunck_size):
file_response = self._client.open(hdfs_path, offset=i, length=i + self.chunck_size)
yield file_response.read()
def append(self, hdfs_path: str, data: bytes):
self.create_if_not_exist(hdfs_path)
self._client.append(hdfs_path, data)
def create_if_not_exist(self, hdfs_path: str):
if not self._client.exists(hdfs_path):
self._client.create(hdfs_path, b"")
def get_messages_number(self, hdfs_path: str):
return int(self.get_file_size(hdfs_path) / self.chunck_size + 1)
def get_file_size(self, hdfs_path):
file_infos = self._client.get_content_summary(hdfs_path)
return file_infos.length
def test(self):
pass
def basic():
client = HdfsClient(hosts='study:50070')
print(client.list_status('/'))
print '判断某个路径是否存在'
print client.exists("/test")
print client.exists("/data/gz/thrift-0.9.2.tar.gz")
client = HdfsClient(hosts='study:50070')
print client.get_file_checksum("/data/gz/bison-2.5.1.tar.gz")
summary = client.get_content_summary("/")
print summary
#文件拷贝--从HDFS拷贝到本地磁盘系统
client.copy_to_local("/data/gz/pip-7.1.2.tar.gz","/root/data/pip-7.1.2.tar.gz")
#文件拷贝--从本地磁盘系统拷贝到HDFS系统中
client.copy_from_local("/root/data/thrift-0.9.2.tar.gz","/data/gz/thrift-0.9.2.tar.gz")
print client.get_home_directory()
