def main():
with open(os.path.expanduser('~') + '/.whdfsc.json', 'r') as f:
test_config = json.load(f)
hdfs = WebHDFS(**test_config)
print " > echo -n '1234567890' > test.txt"
hdfs.create('test.txt', lsrc=__file__, overwrite=True)
print " > echo -n 'abcdefg' >> test.txt"
hdfs.append('test.txt', data='abcdefg\n')
print " > ls test.txt"
print hdfs.list_status('test.txt')
print " > mkdir example"
print hdfs.mkdirs('example')
print " > ls example"
print hdfs.list_status('example')
print " > mv test.txt example/test.txt"
print hdfs.rename('test.txt', 'example/test.txt')
print " > ls example"
print hdfs.list_status('example')
print " > cat example/test.txt"
print hdfs.open('example/test.txt')
print " > rm -r example"
print hdfs.delete('example', recursive=True)
def __init__(self, service_url):
self.service_url = service_url
try:
result = urlparse.urlparse(service_url)
self.host = result.netloc
self.path = result.path
except:
logger.error("Error parsing URL.")
self.__state = State.New
self.__webhdfs = WebHDFS(self.HDFS_SERVICE_HOST,
self.HDFS_SERVICE_PORT, self.HDFS_USER_NAME)
def __init__(self,
group_name,
topic_name,
max_tmp_size=50,
timeout=15,
filename='config.txt'):
BaseConsumer.__init__(self,
group_name,
topic_name,
timeout=timeout,
filename=filename)
self.ftmp = tempfile.NamedTemporaryFile()
# max_tmp_size comes in unit of MB
self.max_tmp_size = max_tmp_size * 1000 * 1000
self.webhdfs = WebHDFS("c0tl.com", 50070, "hdfs")
from webhdfs.webhdfs import WebHDFS
import os, tempfile
import time
import getpass
webhdfs = WebHDFS("localhost", 50070, getpass.getuser())
webhdfs.mkdir("/hello-world")
# create a temporary file
f = tempfile.NamedTemporaryFile()
f.write(b'Hello world!\n')
f.flush()
print "Upload file: " + f.name
webhdfs.copyFromLocal(f.name, "hello-world/test.txt")
webhdfs.copyToLocal("hello-world/test.txt", "test1.txt")
f.close()
from webhdfs.webhdfs import WebHDFS
import os, tempfile
import time
webhdfs = WebHDFS("localhost", 50070, "luckow")
webhdfs.mkdir("/tmp/hello-world/")
# create a temporary file
f = tempfile.NamedTemporaryFile()
f.write(b'Hello world!\n')
f.flush()
print "Upload file: " + f.name
webhdfs.copyFromLocal(f.name,
"/tmp/test.txt")
webhdfs.copyToLocal("/hello-world/test.txt",
"/tmp/test1.txt")
for i in webhdfs.listdir("/hello-world/"):
print str(i)
f.close()