def test_iter_without_trailing_newline(self):
write_data = 'a\nb\nc'
hfile = Hfile(hostname, port, path, mode='w')
self.assertTrue(hfile.write(write_data))
hfile.close()
hfile = Hfile(hostname, port, path)
read_data = ''
for line in hfile:
read_data += line
self.assertEqual(write_data, read_data)
def test_filesystem(self):
hfile = Hfile(hostname, port, path, mode='w')
hfile.close()
fs = Hfilesystem(hostname, port)
self.assertTrue(fs.exists(path))
self.assertFalse(fs.exists(path + 'doesnotexist'))
self.assertTrue(fs.rename(path, path + 'renamed'))
self.assertTrue(fs.delete(path + 'renamed'))
self.assertFalse(fs.delete(path))
def test_filesystem(self):
hfile = Hfile(hostname, port, path, mode='w')
hfile.close()
fs = Hfilesystem(hostname, port)
self.assertTrue(fs.exists(path))
self.assertFalse(fs.exists(path + 'doesnotexist'))
self.assertTrue(fs.rename(path, path + 'renamed'))
self.assertTrue(fs.delete(path + 'renamed'))
self.assertFalse(fs.delete(path))
def test_iter_without_trailing_newline(self):
write_data = 'a\nb\nc'
hfile = Hfile(hostname, port, path, mode='w')
self.assertTrue(hfile.write(write_data))
hfile.close()
hfile = Hfile(hostname, port, path)
read_data = ''
for line in hfile:
read_data += line
self.assertEqual(write_data, read_data)
"""Python HDFS use examples. After reading this example you should have enough information to read and write HDFS files from your programs. """ from hdfs.hfile import Hfile hostname = 'hadoop.twitter.com' port = 8020 hdfs_path = '/user/travis/example' local_path = '/etc/motd' # Let's open local and HDFS files. hfile = Hfile(hostname, port, hdfs_path, mode='w') fh = open(local_path) # Now we'll write lines from a local file into the HDFS file. for line in fh: hfile.write(line) # And close them. fh.close() hfile.close() # Let's read local_path into memory for comparison. motd = open(local_path).read() # Now let's read the data back hfile = Hfile(hostname, port, hdfs_path)
def test_file(self):
hfile = Hfile(hostname, port, path, mode='w')
self.assertTrue(hfile.write(data))
hfile.close()
hfile = Hfile(hostname, port, path)
self.assertTrue(hfile.seek(10))
self.assertEqual(hfile.tell(), 10)
hfile.seek(0)
read_data = hfile.read()
self.assertEqual(read_data, data)
hfile.close()
def test_file(self):
hfile = Hfile(hostname, port, path, mode='w')
self.assertTrue(hfile.write(data))
hfile.close()
hfile = Hfile(hostname, port, path)
self.assertTrue(hfile.seek(10))
self.assertEqual(hfile.tell(), 10)
hfile.seek(0)
read_data = hfile.read()
self.assertEqual(read_data, data)
hfile.close()
#coding:UTF-8
#encoding:UTF-8
import pyhdfs
import sys
for pth in sys.path:
print pth
from hdfs.hfile import Hfile
hostname = '192.168.168.162'
port = 8020
hdfs_path = '/examples/hdfs'
local_path = 'E:\NewData\VSP-outlet-1_tomcat_20150317.log'
hfile = Hfile(hostname, port, hdfs_path, mode='w')
fh = open(local_path)
for line in fh:
hfile.write(line)
fh.close()
hfile.close()