Python Hfile Examples

Example #1

File: hfile_test.py Project: liyinhgqw/Coordinator

  def test_iter_without_trailing_newline(self):
    write_data = 'a\nb\nc'
    hfile = Hfile(hostname, port, path, mode='w')
    self.assertTrue(hfile.write(write_data))
    hfile.close()

    hfile = Hfile(hostname, port, path)
    read_data = ''
    for line in hfile:
      read_data += line

    self.assertEqual(write_data, read_data)

Example #2

File: hfilesystem_test.py Project: DipakHore/python-hdfs

  def test_filesystem(self):
    hfile = Hfile(hostname, port, path, mode='w')
    hfile.close()

    fs = Hfilesystem(hostname, port)

    self.assertTrue(fs.exists(path))
    self.assertFalse(fs.exists(path + 'doesnotexist'))

    self.assertTrue(fs.rename(path, path + 'renamed'))

    self.assertTrue(fs.delete(path + 'renamed'))
    self.assertFalse(fs.delete(path))

Example #3

    def test_filesystem(self):
        hfile = Hfile(hostname, port, path, mode='w')
        hfile.close()

        fs = Hfilesystem(hostname, port)

        self.assertTrue(fs.exists(path))
        self.assertFalse(fs.exists(path + 'doesnotexist'))

        self.assertTrue(fs.rename(path, path + 'renamed'))

        self.assertTrue(fs.delete(path + 'renamed'))
        self.assertFalse(fs.delete(path))

Example #4

    def test_iter_without_trailing_newline(self):
        write_data = 'a\nb\nc'
        hfile = Hfile(hostname, port, path, mode='w')
        self.assertTrue(hfile.write(write_data))
        hfile.close()

        hfile = Hfile(hostname, port, path)
        read_data = ''
        for line in hfile:
            read_data += line

        self.assertEqual(write_data, read_data)

Example #5

File: example.py Project: yunchi/python-hdfs

"""Python HDFS use examples.

After reading this example you should have enough information to read and write
HDFS files from your programs.
"""

from hdfs.hfile import Hfile

hostname = 'hadoop.twitter.com'
port = 8020
hdfs_path = '/user/travis/example'
local_path = '/etc/motd'

# Let's open local and HDFS files.

hfile = Hfile(hostname, port, hdfs_path, mode='w')
fh = open(local_path)

# Now we'll write lines from a local file into the HDFS file.
for line in fh:
  hfile.write(line)

# And close them.
fh.close()
hfile.close()

# Let's read local_path into memory for comparison.
motd = open(local_path).read()

# Now let's read the data back
hfile = Hfile(hostname, port, hdfs_path)

Example #6

File: hfile_test.py Project: liyinhgqw/Coordinator

  def test_file(self):
    hfile = Hfile(hostname, port, path, mode='w')
    self.assertTrue(hfile.write(data))
    hfile.close()

    hfile = Hfile(hostname, port, path)

    self.assertTrue(hfile.seek(10))
    self.assertEqual(hfile.tell(), 10)
    hfile.seek(0)

    read_data = hfile.read()
    self.assertEqual(read_data, data)

    hfile.close()

Example #7

    def test_file(self):
        hfile = Hfile(hostname, port, path, mode='w')
        self.assertTrue(hfile.write(data))
        hfile.close()

        hfile = Hfile(hostname, port, path)

        self.assertTrue(hfile.seek(10))
        self.assertEqual(hfile.tell(), 10)
        hfile.seek(0)

        read_data = hfile.read()
        self.assertEqual(read_data, data)

        hfile.close()

Example #8

#coding:UTF-8
#encoding:UTF-8
import pyhdfs
import sys
for pth in sys.path:
    print pth
from hdfs.hfile import Hfile

hostname = '192.168.168.162'
port = 8020
hdfs_path = '/examples/hdfs'
local_path = 'E:\NewData\VSP-outlet-1_tomcat_20150317.log'

hfile = Hfile(hostname, port, hdfs_path, mode='w')
fh = open(local_path)

for line in fh:
    hfile.write(line)

fh.close()
hfile.close()