def OpenBlob(self, blob_key):
"""Open blob file for streaming.
Args:
blob_key: Blob-key of existing blob to open for reading.
Returns:
Open file stream for reading blob from disk.
"""
logging.debug('reading blob with key %s' %blob_key)
return pyhdfs.open(self._fs, self._FileForBlob(blob_key), 'r')
def OpenBlob(self, blob_key):
"""Open blob file for streaming.
Args:
blob_key: Blob-key of existing blob to open for reading.
Returns:
Open file stream for reading blob from disk.
"""
logging.debug('reading blob with key %s' % blob_key)
return pyhdfs.open(self._fs, self._FileForBlob(blob_key), 'r')
def StoreBlob(self, blob_key, blob_stream):
"""Store blob stream to disk.
Args:
blob_key: Blob key of blob to store.
blob_stream: Stream or stream-like object that will generate blob content.
"""
logging.debug('storing blob with key %s' %blob_key)
blob_key = self._BlobKey(blob_key)
blob_directory = self._DirectoryForBlob(blob_key)
if not pyhdfs.exists(self._fs, blob_directory):
pyhdfs.mkdir(self._fs, blob_directory)
blob_file = self._FileForBlob(blob_key)
hdfs_file = pyhdfs.open(self._fs, blob_file, 'w')
try:
while True:
block = blob_stream.read(1 << 20)
if not block:
break
pyhdfs.write(self._fs, hdfs_file, block)
finally:
pyhdfs.close(self._fs, hdfs_file)
def StoreBlob(self, blob_key, blob_stream):
"""Store blob stream to disk.
Args:
blob_key: Blob key of blob to store.
blob_stream: Stream or stream-like object that will generate blob content.
"""
logging.debug('storing blob with key %s' % blob_key)
blob_key = self._BlobKey(blob_key)
blob_directory = self._DirectoryForBlob(blob_key)
if not pyhdfs.exists(self._fs, blob_directory):
pyhdfs.mkdir(self._fs, blob_directory)
blob_file = self._FileForBlob(blob_key)
hdfs_file = pyhdfs.open(self._fs, blob_file, 'w')
try:
while True:
block = blob_stream.read(1 << 20)
if not block:
break
pyhdfs.write(self._fs, hdfs_file, block)
finally:
pyhdfs.close(self._fs, hdfs_file)
def main():
print("connecting")
fs = pyhdfs.connect(host, port)
try:
print("opening testfile.txt for writing")
f = pyhdfs.open(fs, TEST_ROOT+"testfile.txt", "w")
print( "writing" )
written = pyhdfs.write(fs, f, "hoho\0haha\nxixi")
print( "written %d bytes" % (written) )
print( "flushing" )
pyhdfs.flush(fs, f)
print( "closing file" )
pyhdfs.close(fs, f)
print( "checking existence" )
if pyhdfs.exists(fs, TEST_ROOT+"testfile.txt"):
print( "getting" )
pyhdfs.get(fs, TEST_ROOT+"testfile.txt", "./testfile.txt")
print("putting")
pyhdfs.put(fs, "pyhdfs_test.py", TEST_ROOT)
print( "opening testfile.txt for reading" )
f = pyhdfs.open(fs, TEST_ROOT+"testfile.txt", "r")
print( "reading first 5 bytes" )
s = pyhdfs.read(fs, f, 5)
print( s, len(s) )
print( "reading remaining" )
s = pyhdfs.read(fs, f)
print( s, len(s) )
print( "telling" )
print( pyhdfs.tell(fs, f) )
print( "reading" )
s = pyhdfs.read(fs, f)
print( s, len(s) )
print( "position reading from 5" )
s = pyhdfs.pread(fs, f, 5)
print( s, len(s) )
print( "seeking" )
pyhdfs.seek(fs, f, 1)
print( "telling" )
print( pyhdfs.tell(fs, f) )
print( "closing file" )
pyhdfs.close(fs, f)
print( "updating file time" )
pyhdfs.utime(fs, TEST_ROOT+"testfile.txt", int(time.time()), int(time.time()))
print( "stating file" )
print( pyhdfs.stat(fs, TEST_ROOT+"testfile.txt") )
print( "stating nosuchfile" )
print( pyhdfs.stat(fs, "/test/nosuchfile") )
print( "stating dir" )
print( pyhdfs.stat(fs, "/test") )
print( "mkdir dir testdir" )
print( pyhdfs.mkdir(fs, TEST_ROOT+"testdir") )
print( pyhdfs.stat(fs, TEST_ROOT+"testdir") )
print( "listing directory" )
l = pyhdfs.listdir(fs, TEST_ROOT+"")
for i in l:
print( i )
print( "current working directory" )
print( pyhdfs.getcwd(fs) )
print( "changing to root directory" )
print( pyhdfs.chdir(fs, '/'))
print( "current working directory" )
print( pyhdfs.getcwd(fs) )
finally:
print( "disconnecting")
pyhdfs.disconnect(fs)
def main():
print "connecting"
fs = pyhdfs.connect(host, port)
try:
print "opening /test/foo for writing"
f = pyhdfs.open(fs, "/test/foo", "w")
print "writing"
written = pyhdfs.write(fs, f, "hoho\0haha\nxixi")
print "written %d bytes" % (written)
print "flushing"
pyhdfs.flush(fs, f)
print "closing file"
pyhdfs.close(fs, f)
print "checking existence"
if pyhdfs.exists(fs, "/test/foo"):
print "getting"
pyhdfs.get(fs, "/test/foo", "/tmp/foo.txt")
print "putting"
pyhdfs.put(fs, "pyhdfs_test.py", "/test")
print "opening /test/foo for reading"
f = pyhdfs.open(fs, "/test/foo", "r")
print "reading first 5 bytes"
s = pyhdfs.read(fs, f, 5)
print s, len(s)
print "reading remaining"
s = pyhdfs.read(fs, f)
print s, len(s)
print "telling"
print pyhdfs.tell(fs, f)
print "reading"
s = pyhdfs.read(fs, f)
print s, len(s)
print "position reading from 5"
s = pyhdfs.pread(fs, f, 5)
print s, len(s)
print "seeking"
pyhdfs.seek(fs, f, 1)
print "telling"
print pyhdfs.tell(fs, f)
print "closing file"
pyhdfs.close(fs, f)
print "updating file time"
pyhdfs.utime(fs, "/test/foo", int(time.time()), int(time.time()))
print "stating file"
print pyhdfs.stat(fs, "/test/foo")
print "stating nosuchfile"
print pyhdfs.stat(fs, "/test/nosuchfile")
print "stating dir"
print pyhdfs.stat(fs, "/test")
print "mkdir dir /test/foo"
print pyhdfs.mkdir(fs, "/test/foo")
print "mkdir dir /test/dir/foo"
print pyhdfs.mkdir(fs, "/test/dir/foo")
print pyhdfs.stat(fs, "/test/dir/foo")
print "listing directory"
l = pyhdfs.listdir(fs, "/test")
for i in l:
print i
print "current working directory"
print pyhdfs.getcwd(fs)
print "changing to root directory"
print pyhdfs.chdir(fs, '/')
print "current working directory"
print pyhdfs.getcwd(fs)
finally:
print "disconnecting"
pyhdfs.disconnect(fs)