def StoreBlob(self, blob_key, blob_stream):
    """Store blob stream to disk.

    Args:
      blob_key: Blob key of blob to store.
      blob_stream: Stream or stream-like object that will generate blob content.
    """
    logging.debug('storing blob with key %s' %blob_key)
    blob_key = self._BlobKey(blob_key)
    blob_directory = self._DirectoryForBlob(blob_key)
    if not pyhdfs.exists(self._fs, blob_directory):
      pyhdfs.mkdir(self._fs, blob_directory)
    blob_file = self._FileForBlob(blob_key)
    hdfs_file = pyhdfs.open(self._fs, blob_file, 'w')

    try:
      while True:
        block = blob_stream.read(1 << 20)
        if not block:
          break
        pyhdfs.write(self._fs, hdfs_file, block)
    finally:
      pyhdfs.close(self._fs, hdfs_file)
Esempio n. 2
0
    def StoreBlob(self, blob_key, blob_stream):
        """Store blob stream to disk.

    Args:
      blob_key: Blob key of blob to store.
      blob_stream: Stream or stream-like object that will generate blob content.
    """
        logging.debug('storing blob with key %s' % blob_key)
        blob_key = self._BlobKey(blob_key)
        blob_directory = self._DirectoryForBlob(blob_key)
        if not pyhdfs.exists(self._fs, blob_directory):
            pyhdfs.mkdir(self._fs, blob_directory)
        blob_file = self._FileForBlob(blob_key)
        hdfs_file = pyhdfs.open(self._fs, blob_file, 'w')

        try:
            while True:
                block = blob_stream.read(1 << 20)
                if not block:
                    break
                pyhdfs.write(self._fs, hdfs_file, block)
        finally:
            pyhdfs.close(self._fs, hdfs_file)
Esempio n. 3
0
def main():
    print("connecting")
    fs = pyhdfs.connect(host, port)
    
    try:
        print("opening testfile.txt for writing") 
        f = pyhdfs.open(fs, TEST_ROOT+"testfile.txt", "w")
    
        print( "writing" )
        written = pyhdfs.write(fs, f, "hoho\0haha\nxixi")
        print( "written %d bytes" % (written) )
        
        print( "flushing" )
        pyhdfs.flush(fs, f)
    
        print( "closing file" )
        pyhdfs.close(fs, f)
    
        print( "checking existence" )
        if pyhdfs.exists(fs, TEST_ROOT+"testfile.txt"):
            print( "getting" )
            pyhdfs.get(fs, TEST_ROOT+"testfile.txt", "./testfile.txt")
	    
        print("putting")
        pyhdfs.put(fs, "pyhdfs_test.py", TEST_ROOT)
        
        print( "opening testfile.txt for reading" )
        f = pyhdfs.open(fs, TEST_ROOT+"testfile.txt", "r")
        
        print( "reading first 5 bytes" )
        s = pyhdfs.read(fs, f, 5)
        print( s, len(s) )
        
        print( "reading remaining" )
        s = pyhdfs.read(fs, f)
        print( s, len(s) )
        
        print( "telling" )
        print( pyhdfs.tell(fs, f) )
        
        print( "reading" )
        s = pyhdfs.read(fs, f)
        print( s, len(s) )
        
        print( "position reading from 5" )
        s = pyhdfs.pread(fs, f, 5)
        print( s, len(s) )
        
        print( "seeking" )
        pyhdfs.seek(fs, f, 1)
        
        print( "telling" )
        print( pyhdfs.tell(fs, f) )
        
        print( "closing file" )
        pyhdfs.close(fs, f)

        print( "updating file time" )
        pyhdfs.utime(fs, TEST_ROOT+"testfile.txt", int(time.time()), int(time.time()))        
        
        print( "stating file" )
        print( pyhdfs.stat(fs, TEST_ROOT+"testfile.txt") )
        
        print( "stating nosuchfile" )
        print( pyhdfs.stat(fs, "/test/nosuchfile") )

        print( "stating dir" )
        print( pyhdfs.stat(fs, "/test") )

        print( "mkdir dir testdir" )
        print( pyhdfs.mkdir(fs, TEST_ROOT+"testdir") )

        print( pyhdfs.stat(fs, TEST_ROOT+"testdir") )

        print( "listing directory" )
        l = pyhdfs.listdir(fs, TEST_ROOT+"")
        for i in l:
            print( i )

        print( "current working directory" )
        print( pyhdfs.getcwd(fs) )

        print( "changing to root directory" )
        print( pyhdfs.chdir(fs, '/'))

        print( "current working directory" )
        print( pyhdfs.getcwd(fs) )  
            
    finally:
        print( "disconnecting")
        pyhdfs.disconnect(fs)
Esempio n. 4
0
def main():
    print "connecting"
    fs = pyhdfs.connect(host, port)
    
    try:
        print "opening /test/foo for writing"
        f = pyhdfs.open(fs, "/test/foo", "w")
    
        print "writing"
        written = pyhdfs.write(fs, f, "hoho\0haha\nxixi")
        print "written %d bytes" % (written)
        
        print "flushing"
        pyhdfs.flush(fs, f)
    
        print "closing file"
        pyhdfs.close(fs, f)
    
        print "checking existence"
        if pyhdfs.exists(fs, "/test/foo"):
            print "getting"
            pyhdfs.get(fs, "/test/foo", "/tmp/foo.txt")
	    
	print "putting"
	pyhdfs.put(fs, "pyhdfs_test.py", "/test")
        
        print "opening /test/foo for reading"
        f = pyhdfs.open(fs, "/test/foo", "r")
        
        print "reading first 5 bytes"
        s = pyhdfs.read(fs, f, 5)
        print s, len(s)
        
        print "reading remaining"
        s = pyhdfs.read(fs, f)
        print s, len(s)
        
        print "telling"
        print pyhdfs.tell(fs, f)
        
        print "reading"
        s = pyhdfs.read(fs, f)
        print s, len(s)
        
        print "position reading from 5"
        s = pyhdfs.pread(fs, f, 5)
        print s, len(s)
        
        print "seeking"
        pyhdfs.seek(fs, f, 1)
        
        print "telling"
        print pyhdfs.tell(fs, f)
        
        print "closing file"
        pyhdfs.close(fs, f)

        print "updating file time"
        pyhdfs.utime(fs, "/test/foo", int(time.time()), int(time.time()))        
        
        print "stating file"
        print pyhdfs.stat(fs, "/test/foo")
        
	print "stating nosuchfile"
	print pyhdfs.stat(fs, "/test/nosuchfile")

	print "stating dir"
	print pyhdfs.stat(fs, "/test")

	print "mkdir dir /test/foo"
	print pyhdfs.mkdir(fs, "/test/foo")

	print "mkdir dir /test/dir/foo"
	print pyhdfs.mkdir(fs, "/test/dir/foo")
	print pyhdfs.stat(fs, "/test/dir/foo")

        print "listing directory"
        l = pyhdfs.listdir(fs, "/test")
        for i in l:
            print i
        
        print "current working directory"
        print pyhdfs.getcwd(fs)
        
        print "changing to root directory"
        print pyhdfs.chdir(fs, '/')
        
        print "current working directory"
        print pyhdfs.getcwd(fs)    
        
    finally:
        print "disconnecting"
        pyhdfs.disconnect(fs)