Exemplo n.º 1
0
  def OpenBlob(self, blob_key):
    """Open blob file for streaming.

    Args:
      blob_key: Blob-key of existing blob to open for reading.

    Returns:
      Open file stream for reading blob from disk.
    """
    logging.debug('reading blob with key %s' %blob_key)
    return pyhdfs.open(self._fs, self._FileForBlob(blob_key), 'r')
Exemplo n.º 2
0
  def StoreBlob(self, blob_key, blob_stream):
    """Store blob stream to disk.

    Args:
      blob_key: Blob key of blob to store.
      blob_stream: Stream or stream-like object that will generate blob content.
    """
    logging.debug('storing blob with key %s' %blob_key)
    blob_key = self._BlobKey(blob_key)
    blob_directory = self._DirectoryForBlob(blob_key)
    if not pyhdfs.exists(self._fs, blob_directory):
      pyhdfs.mkdir(self._fs, blob_directory)
    blob_file = self._FileForBlob(blob_key)
    hdfs_file = pyhdfs.open(self._fs, blob_file, 'w')

    try:
      while True:
        block = blob_stream.read(1 << 20)
        if not block:
          break
        pyhdfs.write(self._fs, hdfs_file, block)
    finally:
      pyhdfs.close(self._fs, hdfs_file)
Exemplo n.º 3
0
def main():
    print "connecting"
    fs = pyhdfs.connect(host, port)
    
    try:
        print "opening /test/foo for writing"
        f = pyhdfs.open(fs, "/test/foo", "w")
    
        print "writing"
        written = pyhdfs.write(fs, f, "hoho\0haha\nxixi")
        print "written %d bytes" % (written)
        
        print "flushing"
        pyhdfs.flush(fs, f)
    
        print "closing file"
        pyhdfs.close(fs, f)
    
        print "checking existence"
        if pyhdfs.exists(fs, "/test/foo"):
            print "getting"
            pyhdfs.get(fs, "/test/foo", "/tmp/foo.txt")
	    
	print "putting"
	pyhdfs.put(fs, "pyhdfs_test.py", "/test")
        
        print "opening /test/foo for reading"
        f = pyhdfs.open(fs, "/test/foo", "r")
        
        print "reading first 5 bytes"
        s = pyhdfs.read(fs, f, 5)
        print s, len(s)
        
        print "reading remaining"
        s = pyhdfs.read(fs, f)
        print s, len(s)
        
        print "telling"
        print pyhdfs.tell(fs, f)
        
        print "reading"
        s = pyhdfs.read(fs, f)
        print s, len(s)
        
        print "position reading from 5"
        s = pyhdfs.pread(fs, f, 5)
        print s, len(s)
        
        print "seeking"
        pyhdfs.seek(fs, f, 1)
        
        print "telling"
        print pyhdfs.tell(fs, f)
        
        print "closing file"
        pyhdfs.close(fs, f)

        print "updating file time"
        pyhdfs.utime(fs, "/test/foo", int(time.time()), int(time.time()))        
        
        print "stating file"
        print pyhdfs.stat(fs, "/test/foo")
        
	print "stating nosuchfile"
	print pyhdfs.stat(fs, "/test/nosuchfile")

	print "stating dir"
	print pyhdfs.stat(fs, "/test")

	print "mkdir dir /test/foo"
	print pyhdfs.mkdir(fs, "/test/foo")

	print "mkdir dir /test/dir/foo"
	print pyhdfs.mkdir(fs, "/test/dir/foo")
	print pyhdfs.stat(fs, "/test/dir/foo")

        print "listing directory"
        l = pyhdfs.listdir(fs, "/test")
        for i in l:
            print i
        
        print "current working directory"
        print pyhdfs.getcwd(fs)
        
        print "changing to root directory"
        print pyhdfs.chdir(fs, '/')
        
        print "current working directory"
        print pyhdfs.getcwd(fs)    
        
    finally:
        print "disconnecting"
        pyhdfs.disconnect(fs)