def test_file_attributes(self): downloader = FakeDownloader(data=None) stream = filesystemio.DownloaderStream(downloader) self.assertEqual(stream.mode, 'rb') self.assertTrue(stream.readable()) self.assertFalse(stream.writable()) self.assertTrue(stream.seekable())
def _open(self, path, mime_type='application/octet-stream', compression_type=CompressionTypes.AUTO): stream = io.BufferedReader( filesystemio.DownloaderStream( HdfsDownloader(self._hdfs_client, path)), buffer_size=_DEFAULT_BUFFER_SIZE) return self._add_compression(stream, path, mime_type, compression_type)
def test_read(self): data = b'abcde' downloader = FakeDownloader(data) stream = filesystemio.DownloaderStream(downloader) # Read size is exactly what was passed to read() (unbuffered). self.assertEqual(stream.read(1), data[0:1]) self.assertEqual(downloader.last_read_size, 1) self.assertEqual(stream.read(), data[1:]) self.assertEqual(downloader.last_read_size, len(data) - 1)
def test_read_buffered(self): data = b'abcde' downloader = FakeDownloader(data) buffer_size = 2 stream = io.BufferedReader(filesystemio.DownloaderStream(downloader), buffer_size) # Verify that buffering works and is reading ahead. self.assertEqual(stream.read(1), data[0:1]) self.assertEqual(downloader.last_read_size, buffer_size) self.assertEqual(stream.read(), data[1:])
def test_read_empty(self): downloader = FakeDownloader(data=b'') stream = filesystemio.DownloaderStream(downloader) self.assertEqual(stream.read(), b'')