def test_readlines(self):
self.fs.setbytes('foo', b'barbar\nline1\nline2')
with self.fs.open('foo', 'rb') as f:
f = iotools.make_stream('foo', f, 'rb')
self.assertEqual(list(f), [b'barbar\n', b'line1\n', b'line2'])
with self.fs.open('foo', 'rt') as f:
f = iotools.make_stream('foo', f, 'rb')
self.assertEqual(f.readlines(), ['barbar\n', 'line1\n', 'line2'])
def test_readlines(self):
self.fs.writebytes("foo", b"barbar\nline1\nline2")
with self.fs.open("foo", "rb") as f:
f = iotools.make_stream("foo", f, "rb")
self.assertEqual(list(f), [b"barbar\n", b"line1\n", b"line2"])
with self.fs.open("foo", "rt") as f:
f = iotools.make_stream("foo", f, "rb")
self.assertEqual(f.readlines(), ["barbar\n", "line1\n", "line2"])
def wrapper(self,
path,
mode='rt',
buffering=-1,
encoding=None,
errors=None,
newline=None,
line_buffering=False,
**kwargs):
file_like = f(self,
path,
mode=mode,
buffering=buffering,
encoding=encoding,
errors=errors,
newline=newline,
line_buffering=line_buffering,
**kwargs)
return make_stream(path,
file_like,
mode=mode,
buffering=buffering,
encoding=encoding,
errors=errors,
newline=newline,
line_buffering=line_buffering)
def test_make_stream_reader_writer(self):
f = io.BytesIO(b"Hello")
s = iotools.make_stream("foo", f, "+b", buffering=1)
self.assertIsInstance(s, io.BufferedRandom)
self.assertEqual(s.read(), b"Hello")
s.write(b" World")
self.assertEqual(f.getvalue(), b"Hello World")
def test_make_stream_reader_writer(self):
f = io.BytesIO(b'Hello')
s = iotools.make_stream('foo', f, '+b', buffering=1)
self.assertIsInstance(s, io.BufferedRandom)
self.assertEqual(s.read(), b'Hello')
s.write(b' World')
self.assertEqual(f.getvalue(), b'Hello World')
def openbin(
self,
path, # type: Text
mode="r", # type: Text
buffering=-1, # type: int
**options # type: Any
):
# type: (...) -> BinaryIO
"""Open a binary file-like object.
Arguments:
path (str): A path on the filesystem.
mode (str): Mode to open file (must be a valid non-text mode,
defaults to *r*). Since this method only opens binary files,
the ``b`` in the mode string is implied.
buffering (int): Buffering policy (-1 to use default buffering,
0 to disable buffering, or any positive integer to indicate
a buffer size).
**options: keyword arguments for any additional information
required by the filesystem (if any).
Returns:
io.IOBase: a *file-like* object.
Raises:
fs.errors.FileExpected: If the path is not a file.
fs.errors.FileExists: If the file exists, and *exclusive mode*
is specified (``x`` in the mode).
fs.errors.ResourceNotFound: If the path does not exist.
"""
# TODO: handle BLOB properties here
if mode not in ("r", "rb"):
raise errors.ResourceReadOnly(path)
_res = self._getresource(path)
if not _res:
raise errors.ResourceNotFound(path)
if not isinstance(_res, io.RawIOBase):
if not isinstance(_res, db.DocumentReference) and not isinstance(
_res, db.DocumentSnapshot):
raise TypeError("io stream expected")
# CHECKME: someone wants to read the whole document, so let's give it to them as a json dump
if isinstance(_res, db.DocumentReference):
doc = _res.get()
else:
doc = _res
info = doc.to_dict()
# add other doc properties too?
info.update(doc.__dict__)
data = json.dumps(info, indent=2, default=lambda o: repr(o))
stream = io.BytesIO(data.encode("utf-8"))
name = str(doc.id) + ".json"
return make_stream(name, stream, "rb")
return _res
def test_make_stream(self):
"""Test make_stream"""
with self.get_bin_file() as f:
text = f.read()
self.assert_(isinstance(text, bytes))
with self.get_bin_file() as f:
with iotools.make_stream("data/UTF-8-demo.txt", f, 'rt') as f2:
text = f2.read()
self.assert_(isinstance(text, unicode))
def test_make_stream(self):
"""Test make_stream"""
with self.get_bin_file() as f:
text = f.read()
self.assertTrue(isinstance(text, bytes))
with self.get_bin_file() as f:
with iotools.make_stream("data/UTF-8-demo.txt", f, 'rt') as f2:
text = f2.read()
self.assertTrue(isinstance(text, str))
def test_readinto(self):
self.fs.setbytes('bytes.bin', b'foofoobarbar')
with self.fs.openbin('bytes.bin') as bin_file:
with iotools.make_stream('bytes.bin', bin_file, 'rb') as f:
data = bytearray(3)
bytes_read = f.readinto(data)
self.assertEqual(bytes_read, 3)
self.assertEqual(bytes(data), b'foo')
self.assertEqual(f.readline(1), b'f')
with self.fs.openbin('bytes.bin') as bin_file:
with iotools.make_stream('bytes.bin', bin_file, 'rb') as f:
f.is_io = False
data = bytearray(3)
bytes_read = f.readinto(data)
self.assertEqual(bytes_read, 3)
self.assertEqual(bytes(data), b'foo')
self.assertEqual(f.readline(1), b'f')
def test_readinto1(self):
self.fs.setbytes('bytes.bin', b'foofoobarbar')
with self.fs.openbin('bytes.bin') as bin_file:
with iotools.make_stream('bytes.bin', bin_file, 'rb') as f:
data = bytearray(3)
bytes_read = f.readinto1(data)
self.assertEqual(bytes_read, 3)
self.assertEqual(bytes(data), b'foo')
self.assertEqual(f.readline(1), b'f')
def no_readinto(size):
raise AttributeError
with self.fs.openbin('bytes.bin') as bin_file:
bin_file.readinto = no_readinto
with iotools.make_stream('bytes.bin', bin_file, 'rb') as f:
data = bytearray(3)
bytes_read = f.readinto1(data)
self.assertEqual(bytes_read, 3)
self.assertEqual(bytes(data), b'foo')
self.assertEqual(f.readline(1), b'f')
def test_readinto1(self):
self.fs.writebytes("bytes.bin", b"foofoobarbar")
with self.fs.openbin("bytes.bin") as bin_file:
with iotools.make_stream("bytes.bin", bin_file, "rb") as f:
data = bytearray(3)
bytes_read = f.readinto1(data)
self.assertEqual(bytes_read, 3)
self.assertEqual(bytes(data), b"foo")
self.assertEqual(f.readline(1), b"f")
def no_readinto(size):
raise AttributeError
with self.fs.openbin("bytes.bin") as bin_file:
bin_file.readinto = no_readinto
with iotools.make_stream("bytes.bin", bin_file, "rb") as f:
data = bytearray(3)
bytes_read = f.readinto1(data)
self.assertEqual(bytes_read, 3)
self.assertEqual(bytes(data), b"foo")
self.assertEqual(f.readline(1), b"f")
def test_make_stream(self):
"""Test make_stream"""
self.fs.writebytes("foo.bin", b"foofoo")
with self.fs.openbin("foo.bin") as f:
data = f.read()
self.assertTrue(isinstance(data, bytes))
with self.fs.openbin("text.txt", "wb") as f:
f.write(UNICODE_TEXT.encode("utf-8"))
with self.fs.openbin("text.txt") as f:
with iotools.make_stream("text.txt", f, "rt") as f2:
repr(f2)
text = f2.read()
self.assertIsInstance(text, six.text_type)
def test_make_stream(self):
"""Test make_stream"""
self.fs.setbytes('foo.bin', b'foofoo')
with self.fs.openbin('foo.bin') as f:
data = f.read()
self.assert_(isinstance(data, bytes))
with self.fs.openbin('text.txt', 'wb') as f:
f.write(UNICODE_TEXT.encode('utf-8'))
with self.fs.openbin('text.txt') as f:
with iotools.make_stream("text.txt", f, 'rt') as f2:
repr(f2)
text = f2.read()
self.assertIsInstance(text, six.text_type)
def _getresource(self, path):
# type: (Text) -> bool
"""Get the internal resource for a path (Dir, File or None).
Arguments:
path (str): Path to a resource.
Returns:
resource: internal resource at the given path (Dir, File or None).
"""
_path = self.validatepath(path)
if _path.startswith("/"):
_path = _path[1:]
if _path == "":
# create virtual doc for root
return db.get_client()
parts = _path.split("/")
if len(parts) % 2 == 1:
log.info("Coll %s" % path)
return db.get_coll_ref(_path)
if "[" not in _path or not _path.endswith("]"):
log.info("Doc %s" % path)
return db.get_doc_ref(_path)
# format: id[propname]
log.info("Prop %s" % path)
_path, propname = _path[:-1].split("[", 1)
doc_ref = db.get_doc_ref(_path)
doc = doc_ref.get([propname])
info = doc.to_dict()
# add other doc properties too?
info.update(doc.__dict__)
data = info.get(propname)
if not isinstance(data, (str, bytes)):
data = repr(data)
if isinstance(data, str):
data = data.encode("utf-8")
stream = io.BytesIO(data)
return make_stream(propname, stream, "rb")
def test_make_stream_writer(self):
f = io.BytesIO()
s = iotools.make_stream('foo', f, 'wb', buffering=1)
self.assertIsInstance(s, io.BufferedWriter)
s.write(b'Hello')
self.assertEqual(f.getvalue(), b'Hello')
def test_isatty(self):
with self.fs.openbin("text.txt", "wb") as f:
with iotools.make_stream("text.txt", f, "wb") as f1:
self.assertFalse(f1.isatty())
def test_make_stream_reader(self):
f = io.BytesIO(b"Hello")
s = iotools.make_stream("foo", f, "rb", buffering=1)
self.assertIsInstance(s, io.BufferedReader)
self.assertEqual(s.read(), b"Hello")
def test_make_stream_writer(self):
f = io.BytesIO()
s = iotools.make_stream("foo", f, "wb", buffering=1)
self.assertIsInstance(s, io.BufferedWriter)
s.write(b"Hello")
self.assertEqual(f.getvalue(), b"Hello")
def test_writelines(self):
with self.fs.open("foo", "wb") as f:
f = iotools.make_stream("foo", f, "rb")
f.writelines([b"foo", b"bar", b"baz"])
self.assertEqual(self.fs.readbytes("foo"), b"foobarbaz")
def test_make_stream_reader(self):
f = io.BytesIO(b'Hello')
s = iotools.make_stream('foo', f, 'rb', buffering=1)
self.assertIsInstance(s, io.BufferedReader)
self.assertEqual(s.read(), b'Hello')
def test_writelines(self):
with self.fs.open('foo', 'wb') as f:
f = iotools.make_stream('foo', f, 'rb')
f.writelines([b'foo', b'bar', b'baz'])
self.assertEqual(self.fs.getbytes('foo'), b'foobarbaz')
def _getresource(self, path):
# type: (Text) -> bool
"""Get the internal resource for a path (Dir, File or None).
Arguments:
path (str): Path to a resource.
Returns:
resource: internal resource at the given path (Dir, File or None).
"""
_path = self.validatepath(path)
if _path.startswith("/"):
_path = _path[1:]
if _path == "":
# create virtual instance for root
instance = db.make_instance("__kind__")
instance._properties = []
return instance
parts = _path.split("/")
kind = parts.pop(0)
if kind not in self._kinds:
raise errors.ResourceNotFound(path)
if len(parts) < 1:
instance = db.make_instance(kind)
instance._properties = self._properties[kind]
return instance
# CHECKME: leading / may be eaten already
# id_or_name = "/".join(parts)
id_or_name = parts.pop(0)
id_or_name = id_or_name.replace("§", "/")
if ":" in id_or_name:
id_or_name, parent = id_or_name.split(":", 1)
path_args = parent.split(":")
if id_or_name.isdecimal():
id_or_name = int(id_or_name)
# put back the current kind
key = db.get_key(kind, id_or_name, *path_args)
else:
if id_or_name.isdecimal():
id_or_name = int(id_or_name)
key = db.get_key(kind, id_or_name)
# CHECKME: we need to make the entity ourselves for meta kinds
if kind in self._metakinds:
entity = db.make_entity(key)
else:
entity = db.get_entity(key)
# entity = db.get_entity_by_id(kind, id_or_name)
instance = db.make_instance(kind, entity)
# instance._properties = self._properties[kind]
if len(parts) > 0:
propname = parts.pop(0)
if not hasattr(instance, propname):
raise errors.ResourceNotFound(path)
# return getattr(instance, propname)
data = getattr(instance, propname)
if not isinstance(data, (str, bytes)):
data = repr(data)
if isinstance(data, str):
data = data.encode("utf-8")
stream = io.BytesIO(data)
return make_stream(propname, stream, "rb")
return instance
