def check_write(self, capacity):
if not capacity.can_write_data():
with self.temporary_fs() as fs:
with pytest.raises(UnsupportedOperation):
_ = fs.put_data('a/1.txt', b'a/1.txt content')
with pytest.raises(UnsupportedOperation):
_ = fs.put_data('b/2.txt', BytesIO(b'b/2.txt content'))
with pytest.raises(UnsupportedOperation):
with maybe_close(fs.open('c/3.txt', 'w')) as f:
f.write(b'c/3.txt content')
return
with self.temporary_fs() as fs:
self.assertDictEqual({}, self.get_snapshot(fs))
# put_data
fs.put_data('a/1.txt', b'to be overwritten')
fs.put_data('a/1.txt', b'a/1.txt content')
fs.put_data('b/2.txt', BytesIO(b'b/2.txt content'))
with pytest.raises(TypeError,
match='`data` must be bytes or a '
'file-like object'):
fs.put_data('err1.txt', u'')
with pytest.raises(TypeError,
match='`data` must be bytes or a '
'file-like object'):
fs.put_data('err2.txt', object())
# open
with maybe_close(fs.open('c/3.txt', 'w')) as f:
f.write(b'to be overwritten')
with maybe_close(fs.open('c/3.txt', 'w')) as f:
f.write(b'c/3.txt content')
with pytest.raises(UnsupportedOperation,
match='Invalid open mode'):
with maybe_close(
fs.open('d/4.txt', 'not-a-possible-write-mode')) as f:
f.write(b'')
self.assertDictEqual(
{
'a/1.txt': (b'a/1.txt content', ),
'b/2.txt': (b'b/2.txt content', ),
'c/3.txt': (b'c/3.txt content', )
}, self.get_snapshot(fs))
def iter_files(self, meta_keys=None):
if meta_keys:
raise UnsupportedOperation()
self.init()
for mi in self._file_obj:
if not mi.isdir():
with maybe_close(self._file_obj.extractfile(mi)) as f:
cnt = f.read()
yield self._canonical_path(mi.name), cnt
def iter_files(self, meta_keys=None):
if meta_keys:
raise UnsupportedOperation()
self.init()
for mi in self._file_obj.infolist():
if not self._isdir(mi):
with maybe_close(self._file_obj.open(mi)) as f:
cnt = f.read()
yield self._canonical_path(mi.filename), cnt
def get_data(self, filename):
"""
Get the content of a file.
Args:
filename (str): The name of the file.
Returns:
bytes: The content of a file.
DataFileNotExist: If `filename` does not exist.
"""
with maybe_close(self.open(filename, 'r')) as f:
return f.read()
def put_data(self, filename, data):
"""
Save the content of a file.
Args:
filename (str): The name of the file.
data (bytes or file-like): The content of the file,
or a file-like object with ``read(size)`` method.
Raises:
UnsupportedOperation: If ``WRITE_DATA`` capacity is absent.
"""
if isinstance(data, six.binary_type):
with maybe_close(self.open(filename, 'w')) as f:
f.write(data)
elif hasattr(data, 'read'):
with maybe_close(self.open(filename, 'w')) as f:
while True:
buf = data.read(self._buffer_size)
if not buf:
break
f.write(buf)
else:
raise TypeError('`data` must be bytes or a file-like object.')
def get_snapshot(self, fs):
client = MongoClient(fs.conn_str)
try:
ret = {}
database = client.get_database('admin')
files_coll = database['test.files']
gridfs = GridFS(database, 'test')
for r in files_coll.find({}):
name = r['filename']
with maybe_close(gridfs.get(r['_id'])) as f:
cnt = f.read()
if 'metadata' in r:
ret[name] = (cnt, r['metadata'])
else:
ret[name] = (cnt, )
return ret
finally:
client.close()
def check_read(self, capacity):
names = ['a/1.txt', 'a/2.htm', 'b/1.md', 'b/2.rst', 'c']
if six.PY2:
to_bytes = lambda s: s
else:
to_bytes = lambda s: s.encode('utf-8')
get_content = lambda n: to_bytes(n) + b' content'
snapshot = {n: (get_content(n), ) for n in names}
with self.temporary_fs(snapshot) as fs:
self.assertDictEqual(snapshot, self.get_snapshot(fs))
# count
self.assertEquals(5, fs.count())
if capacity.can_quick_count():
old_iter_names = fs.iter_names
fs.iter_names = Mock(wraps=old_iter_names)
self.assertEquals(5, fs.count())
self.assertFalse(fs.iter_names.called)
fs.iter_names = old_iter_names
# iter, list and sample names
self.assertListEqual(names, sorted(fs.iter_names()))
self.assertIsInstance(fs.list_names(), list)
self.assertListEqual(names, sorted(fs.list_names()))
if capacity.can_random_sample():
for repeated in range(10):
for k in range(len(names)):
samples = fs.sample_names(k)
self.assertIsInstance(samples, list)
self.assertEquals(k, len(samples))
for sample in samples:
self.assertIn(sample, names)
else:
with pytest.raises(UnsupportedOperation):
_ = fs.sample_names(1)
# iter and sample files
self.assertListEqual([(n, to_bytes(n) + b' content')
for n in names], sorted(fs.iter_files()))
if capacity.can_random_sample():
for repeated in range(10):
for k in range(len(names)):
samples = fs.sample_files(k)
self.assertIsInstance(samples, list)
self.assertEquals(k, len(samples))
for sample in samples:
self.assertIn(sample[0], names)
self.assertEquals(get_content(sample[0]),
sample[1])
else:
with pytest.raises(UnsupportedOperation):
_ = fs.sample_names(1)
# retrieve, get data and open
for n in names:
self.assertEquals(get_content(n), fs.retrieve(n))
self.assertEquals(get_content(n), fs.get_data(n))
with maybe_close(fs.open(n, 'r')) as f:
self.assertEquals(get_content(n), f.read())
with pytest.raises(DataFileNotExist):
_ = fs.retrieve(n + '.invalid')
with pytest.raises(DataFileNotExist):
with maybe_close(fs.open(n + '.invalid', 'r')):
pass
# isfile, batch_isfile
for n in names:
self.assertTrue(fs.isfile(n))
self.assertFalse(fs.isfile(n + '.invalid-ext'))
n_dir = n.rsplit('/', 1)[0]
if n_dir != n:
self.assertFalse(fs.isfile(n_dir))
self.assertListEqual([True] * len(names), fs.batch_isfile(names))
self.assertListEqual([True, False] * len(names),
fs.batch_isfile(
sum([[n, n + '.invalid-ext']
for n in names], [])))