def check_write(self, capacity): if not capacity.can_write_data(): with self.temporary_fs() as fs: with pytest.raises(UnsupportedOperation): _ = fs.put_data('a/1.txt', b'a/1.txt content') with pytest.raises(UnsupportedOperation): _ = fs.put_data('b/2.txt', BytesIO(b'b/2.txt content')) with pytest.raises(UnsupportedOperation): with maybe_close(fs.open('c/3.txt', 'w')) as f: f.write(b'c/3.txt content') return with self.temporary_fs() as fs: self.assertDictEqual({}, self.get_snapshot(fs)) # put_data fs.put_data('a/1.txt', b'to be overwritten') fs.put_data('a/1.txt', b'a/1.txt content') fs.put_data('b/2.txt', BytesIO(b'b/2.txt content')) with pytest.raises(TypeError, match='`data` must be bytes or a ' 'file-like object'): fs.put_data('err1.txt', u'') with pytest.raises(TypeError, match='`data` must be bytes or a ' 'file-like object'): fs.put_data('err2.txt', object()) # open with maybe_close(fs.open('c/3.txt', 'w')) as f: f.write(b'to be overwritten') with maybe_close(fs.open('c/3.txt', 'w')) as f: f.write(b'c/3.txt content') with pytest.raises(UnsupportedOperation, match='Invalid open mode'): with maybe_close( fs.open('d/4.txt', 'not-a-possible-write-mode')) as f: f.write(b'') self.assertDictEqual( { 'a/1.txt': (b'a/1.txt content', ), 'b/2.txt': (b'b/2.txt content', ), 'c/3.txt': (b'c/3.txt content', ) }, self.get_snapshot(fs))
def iter_files(self, meta_keys=None): if meta_keys: raise UnsupportedOperation() self.init() for mi in self._file_obj: if not mi.isdir(): with maybe_close(self._file_obj.extractfile(mi)) as f: cnt = f.read() yield self._canonical_path(mi.name), cnt
def iter_files(self, meta_keys=None): if meta_keys: raise UnsupportedOperation() self.init() for mi in self._file_obj.infolist(): if not self._isdir(mi): with maybe_close(self._file_obj.open(mi)) as f: cnt = f.read() yield self._canonical_path(mi.filename), cnt
def get_data(self, filename): """ Get the content of a file. Args: filename (str): The name of the file. Returns: bytes: The content of a file. DataFileNotExist: If `filename` does not exist. """ with maybe_close(self.open(filename, 'r')) as f: return f.read()
def put_data(self, filename, data): """ Save the content of a file. Args: filename (str): The name of the file. data (bytes or file-like): The content of the file, or a file-like object with ``read(size)`` method. Raises: UnsupportedOperation: If ``WRITE_DATA`` capacity is absent. """ if isinstance(data, six.binary_type): with maybe_close(self.open(filename, 'w')) as f: f.write(data) elif hasattr(data, 'read'): with maybe_close(self.open(filename, 'w')) as f: while True: buf = data.read(self._buffer_size) if not buf: break f.write(buf) else: raise TypeError('`data` must be bytes or a file-like object.')
def get_snapshot(self, fs): client = MongoClient(fs.conn_str) try: ret = {} database = client.get_database('admin') files_coll = database['test.files'] gridfs = GridFS(database, 'test') for r in files_coll.find({}): name = r['filename'] with maybe_close(gridfs.get(r['_id'])) as f: cnt = f.read() if 'metadata' in r: ret[name] = (cnt, r['metadata']) else: ret[name] = (cnt, ) return ret finally: client.close()
def check_read(self, capacity): names = ['a/1.txt', 'a/2.htm', 'b/1.md', 'b/2.rst', 'c'] if six.PY2: to_bytes = lambda s: s else: to_bytes = lambda s: s.encode('utf-8') get_content = lambda n: to_bytes(n) + b' content' snapshot = {n: (get_content(n), ) for n in names} with self.temporary_fs(snapshot) as fs: self.assertDictEqual(snapshot, self.get_snapshot(fs)) # count self.assertEquals(5, fs.count()) if capacity.can_quick_count(): old_iter_names = fs.iter_names fs.iter_names = Mock(wraps=old_iter_names) self.assertEquals(5, fs.count()) self.assertFalse(fs.iter_names.called) fs.iter_names = old_iter_names # iter, list and sample names self.assertListEqual(names, sorted(fs.iter_names())) self.assertIsInstance(fs.list_names(), list) self.assertListEqual(names, sorted(fs.list_names())) if capacity.can_random_sample(): for repeated in range(10): for k in range(len(names)): samples = fs.sample_names(k) self.assertIsInstance(samples, list) self.assertEquals(k, len(samples)) for sample in samples: self.assertIn(sample, names) else: with pytest.raises(UnsupportedOperation): _ = fs.sample_names(1) # iter and sample files self.assertListEqual([(n, to_bytes(n) + b' content') for n in names], sorted(fs.iter_files())) if capacity.can_random_sample(): for repeated in range(10): for k in range(len(names)): samples = fs.sample_files(k) self.assertIsInstance(samples, list) self.assertEquals(k, len(samples)) for sample in samples: self.assertIn(sample[0], names) self.assertEquals(get_content(sample[0]), sample[1]) else: with pytest.raises(UnsupportedOperation): _ = fs.sample_names(1) # retrieve, get data and open for n in names: self.assertEquals(get_content(n), fs.retrieve(n)) self.assertEquals(get_content(n), fs.get_data(n)) with maybe_close(fs.open(n, 'r')) as f: self.assertEquals(get_content(n), f.read()) with pytest.raises(DataFileNotExist): _ = fs.retrieve(n + '.invalid') with pytest.raises(DataFileNotExist): with maybe_close(fs.open(n + '.invalid', 'r')): pass # isfile, batch_isfile for n in names: self.assertTrue(fs.isfile(n)) self.assertFalse(fs.isfile(n + '.invalid-ext')) n_dir = n.rsplit('/', 1)[0] if n_dir != n: self.assertFalse(fs.isfile(n_dir)) self.assertListEqual([True] * len(names), fs.batch_isfile(names)) self.assertListEqual([True, False] * len(names), fs.batch_isfile( sum([[n, n + '.invalid-ext'] for n in names], [])))