def read_byte_stream( client, fs: AbstractFileSystem, stream: ByteStream, path: str, chunk_size: int = CHUNK_SIZE, ): logger.info('start reading blob at %s', path) with fs.open(path, mode="rb") as f: try: total_size = f.size() except TypeError: total_size = f.size writer = stream.open_writer(client) try: begin, end = 0, total_size while begin < end: buffer = read_block(f, begin, min(chunk_size, end - begin)) chunk = writer.next(len(buffer)) vineyard.memory_copy(chunk, 0, buffer) begin += len(buffer) except Exception: report_exception() writer.fail() sys.exit(-1) writer.finish() return total_size
def read_stream_collections( client, fs: AbstractFileSystem, queue: "ConcurrentQueue[Tuple[ByteStream, str]]", base_prefix: str, prefix: str, ): metadata_path = os.path.join(prefix, 'metadata.json') blob_path = os.path.join(prefix, 'blob') if fs.exists(metadata_path): metadata = read_metadata(fs, metadata_path) streams = [] for path in fs.listdir(prefix): if path['type'] == 'directory': streams.append( read_stream_collections( client, fs, queue, base_prefix, path['name'] ) ) stream_collection = StreamCollection.new(client, metadata, streams) return stream_collection.id else: # make a blob with fs.open(blob_path, 'rb') as f: try: total_size = f.size() except TypeError: total_size = f.size # create a stream stream = ByteStream.new( client, params={ StreamCollection.KEY_OF_PATH: os.path.relpath( blob_path, base_prefix ), 'length': total_size, }, ) queue.put((stream, blob_path)) return stream.id
def _create_fs_file_instances(filenames, msg): """Create FSFile instances when filesystem is provided.""" filesystems = list(gen_dict_extract(msg.data, 'filesystem')) if filesystems: from satpy.readers import FSFile from fsspec.spec import AbstractFileSystem import json filenames = [ FSFile(filename, AbstractFileSystem.from_json(json.dumps(filesystem))) for filename, filesystem in zip(filenames, filesystems) ] return filenames
def _get_mode(self, mode: str, fs: AbstractFileSystem): if mode: if mode not in ["r", "r+", "a", "a+", "w", "w+"]: raise Exception(f"Invalid mode {mode}") return mode else: try: meta_path = posixpath.join(self._path, "meta.json") if not fs.exists(self._path) or not fs.exists(meta_path): return "a" bytes_ = bytes("Hello", "utf-8") path = posixpath.join(self._path, "mode_test") fs.pipe(path, bytes_) fs.rm(path) except: return "r" return "a"
def test_add_docs_warns(): with pytest.warns(FutureWarning, match="add_docs"): AbstractFileSystem(add_docs=True)
def read_metadata(fs: AbstractFileSystem, path: str) -> Dict: logger.info('start reading metadata at %s', path) with fs.open(path, mode="rb") as f: return json.loads(f.read().decode('utf-8', errors='ignore'))