def read_byte_stream(
client,
fs: AbstractFileSystem,
stream: ByteStream,
path: str,
chunk_size: int = CHUNK_SIZE,
):
logger.info('start reading blob at %s', path)
with fs.open(path, mode="rb") as f:
try:
total_size = f.size()
except TypeError:
total_size = f.size
writer = stream.open_writer(client)
try:
begin, end = 0, total_size
while begin < end:
buffer = read_block(f, begin, min(chunk_size, end - begin))
chunk = writer.next(len(buffer))
vineyard.memory_copy(chunk, 0, buffer)
begin += len(buffer)
except Exception:
report_exception()
writer.fail()
sys.exit(-1)
writer.finish()
return total_size
def read_stream_collections(
client,
fs: AbstractFileSystem,
queue: "ConcurrentQueue[Tuple[ByteStream, str]]",
base_prefix: str,
prefix: str,
):
metadata_path = os.path.join(prefix, 'metadata.json')
blob_path = os.path.join(prefix, 'blob')
if fs.exists(metadata_path):
metadata = read_metadata(fs, metadata_path)
streams = []
for path in fs.listdir(prefix):
if path['type'] == 'directory':
streams.append(
read_stream_collections(
client, fs, queue, base_prefix, path['name']
)
)
stream_collection = StreamCollection.new(client, metadata, streams)
return stream_collection.id
else:
# make a blob
with fs.open(blob_path, 'rb') as f:
try:
total_size = f.size()
except TypeError:
total_size = f.size
# create a stream
stream = ByteStream.new(
client,
params={
StreamCollection.KEY_OF_PATH: os.path.relpath(
blob_path, base_prefix
),
'length': total_size,
},
)
queue.put((stream, blob_path))
return stream.id
def _create_fs_file_instances(filenames, msg):
"""Create FSFile instances when filesystem is provided."""
filesystems = list(gen_dict_extract(msg.data, 'filesystem'))
if filesystems:
from satpy.readers import FSFile
from fsspec.spec import AbstractFileSystem
import json
filenames = [
FSFile(filename,
AbstractFileSystem.from_json(json.dumps(filesystem)))
for filename, filesystem in zip(filenames, filesystems)
]
return filenames
def _get_mode(self, mode: str, fs: AbstractFileSystem):
if mode:
if mode not in ["r", "r+", "a", "a+", "w", "w+"]:
raise Exception(f"Invalid mode {mode}")
return mode
else:
try:
meta_path = posixpath.join(self._path, "meta.json")
if not fs.exists(self._path) or not fs.exists(meta_path):
return "a"
bytes_ = bytes("Hello", "utf-8")
path = posixpath.join(self._path, "mode_test")
fs.pipe(path, bytes_)
fs.rm(path)
except:
return "r"
return "a"
def test_add_docs_warns():
with pytest.warns(FutureWarning, match="add_docs"):
AbstractFileSystem(add_docs=True)
def read_metadata(fs: AbstractFileSystem, path: str) -> Dict:
logger.info('start reading metadata at %s', path)
with fs.open(path, mode="rb") as f:
return json.loads(f.read().decode('utf-8', errors='ignore'))
