def _get_mode(self, mode: str, fs: AbstractFileSystem):
if mode:
if mode not in ["r", "r+", "a", "a+", "w", "w+"]:
raise Exception(f"Invalid mode {mode}")
return mode
else:
try:
meta_path = posixpath.join(self._path, "meta.json")
if not fs.exists(self._path) or not fs.exists(meta_path):
return "a"
bytes_ = bytes("Hello", "utf-8")
path = posixpath.join(self._path, "mode_test")
fs.pipe(path, bytes_)
fs.rm(path)
except:
return "r"
return "a"
def read_stream_collections(
client,
fs: AbstractFileSystem,
queue: "ConcurrentQueue[Tuple[ByteStream, str]]",
base_prefix: str,
prefix: str,
):
metadata_path = os.path.join(prefix, 'metadata.json')
blob_path = os.path.join(prefix, 'blob')
if fs.exists(metadata_path):
metadata = read_metadata(fs, metadata_path)
streams = []
for path in fs.listdir(prefix):
if path['type'] == 'directory':
streams.append(
read_stream_collections(
client, fs, queue, base_prefix, path['name']
)
)
stream_collection = StreamCollection.new(client, metadata, streams)
return stream_collection.id
else:
# make a blob
with fs.open(blob_path, 'rb') as f:
try:
total_size = f.size()
except TypeError:
total_size = f.size
# create a stream
stream = ByteStream.new(
client,
params={
StreamCollection.KEY_OF_PATH: os.path.relpath(
blob_path, base_prefix
),
'length': total_size,
},
)
queue.put((stream, blob_path))
return stream.id
