def upload_feature(feature_name: str, df: pd.DataFrame, fs: AbstractFileSystem) -> None:
containers = fs.ls(".")
# Issue with exist_ok flag: see https://github.com/dask/adlfs/issues/130
if not any(c.startswith(settings.features_container_name.strip("/")) for c in containers):
fs.mkdir(settings.features_container_name)
with fs.open(settings.feature_location(feature_name), mode="wb") as f:
df.to_parquet(f)
def prepare_fs(cls, fs: fsspec.AbstractFileSystem, root: str):
if fs.isdir(root):
# print(f'{fs.protocol}: deleting {root}')
fs.delete(root, recursive=True)
# print(f'{fs.protocol}: making root {root}')
fs.mkdirs(root)
# Write a text file into each subdirectory, so
# we also test that store.get_data_ids() scans
# recursively.
dir_path = root
for subdir_name in DATA_PATH.split('/'):
dir_path += '/' + subdir_name
# print(f'{fs.protocol}: making {dir_path}')
fs.mkdir(dir_path)
file_path = dir_path + '/README.md'
# print(f'{fs.protocol}: writing {file_path}')
with fs.open(file_path, 'w') as fp:
fp.write('\n')