def upload_feature(feature_name: str, df: pd.DataFrame, fs: AbstractFileSystem) -> None:
containers = fs.ls(".")
# Issue with exist_ok flag: see https://github.com/dask/adlfs/issues/130
if not any(c.startswith(settings.features_container_name.strip("/")) for c in containers):
fs.mkdir(settings.features_container_name)
with fs.open(settings.feature_location(feature_name), mode="wb") as f:
df.to_parquet(f)
async def _list(fs: fsspec.AbstractFileSystem, path):
try:
return await fs._ls(path)
except AttributeError:
return fs.ls(path)