def upload_feature(feature_name: str, df: pd.DataFrame, fs: AbstractFileSystem) -> None: containers = fs.ls(".") # Issue with exist_ok flag: see https://github.com/dask/adlfs/issues/130 if not any(c.startswith(settings.features_container_name.strip("/")) for c in containers): fs.mkdir(settings.features_container_name) with fs.open(settings.feature_location(feature_name), mode="wb") as f: df.to_parquet(f)
def prepare_fs(cls, fs: fsspec.AbstractFileSystem, root: str): if fs.isdir(root): # print(f'{fs.protocol}: deleting {root}') fs.delete(root, recursive=True) # print(f'{fs.protocol}: making root {root}') fs.mkdirs(root) # Write a text file into each subdirectory, so # we also test that store.get_data_ids() scans # recursively. dir_path = root for subdir_name in DATA_PATH.split('/'): dir_path += '/' + subdir_name # print(f'{fs.protocol}: making {dir_path}') fs.mkdir(dir_path) file_path = dir_path + '/README.md' # print(f'{fs.protocol}: writing {file_path}') with fs.open(file_path, 'w') as fp: fp.write('\n')