def get_fs(protocol, storage_options=None): """Create a filesystem object from a protocol and options. Parameters ---------- protocol : str The specific protocol to use. storage_options : dict, optional Keywords to pass to the filesystem class. """ if protocol in _filesystems: cls = _filesystems[protocol] elif protocol == 's3': import_required( 's3fs', "Need to install `s3fs` library for s3 support\n" " conda install s3fs -c conda-forge\n" " or\n" " pip install s3fs") import dask.bytes.s3 # noqa, register the s3 backend cls = _filesystems[protocol] elif protocol in ('gs', 'gcs'): import_required( 'gcsfs', "Need to install `gcsfs` library for Google Cloud Storage support\n" " conda install gcsfs -c conda-forge\n" " or\n" " pip install gcsfs") cls = _filesystems[protocol] elif protocol == 'hdfs': cls = get_hdfs_driver(config.get("hdfs_driver", "auto")) elif protocol in ['http', 'https']: import_required( 'requests', "Need to install `requests` for HTTP support\n" " conda install requests\n" " or\n" " pip install requests") import dask.bytes.http # noqa, registers HTTP backend cls = _filesystems[protocol] else: raise ValueError("Unknown protocol %s" % protocol) if storage_options is None: storage_options = {} fs = cls(**storage_options) fs_token = tokenize(cls, protocol, storage_options) return fs, fs_token
def get_fs(protocol, storage_options=None): """Create a filesystem object from a protocol and options. Parameters ---------- protocol : str The specific protocol to use. storage_options : dict, optional Keywords to pass to the filesystem class. """ if protocol in _filesystems: cls = _filesystems[protocol] elif protocol == 's3': import_required('s3fs', "Need to install `s3fs` library for s3 support\n" " conda install s3fs -c conda-forge\n" " or\n" " pip install s3fs") import dask.bytes.s3 # noqa, register the s3 backend cls = _filesystems[protocol] elif protocol in ('gs', 'gcs'): import_required('gcsfs', "Need to install `gcsfs` library for Google Cloud Storage support\n" " conda install gcsfs -c conda-forge\n" " or\n" " pip install gcsfs") cls = _filesystems[protocol] elif protocol == 'hdfs': cls = get_hdfs_driver(config.get("hdfs_driver", "auto")) elif protocol in ['http', 'https']: import_required('requests', "Need to install `requests` for HTTP support\n" " conda install requests\n" " or\n" " pip install requests") import dask.bytes.http # noqa, registers HTTP backend cls = _filesystems[protocol] else: raise ValueError("Unknown protocol %s" % protocol) if storage_options is None: storage_options = {} fs = cls(**storage_options) fs_token = tokenize(cls, protocol, storage_options) return fs, fs_token
def get_fs(protocol, storage_options=None): """Create a filesystem object from a protocol and options. Parameters ---------- protocol : str The specific protocol to use. storage_options : dict, optional Keywords to pass to the filesystem class. """ if protocol in _filesystems: cls = _filesystems[protocol] elif protocol == "s3": import_required( "s3fs", "Need to install `s3fs` library for s3 support\n" " conda install s3fs -c conda-forge\n" " or\n" " pip install s3fs", ) import dask.bytes.s3 # noqa, register the s3 backend cls = _filesystems[protocol] elif protocol in ("gs", "gcs"): import_required( "gcsfs", "Need to install `gcsfs` library for Google Cloud Storage support\n" " conda install gcsfs -c conda-forge\n" " or\n" " pip install gcsfs", ) cls = _filesystems[protocol] elif protocol in ["adl", "adlfs"]: import_required( "dask_adlfs", "Need to install `dask_adlfs` for Azure Datalake " "Storage support.\n" "First install azure-storage via pip or conda:\n" " conda install -c conda-forge azure-storage\n" " or\n" " pip install azure-storage\n" "and then install `dask_adlfs` via pip:\n" " pip install dask-adlfs", ) cls = _filesystems[protocol] elif protocol == "hdfs": cls = get_hdfs_driver(config.get("hdfs_driver", "auto")) elif protocol in ["http", "https"]: import_required( "requests", "Need to install `requests` for HTTP support\n" " conda install requests\n" " or\n" " pip install requests", ) import dask.bytes.http # noqa, registers HTTP backend cls = _filesystems[protocol] else: raise ValueError("Unknown protocol %s" % protocol) if storage_options is None: storage_options = {} fs = cls(**storage_options) fs_token = tokenize(cls, protocol, storage_options) return fs, fs_token