def get_fs(protocol, storage_options=None):
"""Create a filesystem object from a protocol and options.
Parameters
----------
protocol : str
The specific protocol to use.
storage_options : dict, optional
Keywords to pass to the filesystem class.
"""
if protocol in _filesystems:
cls = _filesystems[protocol]
elif protocol == 's3':
import_required(
's3fs', "Need to install `s3fs` library for s3 support\n"
" conda install s3fs -c conda-forge\n"
" or\n"
" pip install s3fs")
import dask.bytes.s3 # noqa, register the s3 backend
cls = _filesystems[protocol]
elif protocol in ('gs', 'gcs'):
import_required(
'gcsfs',
"Need to install `gcsfs` library for Google Cloud Storage support\n"
" conda install gcsfs -c conda-forge\n"
" or\n"
" pip install gcsfs")
cls = _filesystems[protocol]
elif protocol == 'hdfs':
cls = get_hdfs_driver(config.get("hdfs_driver", "auto"))
elif protocol in ['http', 'https']:
import_required(
'requests', "Need to install `requests` for HTTP support\n"
" conda install requests\n"
" or\n"
" pip install requests")
import dask.bytes.http # noqa, registers HTTP backend
cls = _filesystems[protocol]
else:
raise ValueError("Unknown protocol %s" % protocol)
if storage_options is None:
storage_options = {}
fs = cls(**storage_options)
fs_token = tokenize(cls, protocol, storage_options)
return fs, fs_token
def get_fs(protocol, storage_options=None):
"""Create a filesystem object from a protocol and options.
Parameters
----------
protocol : str
The specific protocol to use.
storage_options : dict, optional
Keywords to pass to the filesystem class.
"""
if protocol in _filesystems:
cls = _filesystems[protocol]
elif protocol == 's3':
import_required('s3fs',
"Need to install `s3fs` library for s3 support\n"
" conda install s3fs -c conda-forge\n"
" or\n"
" pip install s3fs")
import dask.bytes.s3 # noqa, register the s3 backend
cls = _filesystems[protocol]
elif protocol in ('gs', 'gcs'):
import_required('gcsfs',
"Need to install `gcsfs` library for Google Cloud Storage support\n"
" conda install gcsfs -c conda-forge\n"
" or\n"
" pip install gcsfs")
cls = _filesystems[protocol]
elif protocol == 'hdfs':
cls = get_hdfs_driver(config.get("hdfs_driver", "auto"))
elif protocol in ['http', 'https']:
import_required('requests',
"Need to install `requests` for HTTP support\n"
" conda install requests\n"
" or\n"
" pip install requests")
import dask.bytes.http # noqa, registers HTTP backend
cls = _filesystems[protocol]
else:
raise ValueError("Unknown protocol %s" % protocol)
if storage_options is None:
storage_options = {}
fs = cls(**storage_options)
fs_token = tokenize(cls, protocol, storage_options)
return fs, fs_token
def get_fs(protocol, storage_options=None):
"""Create a filesystem object from a protocol and options.
Parameters
----------
protocol : str
The specific protocol to use.
storage_options : dict, optional
Keywords to pass to the filesystem class.
"""
if protocol in _filesystems:
cls = _filesystems[protocol]
elif protocol == "s3":
import_required(
"s3fs",
"Need to install `s3fs` library for s3 support\n"
" conda install s3fs -c conda-forge\n"
" or\n"
" pip install s3fs",
)
import dask.bytes.s3 # noqa, register the s3 backend
cls = _filesystems[protocol]
elif protocol in ("gs", "gcs"):
import_required(
"gcsfs",
"Need to install `gcsfs` library for Google Cloud Storage support\n"
" conda install gcsfs -c conda-forge\n"
" or\n"
" pip install gcsfs",
)
cls = _filesystems[protocol]
elif protocol in ["adl", "adlfs"]:
import_required(
"dask_adlfs",
"Need to install `dask_adlfs` for Azure Datalake "
"Storage support.\n"
"First install azure-storage via pip or conda:\n"
" conda install -c conda-forge azure-storage\n"
" or\n"
" pip install azure-storage\n"
"and then install `dask_adlfs` via pip:\n"
" pip install dask-adlfs",
)
cls = _filesystems[protocol]
elif protocol == "hdfs":
cls = get_hdfs_driver(config.get("hdfs_driver", "auto"))
elif protocol in ["http", "https"]:
import_required(
"requests",
"Need to install `requests` for HTTP support\n"
" conda install requests\n"
" or\n"
" pip install requests",
)
import dask.bytes.http # noqa, registers HTTP backend
cls = _filesystems[protocol]
else:
raise ValueError("Unknown protocol %s" % protocol)
if storage_options is None:
storage_options = {}
fs = cls(**storage_options)
fs_token = tokenize(cls, protocol, storage_options)
return fs, fs_token
