def known_to_fsspec(path: str) -> bool:
import_fsspec('known_to_fsspec')
from fsspec.registry import known_implementations
for protocol in known_implementations.keys():
if path.startswith(protocol + '://') or path.startswith(protocol +
'::'):
return True
return False
def _is_url(maybe_url):
"""
Determine whether string is a URL or not using marshmallow and the URL
schemes available through fsspec.
"""
schemes = (set(["http"])
| known_implementations.keys()
| set(list(fsspec.registry)))
try:
ma.validate.URL(schemes=schemes, require_tld=False)(maybe_url)
return True
except ma.exceptions.ValidationError:
return False
def init_validate(self):
# ------------------------------------------------------------- 01
# call super
super().init_validate()
# ------------------------------------------------------------- 02
# validate protocol
if self.protocol in known_implementations.keys():
_err_msg = "This protocol is known by gcsfs but not yet " \
"implemented in `toolcraft`. Please raise PR for support ..."
else:
_err_msg = "this protocol is not known by gcsfs and cannot be " \
"supported by `toolcraft`"
e.validation.ShouldBeOneOf(
value=self.protocol, values=['gs', 'gcs', 'file'],
msgs=[_err_msg]
).raise_if_failed()
# ------------------------------------------------------------- 03
# protocol specific validations
# ------------------------------------------------------------- 03.01
# if gcs related protocol validate kwargs
if self.protocol in ['gs', 'gcs']:
# kwargs must be supplied
if self.kwargs is None:
raise e.validation.NotAllowed(
msgs=[f"Please supply mandatory `kwargs` dict in config "
f"file {settings.TC_CONFIG_FILE} for file_system {self.fs_name}"]
)
# get credentials
if 'credentials' not in self.kwargs.keys():
raise e.validation.NotAllowed(
msgs=[f"Please supply mandatory kwarg `credentials` in config "
f"file {settings.TC_CONFIG_FILE} for file_system {self.fs_name}"]
)
_credentials_file = settings.TC_HOME / self.kwargs['credentials']
if not _credentials_file.exists():
raise e.validation.NotAllowed(
msgs=[
f"Cannot find credential file {_credentials_file}",
f"Please create it using information from here:",
"https://cloud.google.com/docs/authentication/"
"getting-started#create-service-account-console",
]
)
# get project and bucket
if 'project' not in self.kwargs.keys():
raise e.validation.NotAllowed(
msgs=[f"Please supply mandatory kwarg `project` in config "
f"file {settings.TC_CONFIG_FILE} for file_system {self.fs_name}"]
)
if 'bucket' not in self.kwargs.keys():
raise e.validation.NotAllowed(
msgs=[f"Please supply mandatory kwarg `bucket` in config "
f"file {settings.TC_CONFIG_FILE} for file_system {self.fs_name}"]
)
# ------------------------------------------------------------- 04
# call property fs as it also validates creation
_ = self.fs
# ------------------------------------------------------------- 05
# if root_dir in fs_config end with sep then raise error as we will be adding it
if self.root_dir.endswith(self.fs.sep):
raise e.code.NotAllowed(
msgs=[
f"Please do not supply seperator `{self.fs.sep}` at end for "
f"root_dir in config files. As we will take care of that ..."
]
)
# if CWD make sure that root_dir always start with .
if self.fs_name == "CWD":
if not self.root_dir.startswith("."):
raise e.validation.NotAllowed(
msgs=[
f"For CWD file_system always make sure that root_dir "
f"starts with '.', found {self.root_dir}"
]
)
def get(self):
# dbm = access.DBMan(LOST_CONFIG)
# identity = get_jwt_identity()
# user = dbm.get_user_by_id(identity)
return list(known_implementations.keys())