def sanity_check(self):
""" sanity check a config
check that we have a cloud and storagePath
if aws, check can read credentials
if google, check ProjectName
Returns:
(T,) if good
(F, issues) if bad
"""
key = 'Cloud'
core = self._config[Config.SECTION_CORE]
if key.lower() not in [k.lower()
for k in core.keys()] or len(core[key]) < 1:
raise ConfigError(
'Please set {} in section {} in config file'.format(
key, Config.SECTION_CORE))
# now that a cloud is chosen, can check StoragePath
storage_path = self._cloud.storage_path
if storage_path is None or len(storage_path) == 0:
raise ConfigError(
'Please set StoragePath = bucket/{optional path} '
'in config file in a cloud specific section')
self._cloud.sanity_check()
def sanity_check(self):
""" sanity check a config
check that we have a cloud and storagePath
if aws, check can read credentials
if google, check ProjectName
Returns:
(T,) if good
(F, issues) if bad
"""
key = 'Cloud'
if key.lower() not in self._config['Global'].keys() or len(
self._config['Global'][key]) < 1:
raise ConfigError(
'Please set %s in section Global in config file' % key)
# now that a cloud is chosen, can check StoragePath
storage_path = self._cloud.storage_path
if storage_path is None or len(storage_path) == 0:
raise ConfigError(
'Please set StoragePath = bucket/{optional path} '
'in config file in a cloud specific section')
self._cloud.sanity_check()
def sanity_check(self):
""" sanity check a config
check that we have a cloud and storagePath
if aws, check can read credentials
if google, check ProjectName
Returns:
(T,) if good
(F, issues) if bad
"""
for key in ['Cloud']:
if key.lower() not in self._config['Global'].keys() or len(
self._config['Global'][key]) < 1:
raise ConfigError(
'Please set %s in section Global in config file %s' %
(key, self.file))
# now that a cloud is chosen, can check StoragePath
sp = self._cloud.storage_path
if sp is None or len(sp) == 0:
raise ConfigError(
'Please set StoragePath = bucket/{optional path} in conf file "%s" '
'either in Global or a cloud specific section' % self.CONFIG)
self._cloud.sanity_check()
def __init__(self, settings):
assert isinstance(settings, dvc.settings.Settings)
#To handle ConfigI case
if not hasattr(settings.config, '_config'):
self._settings = settings
self._cloud = DataCloudBase(None)
return
self._settings = settings
self._config = self._settings.config._config
cloud_type = self._config['Global'].get('Cloud', '').strip().upper()
if cloud_type not in self.CLOUD_MAP.keys():
raise ConfigError('Wrong cloud type %s specified' % cloud_type)
if cloud_type not in self._config.keys():
raise ConfigError('Can\'t find cloud section \'[%s]\' in config' %
cloud_type)
cloud_settings = self.get_cloud_settings(self._config, cloud_type,
self._settings.path_factory)
self.typ = cloud_type
self._cloud = self.CLOUD_MAP[cloud_type](cloud_settings)
self.sanity_check()
def _check(self, conf, section, opt=None):
if section not in conf:
msg = "section {} doesn't exist"
raise ConfigError(msg.format(self.args.name))
if opt and opt not in conf[section]:
msg = "option {} doesn't exist"
raise ConfigError(msg.format(self.args.name))
def _check(self, conf, remote, section, opt=None):
name = "remote" if remote else "section"
if section not in conf:
raise ConfigError(f"{name} '{section}' doesn't exist")
if opt and opt not in conf[section]:
raise ConfigError(
f"option '{opt}' doesn't exist in {name} '{section}'")
def _prepare_credentials(self, **config):
import aiohttp
from fsspec.asyn import fsspec_loop
from dvc.config import ConfigError
credentials = {}
client_kwargs = credentials.setdefault("client_kwargs", {})
if config.get("auth"):
user = config.get("user")
password = config.get("password")
custom_auth_header = config.get("custom_auth_header")
if password is None and config.get("ask_password"):
password = ask_password(config.get("url"), user or "custom")
auth_method = config["auth"]
if auth_method == "basic":
if user is None or password is None:
raise ConfigError(
"HTTP 'basic' authentication require both "
"'user' and 'password'")
client_kwargs["auth"] = aiohttp.BasicAuth(user, password)
elif auth_method == "custom":
if custom_auth_header is None or password is None:
raise ConfigError(
"HTTP 'custom' authentication require both "
"'custom_auth_header' and 'password'")
credentials["headers"] = {custom_auth_header: password}
else:
raise NotImplementedError(
f"Auth method {auth_method!r} is not supported.")
# Force cleanup of closed SSL transports.
# https://github.com/iterative/dvc/issues/7414
connector_kwargs = {"enable_cleanup_closed": True}
if "ssl_verify" in config:
connector_kwargs.update(ssl=make_context(config["ssl_verify"]))
with fsspec_loop():
client_kwargs["connector"] = aiohttp.TCPConnector(
**connector_kwargs)
# The connector should not be owned by aiohttp.ClientSession since
# it is closed by fsspec (HTTPFileSystem.close_session)
client_kwargs["connector_owner"] = False
# Allow reading proxy configurations from the environment.
client_kwargs["trust_env"] = True
credentials["get_client"] = self.get_client
self.upload_method = config.get("method", "POST")
return credentials
def _init_remote(self, remote):
section = Config.SECTION_REMOTE_FMT.format(remote)
cloud_config = self._config.get(section, None)
if not cloud_config:
msg = "Can't find remote section '{}' in config"
raise ConfigError(msg.format(section))
cloud_type = self.supported(cloud_config)
if not cloud_type:
raise ConfigError("Unsupported cloud '{}'".format(cloud_config))
return self._init_cloud(cloud_config, cloud_type)
def _init_remote(self, remote):
section = Config.SECTION_REMOTE_FMT.format(remote)
cloud_config = self._config.get(section, None)
if not cloud_config:
raise ConfigError(
"Can't find remote section '{}' in config".format(section))
url = cloud_config[Config.SECTION_REMOTE_URL]
cloud_type = self.supported(url)
if not cloud_type:
raise ConfigError("Unsupported url '{}'".format(url))
return self._init_cloud(cloud_config, cloud_type)
def _prepare_credentials(self, **config):
import aiohttp
from fsspec.asyn import fsspec_loop
from dvc.config import ConfigError
credentials = {}
client_kwargs = credentials.setdefault("client_kwargs", {})
if config.get("auth"):
user = config.get("user")
password = config.get("password")
custom_auth_header = config.get("custom_auth_header")
if password is None and config.get("ask_password"):
password = ask_password(config.get("url"), user or "custom")
auth_method = config["auth"]
if auth_method == "basic":
if user is None or password is None:
raise ConfigError(
"HTTP 'basic' authentication require both "
"'user' and 'password'"
)
client_kwargs["auth"] = aiohttp.BasicAuth(user, password)
elif auth_method == "custom":
if custom_auth_header is None or password is None:
raise ConfigError(
"HTTP 'custom' authentication require both "
"'custom_auth_header' and 'password'"
)
credentials["headers"] = {custom_auth_header: password}
else:
raise NotImplementedError(
f"Auth method {auth_method!r} is not supported."
)
if "ssl_verify" in config:
with fsspec_loop():
client_kwargs["connector"] = aiohttp.TCPConnector(
ssl=make_context(config["ssl_verify"])
)
# Allow reading proxy configurations from the environment.
client_kwargs["trust_env"] = True
credentials["get_client"] = self.get_client
self.upload_method = config.get("method", "POST")
return credentials
def run(self):
from dvc.config import ConfigError
if self.args.name is None and not self.args.unset:
conf = self.config.read(self.args.level)
try:
ui.write(conf["core"]["remote"])
except KeyError:
ui.write("No default remote set")
return 1
else:
with self.config.edit(self.args.level) as conf:
if self.args.unset:
conf["core"].pop("remote", None)
else:
merged_conf = self.config.load_config_to_level(
self.args.level
)
if (
self.args.name in conf["remote"]
or self.args.name in merged_conf["remote"]
):
conf["core"]["remote"] = self.args.name
else:
raise ConfigError(
"default remote must be present in remote list."
)
return 0
def get_cloud_fs(repo, **kwargs):
from dvc.config import ConfigError
from dvc.config_schema import SCHEMA, Invalid
repo_config = repo.config if repo else {}
core_config = repo_config.get("core", {})
remote_conf = get_fs_config(repo, repo_config, **kwargs)
try:
remote_conf = SCHEMA["remote"][str](remote_conf)
except Invalid as exc:
raise ConfigError(str(exc)) from None
if "jobs" not in remote_conf:
jobs = core_config.get("jobs")
if jobs:
remote_conf["jobs"] = jobs
if "checksum_jobs" not in remote_conf:
checksum_jobs = core_config.get("checksum_jobs")
if checksum_jobs:
remote_conf["checksum_jobs"] = checksum_jobs
cls = get_fs_cls(remote_conf)
if cls == GDriveFileSystem and repo:
remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir
url = remote_conf.pop("url")
fs_path = cls._strip_protocol(url) # pylint:disable=protected-access
extras = cls._get_kwargs_from_urls(url) # pylint:disable=protected-access
conf = {**extras, **remote_conf} # remote config takes priority
return cls, conf, fs_path
def get_cloud_fs(repo, **kwargs):
from dvc.config import ConfigError
from dvc.config_schema import SCHEMA, Invalid
repo_config = repo.config if repo else {}
core_config = repo_config.get("core", {})
remote_conf = get_fs_config(repo_config, **kwargs)
try:
remote_conf = SCHEMA["remote"][str](remote_conf)
except Invalid as exc:
raise ConfigError(str(exc)) from None
if "jobs" not in remote_conf:
jobs = core_config.get("jobs")
if jobs:
remote_conf["jobs"] = jobs
if "checksum_jobs" not in remote_conf:
checksum_jobs = core_config.get("checksum_jobs")
if checksum_jobs:
remote_conf["checksum_jobs"] = checksum_jobs
cls = get_fs_cls(remote_conf)
if cls == GDriveFileSystem and repo:
remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir
return cls, remote_conf
def run(self):
from dvc.config import ConfigError
all_config = self.config.load_config_to_level(None)
if self.args.new in all_config.get("remote", {}):
raise ConfigError(
"Rename failed. Remote name '{}' already exists.".format(
{self.args.new}
)
)
with self.config.edit(self.args.level) as conf:
self._check_exists(conf)
conf["remote"][self.args.new] = conf["remote"][self.args.name]
del conf["remote"][self.args.name]
self._rename_default(conf)
up_to_level = self.args.level or "repo"
for level in reversed(self.config.LEVELS):
if level == up_to_level:
break
with self.config.edit(level) as level_conf:
self._rename_default(level_conf)
return 0
def _client(self):
from webdav3.client import Client
# Set password or ask for it
if self.ask_password and self.password is None and self.token is None:
self.password = ask_password(self.hostname, self.user)
# Setup webdav client options dictionary
options = {
"webdav_hostname": self.hostname,
"webdav_login": self.user,
"webdav_password": self.password,
"webdav_token": self.token,
"webdav_cert_path": self.cert_path,
"webdav_key_path": self.key_path,
"webdav_timeout": self.timeout,
"webdav_chunk_size": self.CHUNK_SIZE,
}
client = Client(options)
# Check whether client options are valid
if not client.valid():
raise ConfigError(
f"Configuration for WebDAV {self.hostname} is invalid.")
# Check whether connection is valid (root should always exist)
if not client.check(self.path_info.path):
raise WebDAVConnectionError(self.hostname)
return client
def _append_aws_grants_to_extra_args(self, config):
# Keys for extra_args can be one of the following list:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS
"""
ALLOWED_UPLOAD_ARGS = [
'ACL', 'CacheControl', 'ContentDisposition', 'ContentEncoding',
'ContentLanguage', 'ContentType', 'Expires', 'GrantFullControl',
'GrantRead', 'GrantReadACP', 'GrantWriteACP', 'Metadata',
'RequestPayer', 'ServerSideEncryption', 'StorageClass',
'SSECustomerAlgorithm', 'SSECustomerKey', 'SSECustomerKeyMD5',
'SSEKMSKeyId', 'WebsiteRedirectLocation'
]
"""
grants = {
"grant_full_control": "GrantFullControl",
"grant_read": "GrantRead",
"grant_read_acp": "GrantReadACP",
"grant_write_acp": "GrantWriteACP",
}
for grant_option, extra_args_key in grants.items():
if config.get(grant_option):
if self.acl:
raise ConfigError(
"`acl` and `grant_*` AWS S3 config options "
"are mutually exclusive"
)
self.extra_args[extra_args_key] = config.get(grant_option)
def __init__compat(self):
name = self._core.get(Config.SECTION_CORE_CLOUD, '').strip().lower()
if name == '':
self._cloud = None
return
cloud_type = self.CLOUD_MAP.get(name, None)
if not cloud_type:
msg = "Wrong cloud type {} specified"
raise ConfigError(msg.format(name))
cloud_config = self._config.get(name, None)
if not cloud_config:
msg = "Can't find cloud section '{}' in config"
raise ConfigError(msg.format(name))
return self._init_cloud(cloud_config, cloud_type)
def __init__(self, config):
self._config = config
cloud_type = self._config['Global'].get('Cloud', '').strip().upper()
if cloud_type not in self.CLOUD_MAP.keys():
raise ConfigError('Wrong cloud type %s specified' % cloud_type)
if cloud_type not in self._config.keys():
raise ConfigError('Can\'t find cloud section \'[%s]\' in config' %
cloud_type)
cloud_settings = self.get_cloud_settings(self._config, cloud_type)
self.typ = cloud_type
self._cloud = self.CLOUD_MAP[cloud_type](cloud_settings)
self.sanity_check()
def _init_remote(self, remote):
section = Config.SECTION_REMOTE_FMT.format(remote).lower()
cloud_config = self._config.get(section, None)
if not cloud_config:
msg = "can't find remote section '{}' in config"
raise ConfigError(msg.format(section))
return Remote(self.repo, cloud_config)
def __init__compat(self):
cloud_name = self._config[Config.SECTION_CORE].get(
Config.SECTION_CORE_CLOUD, '').strip().lower()
if cloud_name == '':
self._cloud = None
return
cloud_type = self.CLOUD_MAP.get(cloud_name, None)
if not cloud_type:
raise ConfigError('Wrong cloud type %s specified' % cloud_name)
cloud_config = self._config.get(cloud_name, None)
if not cloud_config:
raise ConfigError('Can\'t find cloud section \'[%s]\' in config' %
cloud_name)
return self._init_cloud(cloud_config, cloud_type)
def _get_cloud(self, remote):
if remote:
return self._init_remote(remote)
if self._cloud:
return self._cloud
raise ConfigError("No remote repository specified. Setup default repository " \
"with 'dvc config core.remote <name>' or use '-r <name>'.")
def get_cloud_tree(repo, **kwargs):
from dvc.config import SCHEMA, ConfigError, Invalid
remote_conf = _get_conf(repo, **kwargs)
try:
remote_conf = SCHEMA["remote"][str](remote_conf)
except Invalid as exc:
raise ConfigError(str(exc)) from None
return _get_tree(remote_conf)(repo, remote_conf)
def _prepare_credentials(self, **config):
from dvc.config import ConfigError
from dvc.utils.flatten import flatten, unflatten
login_info = defaultdict(dict)
# credentials
login_info["key"] = config.get("access_key_id")
login_info["secret"] = config.get("secret_access_key")
login_info["token"] = config.get("session_token")
# session configuration
login_info["profile"] = config.get("profile")
login_info["use_ssl"] = config.get("use_ssl", True)
# extra client configuration
client = login_info["client_kwargs"]
client["region_name"] = config.get("region")
client["endpoint_url"] = config.get("endpointurl")
client["verify"] = config.get("ssl_verify")
# encryptions
additional = login_info["s3_additional_kwargs"]
additional["ServerSideEncryption"] = config.get("sse")
additional["SSEKMSKeyId"] = config.get("sse_kms_key_id")
additional["ACL"] = config.get("acl")
for grant_option, grant_key in self._GRANTS.items():
if config.get(grant_option):
if additional["ACL"]:
raise ConfigError(
"`acl` and `grant_*` AWS S3 config options "
"are mutually exclusive")
additional[grant_key] = config[grant_option]
# config kwargs
session_config = login_info["config_kwargs"]
session_config["s3"] = self._load_aws_config_file(
login_info["profile"])
shared_creds = config.get("credentialpath")
if shared_creds:
os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
if (client["region_name"] is None
and session_config["s3"].get("region_name") is None
and os.getenv("AWS_REGION") is None):
# Enable bucket region caching
login_info["cache_regions"] = config.get("cache_regions", True)
config_path = config.get("configpath")
if config_path:
os.environ.setdefault("AWS_CONFIG_FILE", config_path)
return unflatten({
key: value
for key, value in flatten(login_info).items() if value is not None
})
def __init__(self, repo, config):
super().__init__(repo, config)
url = config.get(Config.SECTION_REMOTE_URL)
self.path_info = self.path_cls(url) if url else None
if not self.no_traverse:
raise ConfigError(
"HTTP doesn't support traversing the remote to list existing "
"files. Use: `dvc remote modify <name> no_traverse true`")
def get_cloud_tree(repo, **kwargs):
from dvc.config import ConfigError
from dvc.config_schema import SCHEMA, Invalid
remote_conf = get_tree_config(repo.config, **kwargs)
try:
remote_conf = SCHEMA["remote"][str](remote_conf)
except Invalid as exc:
raise ConfigError(str(exc)) from None
return get_tree_cls(remote_conf)(repo, remote_conf)
def get_settings(self, name):
"""
Args:
name (str): The name of the remote that we want to retrieve
Returns:
dict: The content beneath the given remote name.
Example:
>>> config = {'remote "server"': {'url': 'ssh://localhost/'}}
>>> get_settings("server")
{'url': 'ssh://localhost/'}
"""
settings = self.config.config.get(
Config.SECTION_REMOTE_FMT.format(name.lower())
)
if settings is None:
raise ConfigError(
"unable to find remote section '{}'".format(name)
)
parsed = urlparse(settings["url"])
# Support for cross referenced remotes.
# This will merge the settings, giving priority to the outer reference.
# For example, having:
#
# dvc remote add server ssh://localhost
# dvc remote modify server user root
# dvc remote modify server ask_password true
#
# dvc remote add images remote://server/tmp/pictures
# dvc remote modify images user alice
# dvc remote modify images ask_password false
# dvc remote modify images password asdf1234
#
# Results on a config dictionary like:
#
# {
# "url": "ssh://localhost/tmp/pictures",
# "user": "******",
# "password": "******",
# "ask_password": False,
# }
#
if parsed.scheme == "remote":
reference = self.get_settings(parsed.netloc)
url = posixpath.join(reference["url"], parsed.path.lstrip("/"))
merged = reference.copy()
merged.update(settings)
merged["url"] = url
return merged
return settings
def _check_before_rename(self):
from dvc.machine import validate_name
validate_name(self.args.new)
all_config = self.config.load_config_to_level(None)
if self.args.new in all_config.get("machine", {}):
raise ConfigError(
"Rename failed. Machine '{}' already exists.".format(
self.args.new))
ui.write(f"Rename machine '{self.args.name}' to '{self.args.new}'.")
def _get_cloud(self, remote, cmd):
if remote:
return self._init_remote(remote)
if self._cloud:
return self._cloud
raise ConfigError(
"No remote repository specified. Setup default repository with\n"
" dvc config core.remote <name>\n"
"or use:\n"
" dvc {} -r <name>\n".format(cmd))
def get_remote(self, remote=None, command="<command>"):
if not remote:
remote = self._core.get(Config.SECTION_CORE_REMOTE)
if remote:
return self._init_remote(remote)
raise ConfigError(
"No remote repository specified. Setup default repository with\n"
" dvc config core.remote <name>\n"
"or use:\n"
" dvc {} -r <name>\n".format(command))
def _init_compat(self):
name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower()
if name == "":
return None
cloud_type = self.CLOUD_MAP.get(name, None)
if not cloud_type:
msg = "wrong cloud type '{}' specified".format(name)
raise ConfigError(msg)
cloud_config = self._config.get(name, None)
if not cloud_config:
msg = "can't find cloud section '{}' in config".format(name)
raise ConfigError(msg)
# NOTE: check if the class itself has everything needed for operation.
# E.g. all the imported packages.
if not cloud_type.supported(cloud_type.compat_config(cloud_config)):
raise ConfigError("unsupported cloud '{}'".format(name))
return self._init_cloud(cloud_config, cloud_type)