def _is_enabled(cmd=None):
from dvc.config import Config
from dvc.project import Project
from dvc.exceptions import NotDvcProjectError
from dvc.command.daemon import CmdDaemonBase
if os.getenv("DVC_TEST"):
return False
if isinstance(cmd, CmdDaemonBase):
return False
if cmd is None or not hasattr(cmd, "config"):
try:
dvc_dir = Project.find_dvc_dir()
config = Config(dvc_dir)
assert config is not None
except NotDvcProjectError:
config = Config(validate=False)
assert config is not None
else:
config = cmd.config
assert config is not None
core = config.config.get(Config.SECTION_CORE, {})
enabled = core.get(Config.SECTION_CORE_ANALYTICS, True)
logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
return enabled
def _is_enabled(cmd=None):
from dvc.config import Config
from dvc.repo import Repo
from dvc.exceptions import NotDvcRepoError
from dvc.command.daemon import CmdDaemonBase
if os.getenv("DVC_TEST"):
return False
if isinstance(cmd, CmdDaemonBase):
return False
if cmd is None or not hasattr(cmd, "config"):
try:
dvc_dir = Repo.find_dvc_dir()
config = Config(dvc_dir)
assert config is not None
except NotDvcRepoError:
config = Config(validate=False)
assert config is not None
else:
config = cmd.config
assert config is not None
enabled = Analytics._is_enabled_config(config)
logger.debug(
"Analytics is {}.".format("enabled" if enabled else "disabled")
)
return enabled
def aws_credentials_specified_test(self, isfile_function):
""" in presence of [AWS] -> CredentialPath, use those credentials """
c = (
"[Global]",
"LogLevel =",
"DataDir = ",
"CacheDir = ",
"StateDir = ",
"Cloud = amazon",
"[AWS]",
"StoragePath = awssb/aws_storage_path",
"CredentialPath = some_credential_path",
)
s = StringIO.StringIO('\n'.join(c))
patcher = mock.patch(
'__builtin__.open',
side_effect=self.mocked_open_aws_default_credentials)
patcher.start()
conf = Config(s, conf_pseudo_file=s)
aws_creds = conf.get_aws_credentials()
patcher.stop()
self.assertEqual(aws_creds[0], 'override_access_id')
self.assertEqual(aws_creds[1], 'override_sekret')
def aws_credentials_default_test(self):
""" in absence of [AWS] -> CredentialPath, aws creds should be read from ~/.aws/credentials """
default_path = os.path.expanduser('~/.aws/credentials')
c = (
"[Global]",
"LogLevel =",
"DataDir = ",
"CacheDir = ",
"StateDir = ",
"Cloud = amazon",
"[AWS]",
"StoragePath = awssb/aws_storage_path",
"CredentialPath =",
)
s = StringIO.StringIO('\n'.join(c))
patcher = mock.patch(
'__builtin__.open',
side_effect=self.mocked_open_aws_default_credentials)
patcher.start()
conf = Config(s, conf_pseudo_file=s)
aws_creds = conf.get_aws_credentials()
patcher.stop()
self.assertEqual(aws_creds[0], 'default_access_id')
self.assertEqual(aws_creds[1], 'default_sekret')
def __init__(self, args):
from dvc.project import Project, NotDvcProjectError
self.args = args
try:
dvc_dir = os.path.join(Project.find_root(), Project.DVC_DIR)
saved_exc = None
except NotDvcProjectError as exc:
dvc_dir = None
saved_exc = exc
self.config = Config(dvc_dir, validate=False)
if self.args.system:
self.configobj = self.config._system_config
elif self.args.glob:
self.configobj = self.config._global_config
elif self.args.local:
if dvc_dir is None:
raise saved_exc
self.configobj = self.config._local_config
else:
if dvc_dir is None:
raise saved_exc
self.configobj = self.config._project_config
def test_s3_ssl_verify(tmp_dir, dvc):
config = Config(validate=False)
with config.edit() as conf:
conf["remote"]["remote-name"] = {"url": "s3://bucket/dvc"}
assert "ssl_verify" not in config["remote"]["remote-name"]
with config.edit() as conf:
section = conf["remote"]["remote-name"]
section["ssl_verify"] = False
assert (tmp_dir / ".dvc" / "config").read_text() == textwrap.dedent("""\
[core]
no_scm = True
['remote "remote-name"']
url = s3://bucket/dvc
ssl_verify = False
""")
with config.edit() as conf:
section = conf["remote"]["remote-name"]
section["ssl_verify"] = "/path/to/custom/cabundle.pem"
assert (tmp_dir / ".dvc" / "config").read_text() == textwrap.dedent("""\
[core]
no_scm = True
['remote "remote-name"']
url = s3://bucket/dvc
ssl_verify = /path/to/custom/cabundle.pem
""")
def storage_path_hierarchy_test(self):
""" StoragePath should be read first from global and then from cloud section """
c = ("[Global]", "LogLevel =", "DataDir = ", "CacheDir = ",
"StateDir = ", "Cloud = aws",
"StoragePath = globalsb/global_storage_path", "[AWS]",
"StoragePath = awssb/aws_storage_path", "CredentialPath =",
"[GCP]", "StoragePath = googlesb/google_storage_path")
s = StringIO('\n'.join(c))
conf = Config(s, conf_pseudo_file=s)
cloud = DataCloud(Settings(None, None, conf))
self.assertEqual(cloud.typ, 'AWS')
self.assertEqual(cloud._cloud.storage_bucket, 'globalsb')
self.assertEqual(cloud._cloud.storage_prefix, 'global_storage_path')
c = ("[Global]", "LogLevel =", "DataDir = ", "CacheDir = ",
"StateDir = ", "Cloud = Aws", "[AWS]",
"StoragePath = awssb/aws_storage_path", "CredentialPath =",
"[GCP]", "StoragePath = googlesb/google_storage_path")
s = StringIO('\n'.join(c))
conf = Config(s, conf_pseudo_file=s)
cloud = DataCloud(Settings(None, None, conf))
self.assertEqual(cloud.typ, 'AWS')
self.assertEqual(cloud._cloud.storage_bucket, 'awssb')
self.assertEqual(cloud._cloud.storage_prefix, 'aws_storage_path')
class CmdConfig(CmdBaseNoRepo):
def __init__(self, args):
super().__init__(args)
self.config = Config(validate=False)
def run(self):
section, opt = self.args.name.lower().strip().split(".", 1)
if self.args.unset:
self.config.unset(section, opt, level=self.args.level)
elif self.args.value is None:
logger.info(self.config.get(section, opt, level=self.args.level))
else:
self.config.set(section,
opt,
self.args.value,
level=self.args.level)
is_write = self.args.unset or self.args.value is not None
if is_write and self.args.name == "cache.type":
logger.warning(
"You have changed the 'cache.type' option. This doesn't update"
" any existing workspace file links, but it can be done with:"
"\n dvc checkout --relink")
return 0
class CmdConfig(CmdBaseNoRepo):
def __init__(self, args):
super().__init__(args)
self.config = Config(validate=False)
def run(self):
if self.args.list:
if any((self.args.name, self.args.value, self.args.unset)):
logger.error(
"-l/--list can't be used together with any of these "
"options: -u/--unset, name, value")
return 1
conf = self.config.load_one(self.args.level)
logger.info("\n".join(self._format_config(conf)))
return 0
if self.args.name is None:
logger.error("name argument is required")
return 1
section, opt = self.args.name.lower().strip().split(".", 1)
if self.args.value is None and not self.args.unset:
conf = self.config.load_one(self.args.level)
self._check(conf, section, opt)
logger.info(conf[section][opt])
return 0
with self.config.edit(self.args.level) as conf:
if self.args.unset:
self._check(conf, section, opt)
del conf[section][opt]
else:
self._check(conf, section)
conf[section][opt] = self.args.value
if self.args.name == "cache.type":
logger.warning(
"You have changed the 'cache.type' option. This doesn't update"
" any existing workspace file links, but it can be done with:"
"\n dvc checkout --relink")
return 0
def _check(self, conf, section, opt=None):
if section not in conf:
msg = "section {} doesn't exist"
raise ConfigError(msg.format(self.args.name))
if opt and opt not in conf[section]:
msg = "option {} doesn't exist"
raise ConfigError(msg.format(self.args.name))
@staticmethod
def _format_config(config):
for key, value in flatten(config).items():
yield f"{key}={value}"
def _get_current_config():
from dvc.config import Config
from dvc.repo import Repo
from dvc.exceptions import NotDvcRepoError
try:
dvc_dir = Repo.find_dvc_dir()
config = Config(dvc_dir)
except NotDvcRepoError:
config = Config(validate=False)
return config
def test_get_tree(tmp_dir, scm):
tmp_dir.scm_gen("foo", "foo", commit="add foo")
tree = scm.get_tree("master")
config = Config(tree=tree)
assert config.tree == tree
assert config.wtree != tree
assert isinstance(config.wtree, LocalTree)
assert config._get_tree("repo") == tree
assert config._get_tree("local") == config.wtree
assert config._get_tree("global") == config.wtree
assert config._get_tree("system") == config.wtree
def test_get_fs(tmp_dir, scm):
tmp_dir.scm_gen("foo", "foo", commit="add foo")
fs = scm.get_fs("master")
config = Config(fs=fs)
assert config.fs == fs
assert config.wfs != fs
assert isinstance(config.wfs, LocalFileSystem)
assert config._get_fs("repo") == fs
assert config._get_fs("local") == config.wfs
assert config._get_fs("global") == config.wfs
assert config._get_fs("system") == config.wfs
def run(self):
import os
from dvc.repo import Repo
from dvc.config import Config
from dvc.updater import Updater
root_dir = Repo.find_root()
dvc_dir = os.path.join(root_dir, Repo.DVC_DIR)
config = Config(dvc_dir, validate=False)
hardlink_lock = config.get("core", {}).get("hardlink_lock", False)
updater = Updater(dvc_dir, hardlink_lock=hardlink_lock)
updater.fetch(detach=False)
return 0
def is_enabled(cmd=None):
from dvc.config import Config, to_bool
from dvc.command.daemon import CmdDaemonBase
if env2bool("DVC_TEST"):
return False
if isinstance(cmd, CmdDaemonBase):
return False
core = Config(validate=False).config.get(Config.SECTION_CORE, {})
enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true"))
logger.debug(
"Analytics is {}.".format("enabled" if enabled else "disabled"))
return enabled
def test_init_no_scm_cli(tmp_dir):
ret = main(["init", "--no-scm"])
assert ret == 0
dvc_path = tmp_dir / DvcRepo.DVC_DIR
assert dvc_path.is_dir()
assert Config(os.fspath(dvc_path))["core"]["no_scm"]
def run(cmd_class, parse_config=True, args_start_loc=2):
"""
Arguments:
args_start_loc (int): where the arguments this command should use start
"""
try:
runtime_git = GitWrapper()
if parse_config:
runtime_config = Config(Runtime.conf_file_path(runtime_git.git_dir))
else:
runtime_config = ConfigI()
args = sys.argv[args_start_loc:]
# To make argparse print "usage: dvc cmd" instead of "usage: dvc"
sys.argv[0] = sys.argv[0] + " " + sys.argv[1]
instance = cmd_class(Settings(args, runtime_git, runtime_config))
sys.exit(instance.run())
except DvcException as e:
Logger.error(e)
sys.exit(1)
def aws_credentials_default_tes_t_DISABLED(self):
""" in absence of [AWS] -> CredentialPath, aws creds should be read from ~/.aws/credentials """
default_path = os.path.expanduser('~/.aws/credentials')
c = (
"[Global]",
"LogLevel =",
"DataDir = ",
"CacheDir = ",
"StateDir = ",
"Cloud = aws",
"[AWS]",
"StoragePath = awssb/aws_storage_path",
"CredentialPath =",
)
s = StringIO('\n'.join(c))
patcher = mock.patch(
builtin_module_name + '.open',
side_effect=self.mocked_open_aws_default_credentials)
# patcher.start()
conf = Config(s, conf_pseudo_file=s)
cloud = DataCloud(Settings(None, None, conf))
aws_creds = cloud._cloud._get_credentials()
patcher.stop()
self.assertEqual(aws_creds[0], 'default_access_id')
self.assertEqual(aws_creds[1], 'default_sekret')
def init(root_dir=os.curdir, no_scm=False):
"""
Initiate dvc project in directory.
Args:
root_dir: Path to project's root directory.
Returns:
Project instance.
Raises:
KeyError: Raises an exception.
"""
root_dir = os.path.abspath(root_dir)
dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
scm = SCM(root_dir)
if type(scm) == Base and not no_scm:
msg = "{} is not tracked by any supported scm tool(e.g. git).".format(
root_dir)
raise InitError(msg)
os.mkdir(dvc_dir)
config = Config.init(dvc_dir)
proj = Project(root_dir)
scm.add([config.config_file])
if scm.ignore_file():
scm.add([os.path.join(dvc_dir, scm.ignore_file())])
return proj
def __init__(self, root_dir):
from dvc.logger import Logger
from dvc.config import Config
from dvc.state import LinkState, State
from dvc.lock import Lock
from dvc.scm import SCM
from dvc.cache import Cache
from dvc.data_cloud import DataCloud
from dvc.updater import Updater
from dvc.prompt import Prompt
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
self.config = Config(self.dvc_dir)
self.scm = SCM(self.root_dir)
self.lock = Lock(self.dvc_dir)
# NOTE: storing state and link_state in the repository itself to avoid
# any possible state corruption in 'shared cache dir' scenario.
self.state = State(self)
self.link_state = LinkState(self)
core = self.config._config[Config.SECTION_CORE]
self.logger = Logger(core.get(Config.SECTION_CORE_LOGLEVEL, None))
self.cache = Cache(self)
self.cloud = DataCloud(self, config=self.config._config)
self.updater = Updater(self.dvc_dir)
self.prompt = Prompt()
self._ignore()
self.updater.check()
def init(root_dir=os.curdir):
"""
Initiate dvc project in directory.
Args:
root_dir: Path to project's root directory.
Returns:
Project instance.
Raises:
KeyError: Raises an exception.
"""
root_dir = os.path.abspath(root_dir)
dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
os.mkdir(dvc_dir)
config = Config.init(dvc_dir)
cache = Cache.init(dvc_dir)
state = State.init(root_dir, dvc_dir)
lock = Lock(dvc_dir)
scm = SCM(root_dir)
scm.ignore_list([cache.cache_dir, state.state_file, lock.lock_file])
ignore_file = os.path.join(dvc_dir, scm.ignore_file())
scm.add([config.config_file, ignore_file])
return Project(root_dir)
def __init__(self, root_dir=None, scm=None, rev=None):
from dvc.state import State, StateNoop
from dvc.lock import make_lock
from dvc.scm import SCM
from dvc.cache import Cache
from dvc.data_cloud import DataCloud
from dvc.repo.metrics import Metrics
from dvc.repo.plots import Plots
from dvc.repo.params import Params
from dvc.scm.tree import WorkingTree
from dvc.utils.fs import makedirs
from dvc.stage.cache import StageCache
if scm:
# use GitTree instead of WorkingTree as default repo tree instance
tree = scm.get_tree(rev)
self.root_dir = self.find_root(root_dir, tree)
self.scm = scm
self.tree = tree
self.state = StateNoop()
else:
root_dir = self.find_root(root_dir)
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
self.tree = WorkingTree(self.root_dir)
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
self.config = Config(self.dvc_dir, tree=self.tree)
if not scm:
no_scm = self.config["core"].get("no_scm", False)
self.scm = SCM(self.root_dir, no_scm=no_scm)
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
self.index_dir = os.path.join(self.tmp_dir, "index")
makedirs(self.index_dir, exist_ok=True)
hardlink_lock = self.config["core"].get("hardlink_lock", False)
self.lock = make_lock(
os.path.join(self.tmp_dir, "lock"),
tmp_dir=self.tmp_dir,
hardlink_lock=hardlink_lock,
friendly=True,
)
self.cache = Cache(self)
self.cloud = DataCloud(self)
if not scm:
# NOTE: storing state and link_state in the repository itself to
# avoid any possible state corruption in 'shared cache dir'
# scenario.
self.state = State(self.cache.local)
self.stage_cache = StageCache(self)
self.metrics = Metrics(self)
self.plots = Plots(self)
self.params = Params(self)
self._ignore()
def __init__(self, root_dir=None):
from dvc.config import Config
from dvc.state import State
from dvc.lock import Lock
from dvc.scm import SCM
from dvc.cache import Cache
from dvc.data_cloud import DataCloud
from dvc.updater import Updater
root_dir = self.find_root(root_dir)
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
self.config = Config(self.dvc_dir)
self.scm = SCM(self.root_dir, project=self)
self.lock = Lock(self.dvc_dir)
# NOTE: storing state and link_state in the repository itself to avoid
# any possible state corruption in 'shared cache dir' scenario.
self.state = State(self, self.config.config)
core = self.config.config[Config.SECTION_CORE]
logger.set_level(core.get(Config.SECTION_CORE_LOGLEVEL))
self.cache = Cache(self)
self.cloud = DataCloud(self, config=self.config.config)
self.updater = Updater(self.dvc_dir)
self.files_to_git_add = []
self._ignore()
self.updater.check()
def _find_or_create_user_id():
"""
The user's ID is stored on a file under the global config directory.
The file should contain a JSON with a "user_id" key:
{"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"}
IDs are generated randomly with UUID.
"""
config_dir = Config.get_global_config_dir()
fname = os.path.join(config_dir, "user_id")
lockfile = os.path.join(config_dir, "user_id.lock")
# Since the `fname` and `lockfile` are under the global config,
# we need to make sure such directory exist already.
makedirs(config_dir, exist_ok=True)
try:
with Lock(lockfile):
try:
with open(fname, "r") as fobj:
user_id = json.load(fobj)["user_id"]
except (FileNotFoundError, ValueError, KeyError):
user_id = str(uuid.uuid4())
with open(fname, "w") as fobj:
json.dump({"user_id": user_id}, fobj)
return user_id
except LockError:
logger.debug(
"Failed to acquire '{lockfile}'".format(lockfile=lockfile))
def is_enabled(self):
from dvc.config import Config, to_bool
enabled = to_bool(
Config(validate=False).get("core", {}).get("check_update", "true"))
logger.debug(
"Check for update is {}abled.".format("en" if enabled else "dis"))
return enabled
def test_remote_config_no_traverse():
d = Config.COMPILED_SCHEMA({'remote "myremote"': {"url": "url"}})
assert "no_traverse" not in d['remote "myremote"']
d = Config.COMPILED_SCHEMA(
{'remote "myremote"': {
"url": "url",
"no_traverse": "fAlSe"
}})
assert not d['remote "myremote"']["no_traverse"]
d = Config.COMPILED_SCHEMA(
{'remote "myremote"': {
"url": "url",
"no_traverse": "tRuE"
}})
assert d['remote "myremote"']["no_traverse"]
def init(root_dir=os.curdir, no_scm=False, force=False):
"""
Initiate dvc project in directory.
Args:
root_dir: Path to project's root directory.
Returns:
Project instance.
Raises:
KeyError: Raises an exception.
"""
import colorama
import shutil
from dvc.scm import SCM, Base
from dvc.config import Config
from dvc.logger import logger
root_dir = os.path.abspath(root_dir)
dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
scm = SCM(root_dir)
if type(scm) == Base and not no_scm:
msg = "{} is not tracked by any supported scm tool(e.g. git)."
raise InitError(msg.format(root_dir))
if os.path.isdir(dvc_dir):
if not force:
msg = "'{}' exists. Use '-f' to force."
raise InitError(msg.format(os.path.relpath(dvc_dir)))
shutil.rmtree(dvc_dir)
os.mkdir(dvc_dir)
config = Config.init(dvc_dir)
proj = Project(root_dir)
scm.add([config.config_file])
if scm.ignore_file():
scm.add([os.path.join(dvc_dir, scm.ignore_file())])
logger.info('\nYou can now commit the changes to git.')
logger.info(
"\n"
"{yellow}What's next?{nc}\n"
"{yellow}------------{nc}\n"
"- Check out the documentation: {blue}https://dvc.org/doc{nc}\n"
"- Get help and share ideas: {blue}https://dvc.org/chat{nc}\n"
"- Star us on GitHub: {blue}https://github.com/iterative/dvc{nc}"
.format(yellow=colorama.Fore.YELLOW,
blue=colorama.Fore.BLUE,
green=colorama.Fore.GREEN,
nc=colorama.Fore.RESET)
)
return proj
def init(root_dir=os.curdir, no_scm=False, force=False):
"""
Creates an empty project on the given directory -- basically a
`.dvc` directory with subdirectories for configuration and cache.
It should be tracked by a SCM or use the `--no-scm` flag.
If the given directory is not empty, you must use the `--force`
flag to override it.
Args:
root_dir: Path to project's root directory.
Returns:
Project instance.
Raises:
KeyError: Raises an exception.
"""
import shutil
from dvc.scm import SCM, Base
from dvc.config import Config
root_dir = os.path.abspath(root_dir)
dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
scm = SCM(root_dir)
if type(scm) == Base and not no_scm:
raise InitError(
"{project} is not tracked by any supported scm tool"
" (e.g. git). Use '--no-scm' if you don't want to use any scm.".format(
project=root_dir
)
)
if os.path.isdir(dvc_dir):
if not force:
raise InitError(
"'{project}' exists. Use '-f' to force.".format(
project=os.path.relpath(dvc_dir)
)
)
shutil.rmtree(dvc_dir)
os.mkdir(dvc_dir)
config = Config.init(dvc_dir)
proj = Project(root_dir)
scm.add([config.config_file])
if scm.ignore_file:
scm.add([os.path.join(dvc_dir, scm.ignore_file)])
logger.info("\nYou can now commit the changes to git.\n")
proj._welcome_message()
return proj
def is_enabled():
if env2bool("DVC_TEST"):
return False
enabled = to_bool(
Config(validate=False).get("core", {}).get("analytics", "true"))
logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
return enabled
class CmdConfig(CmdBaseNoRepo):
def __init__(self, args):
super(CmdConfig, self).__init__(args)
self.config = Config(validate=False)
def run(self):
section, opt = self.args.name.lower().strip().split(".", 1)
if self.args.unset:
self.config.unset(section, opt, level=self.args.level)
elif self.args.value is None:
logger.info(self.config.get(section, opt, level=self.args.level))
else:
self.config.set(section,
opt,
self.args.value,
level=self.args.level)
return 0
def is_enabled():
if env2bool("DVC_TEST"):
return False
enabled = to_bool(
Config(validate=False).config.get(Config.SECTION_CORE, {}).get(
Config.SECTION_CORE_ANALYTICS, "true"))
logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
return enabled
