def _is_enabled(cmd=None): from dvc.config import Config from dvc.project import Project from dvc.exceptions import NotDvcProjectError from dvc.command.daemon import CmdDaemonBase if os.getenv("DVC_TEST"): return False if isinstance(cmd, CmdDaemonBase): return False if cmd is None or not hasattr(cmd, "config"): try: dvc_dir = Project.find_dvc_dir() config = Config(dvc_dir) assert config is not None except NotDvcProjectError: config = Config(validate=False) assert config is not None else: config = cmd.config assert config is not None core = config.config.get(Config.SECTION_CORE, {}) enabled = core.get(Config.SECTION_CORE_ANALYTICS, True) logger.debug("Analytics is {}abled.".format("en" if enabled else "dis")) return enabled
def _is_enabled(cmd=None): from dvc.config import Config from dvc.repo import Repo from dvc.exceptions import NotDvcRepoError from dvc.command.daemon import CmdDaemonBase if os.getenv("DVC_TEST"): return False if isinstance(cmd, CmdDaemonBase): return False if cmd is None or not hasattr(cmd, "config"): try: dvc_dir = Repo.find_dvc_dir() config = Config(dvc_dir) assert config is not None except NotDvcRepoError: config = Config(validate=False) assert config is not None else: config = cmd.config assert config is not None enabled = Analytics._is_enabled_config(config) logger.debug( "Analytics is {}.".format("enabled" if enabled else "disabled") ) return enabled
def aws_credentials_specified_test(self, isfile_function): """ in presence of [AWS] -> CredentialPath, use those credentials """ c = ( "[Global]", "LogLevel =", "DataDir = ", "CacheDir = ", "StateDir = ", "Cloud = amazon", "[AWS]", "StoragePath = awssb/aws_storage_path", "CredentialPath = some_credential_path", ) s = StringIO.StringIO('\n'.join(c)) patcher = mock.patch( '__builtin__.open', side_effect=self.mocked_open_aws_default_credentials) patcher.start() conf = Config(s, conf_pseudo_file=s) aws_creds = conf.get_aws_credentials() patcher.stop() self.assertEqual(aws_creds[0], 'override_access_id') self.assertEqual(aws_creds[1], 'override_sekret')
def aws_credentials_default_test(self): """ in absence of [AWS] -> CredentialPath, aws creds should be read from ~/.aws/credentials """ default_path = os.path.expanduser('~/.aws/credentials') c = ( "[Global]", "LogLevel =", "DataDir = ", "CacheDir = ", "StateDir = ", "Cloud = amazon", "[AWS]", "StoragePath = awssb/aws_storage_path", "CredentialPath =", ) s = StringIO.StringIO('\n'.join(c)) patcher = mock.patch( '__builtin__.open', side_effect=self.mocked_open_aws_default_credentials) patcher.start() conf = Config(s, conf_pseudo_file=s) aws_creds = conf.get_aws_credentials() patcher.stop() self.assertEqual(aws_creds[0], 'default_access_id') self.assertEqual(aws_creds[1], 'default_sekret')
def __init__(self, args): from dvc.project import Project, NotDvcProjectError self.args = args try: dvc_dir = os.path.join(Project.find_root(), Project.DVC_DIR) saved_exc = None except NotDvcProjectError as exc: dvc_dir = None saved_exc = exc self.config = Config(dvc_dir, validate=False) if self.args.system: self.configobj = self.config._system_config elif self.args.glob: self.configobj = self.config._global_config elif self.args.local: if dvc_dir is None: raise saved_exc self.configobj = self.config._local_config else: if dvc_dir is None: raise saved_exc self.configobj = self.config._project_config
def test_s3_ssl_verify(tmp_dir, dvc): config = Config(validate=False) with config.edit() as conf: conf["remote"]["remote-name"] = {"url": "s3://bucket/dvc"} assert "ssl_verify" not in config["remote"]["remote-name"] with config.edit() as conf: section = conf["remote"]["remote-name"] section["ssl_verify"] = False assert (tmp_dir / ".dvc" / "config").read_text() == textwrap.dedent("""\ [core] no_scm = True ['remote "remote-name"'] url = s3://bucket/dvc ssl_verify = False """) with config.edit() as conf: section = conf["remote"]["remote-name"] section["ssl_verify"] = "/path/to/custom/cabundle.pem" assert (tmp_dir / ".dvc" / "config").read_text() == textwrap.dedent("""\ [core] no_scm = True ['remote "remote-name"'] url = s3://bucket/dvc ssl_verify = /path/to/custom/cabundle.pem """)
def storage_path_hierarchy_test(self): """ StoragePath should be read first from global and then from cloud section """ c = ("[Global]", "LogLevel =", "DataDir = ", "CacheDir = ", "StateDir = ", "Cloud = aws", "StoragePath = globalsb/global_storage_path", "[AWS]", "StoragePath = awssb/aws_storage_path", "CredentialPath =", "[GCP]", "StoragePath = googlesb/google_storage_path") s = StringIO('\n'.join(c)) conf = Config(s, conf_pseudo_file=s) cloud = DataCloud(Settings(None, None, conf)) self.assertEqual(cloud.typ, 'AWS') self.assertEqual(cloud._cloud.storage_bucket, 'globalsb') self.assertEqual(cloud._cloud.storage_prefix, 'global_storage_path') c = ("[Global]", "LogLevel =", "DataDir = ", "CacheDir = ", "StateDir = ", "Cloud = Aws", "[AWS]", "StoragePath = awssb/aws_storage_path", "CredentialPath =", "[GCP]", "StoragePath = googlesb/google_storage_path") s = StringIO('\n'.join(c)) conf = Config(s, conf_pseudo_file=s) cloud = DataCloud(Settings(None, None, conf)) self.assertEqual(cloud.typ, 'AWS') self.assertEqual(cloud._cloud.storage_bucket, 'awssb') self.assertEqual(cloud._cloud.storage_prefix, 'aws_storage_path')
class CmdConfig(CmdBaseNoRepo): def __init__(self, args): super().__init__(args) self.config = Config(validate=False) def run(self): section, opt = self.args.name.lower().strip().split(".", 1) if self.args.unset: self.config.unset(section, opt, level=self.args.level) elif self.args.value is None: logger.info(self.config.get(section, opt, level=self.args.level)) else: self.config.set(section, opt, self.args.value, level=self.args.level) is_write = self.args.unset or self.args.value is not None if is_write and self.args.name == "cache.type": logger.warning( "You have changed the 'cache.type' option. This doesn't update" " any existing workspace file links, but it can be done with:" "\n dvc checkout --relink") return 0
class CmdConfig(CmdBaseNoRepo): def __init__(self, args): super().__init__(args) self.config = Config(validate=False) def run(self): if self.args.list: if any((self.args.name, self.args.value, self.args.unset)): logger.error( "-l/--list can't be used together with any of these " "options: -u/--unset, name, value") return 1 conf = self.config.load_one(self.args.level) logger.info("\n".join(self._format_config(conf))) return 0 if self.args.name is None: logger.error("name argument is required") return 1 section, opt = self.args.name.lower().strip().split(".", 1) if self.args.value is None and not self.args.unset: conf = self.config.load_one(self.args.level) self._check(conf, section, opt) logger.info(conf[section][opt]) return 0 with self.config.edit(self.args.level) as conf: if self.args.unset: self._check(conf, section, opt) del conf[section][opt] else: self._check(conf, section) conf[section][opt] = self.args.value if self.args.name == "cache.type": logger.warning( "You have changed the 'cache.type' option. This doesn't update" " any existing workspace file links, but it can be done with:" "\n dvc checkout --relink") return 0 def _check(self, conf, section, opt=None): if section not in conf: msg = "section {} doesn't exist" raise ConfigError(msg.format(self.args.name)) if opt and opt not in conf[section]: msg = "option {} doesn't exist" raise ConfigError(msg.format(self.args.name)) @staticmethod def _format_config(config): for key, value in flatten(config).items(): yield f"{key}={value}"
def _get_current_config(): from dvc.config import Config from dvc.repo import Repo from dvc.exceptions import NotDvcRepoError try: dvc_dir = Repo.find_dvc_dir() config = Config(dvc_dir) except NotDvcRepoError: config = Config(validate=False) return config
def test_get_tree(tmp_dir, scm): tmp_dir.scm_gen("foo", "foo", commit="add foo") tree = scm.get_tree("master") config = Config(tree=tree) assert config.tree == tree assert config.wtree != tree assert isinstance(config.wtree, LocalTree) assert config._get_tree("repo") == tree assert config._get_tree("local") == config.wtree assert config._get_tree("global") == config.wtree assert config._get_tree("system") == config.wtree
def test_get_fs(tmp_dir, scm): tmp_dir.scm_gen("foo", "foo", commit="add foo") fs = scm.get_fs("master") config = Config(fs=fs) assert config.fs == fs assert config.wfs != fs assert isinstance(config.wfs, LocalFileSystem) assert config._get_fs("repo") == fs assert config._get_fs("local") == config.wfs assert config._get_fs("global") == config.wfs assert config._get_fs("system") == config.wfs
def run(self): import os from dvc.repo import Repo from dvc.config import Config from dvc.updater import Updater root_dir = Repo.find_root() dvc_dir = os.path.join(root_dir, Repo.DVC_DIR) config = Config(dvc_dir, validate=False) hardlink_lock = config.get("core", {}).get("hardlink_lock", False) updater = Updater(dvc_dir, hardlink_lock=hardlink_lock) updater.fetch(detach=False) return 0
def is_enabled(cmd=None): from dvc.config import Config, to_bool from dvc.command.daemon import CmdDaemonBase if env2bool("DVC_TEST"): return False if isinstance(cmd, CmdDaemonBase): return False core = Config(validate=False).config.get(Config.SECTION_CORE, {}) enabled = to_bool(core.get(Config.SECTION_CORE_ANALYTICS, "true")) logger.debug( "Analytics is {}.".format("enabled" if enabled else "disabled")) return enabled
def test_init_no_scm_cli(tmp_dir): ret = main(["init", "--no-scm"]) assert ret == 0 dvc_path = tmp_dir / DvcRepo.DVC_DIR assert dvc_path.is_dir() assert Config(os.fspath(dvc_path))["core"]["no_scm"]
def run(cmd_class, parse_config=True, args_start_loc=2): """ Arguments: args_start_loc (int): where the arguments this command should use start """ try: runtime_git = GitWrapper() if parse_config: runtime_config = Config(Runtime.conf_file_path(runtime_git.git_dir)) else: runtime_config = ConfigI() args = sys.argv[args_start_loc:] # To make argparse print "usage: dvc cmd" instead of "usage: dvc" sys.argv[0] = sys.argv[0] + " " + sys.argv[1] instance = cmd_class(Settings(args, runtime_git, runtime_config)) sys.exit(instance.run()) except DvcException as e: Logger.error(e) sys.exit(1)
def aws_credentials_default_tes_t_DISABLED(self): """ in absence of [AWS] -> CredentialPath, aws creds should be read from ~/.aws/credentials """ default_path = os.path.expanduser('~/.aws/credentials') c = ( "[Global]", "LogLevel =", "DataDir = ", "CacheDir = ", "StateDir = ", "Cloud = aws", "[AWS]", "StoragePath = awssb/aws_storage_path", "CredentialPath =", ) s = StringIO('\n'.join(c)) patcher = mock.patch( builtin_module_name + '.open', side_effect=self.mocked_open_aws_default_credentials) # patcher.start() conf = Config(s, conf_pseudo_file=s) cloud = DataCloud(Settings(None, None, conf)) aws_creds = cloud._cloud._get_credentials() patcher.stop() self.assertEqual(aws_creds[0], 'default_access_id') self.assertEqual(aws_creds[1], 'default_sekret')
def init(root_dir=os.curdir, no_scm=False): """ Initiate dvc project in directory. Args: root_dir: Path to project's root directory. Returns: Project instance. Raises: KeyError: Raises an exception. """ root_dir = os.path.abspath(root_dir) dvc_dir = os.path.join(root_dir, Project.DVC_DIR) scm = SCM(root_dir) if type(scm) == Base and not no_scm: msg = "{} is not tracked by any supported scm tool(e.g. git).".format( root_dir) raise InitError(msg) os.mkdir(dvc_dir) config = Config.init(dvc_dir) proj = Project(root_dir) scm.add([config.config_file]) if scm.ignore_file(): scm.add([os.path.join(dvc_dir, scm.ignore_file())]) return proj
def __init__(self, root_dir): from dvc.logger import Logger from dvc.config import Config from dvc.state import LinkState, State from dvc.lock import Lock from dvc.scm import SCM from dvc.cache import Cache from dvc.data_cloud import DataCloud from dvc.updater import Updater from dvc.prompt import Prompt self.root_dir = os.path.abspath(os.path.realpath(root_dir)) self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) self.config = Config(self.dvc_dir) self.scm = SCM(self.root_dir) self.lock = Lock(self.dvc_dir) # NOTE: storing state and link_state in the repository itself to avoid # any possible state corruption in 'shared cache dir' scenario. self.state = State(self) self.link_state = LinkState(self) core = self.config._config[Config.SECTION_CORE] self.logger = Logger(core.get(Config.SECTION_CORE_LOGLEVEL, None)) self.cache = Cache(self) self.cloud = DataCloud(self, config=self.config._config) self.updater = Updater(self.dvc_dir) self.prompt = Prompt() self._ignore() self.updater.check()
def init(root_dir=os.curdir): """ Initiate dvc project in directory. Args: root_dir: Path to project's root directory. Returns: Project instance. Raises: KeyError: Raises an exception. """ root_dir = os.path.abspath(root_dir) dvc_dir = os.path.join(root_dir, Project.DVC_DIR) os.mkdir(dvc_dir) config = Config.init(dvc_dir) cache = Cache.init(dvc_dir) state = State.init(root_dir, dvc_dir) lock = Lock(dvc_dir) scm = SCM(root_dir) scm.ignore_list([cache.cache_dir, state.state_file, lock.lock_file]) ignore_file = os.path.join(dvc_dir, scm.ignore_file()) scm.add([config.config_file, ignore_file]) return Project(root_dir)
def __init__(self, root_dir=None, scm=None, rev=None): from dvc.state import State, StateNoop from dvc.lock import make_lock from dvc.scm import SCM from dvc.cache import Cache from dvc.data_cloud import DataCloud from dvc.repo.metrics import Metrics from dvc.repo.plots import Plots from dvc.repo.params import Params from dvc.scm.tree import WorkingTree from dvc.utils.fs import makedirs from dvc.stage.cache import StageCache if scm: # use GitTree instead of WorkingTree as default repo tree instance tree = scm.get_tree(rev) self.root_dir = self.find_root(root_dir, tree) self.scm = scm self.tree = tree self.state = StateNoop() else: root_dir = self.find_root(root_dir) self.root_dir = os.path.abspath(os.path.realpath(root_dir)) self.tree = WorkingTree(self.root_dir) self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) self.config = Config(self.dvc_dir, tree=self.tree) if not scm: no_scm = self.config["core"].get("no_scm", False) self.scm = SCM(self.root_dir, no_scm=no_scm) self.tmp_dir = os.path.join(self.dvc_dir, "tmp") self.index_dir = os.path.join(self.tmp_dir, "index") makedirs(self.index_dir, exist_ok=True) hardlink_lock = self.config["core"].get("hardlink_lock", False) self.lock = make_lock( os.path.join(self.tmp_dir, "lock"), tmp_dir=self.tmp_dir, hardlink_lock=hardlink_lock, friendly=True, ) self.cache = Cache(self) self.cloud = DataCloud(self) if not scm: # NOTE: storing state and link_state in the repository itself to # avoid any possible state corruption in 'shared cache dir' # scenario. self.state = State(self.cache.local) self.stage_cache = StageCache(self) self.metrics = Metrics(self) self.plots = Plots(self) self.params = Params(self) self._ignore()
def __init__(self, root_dir=None): from dvc.config import Config from dvc.state import State from dvc.lock import Lock from dvc.scm import SCM from dvc.cache import Cache from dvc.data_cloud import DataCloud from dvc.updater import Updater root_dir = self.find_root(root_dir) self.root_dir = os.path.abspath(os.path.realpath(root_dir)) self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) self.config = Config(self.dvc_dir) self.scm = SCM(self.root_dir, project=self) self.lock = Lock(self.dvc_dir) # NOTE: storing state and link_state in the repository itself to avoid # any possible state corruption in 'shared cache dir' scenario. self.state = State(self, self.config.config) core = self.config.config[Config.SECTION_CORE] logger.set_level(core.get(Config.SECTION_CORE_LOGLEVEL)) self.cache = Cache(self) self.cloud = DataCloud(self, config=self.config.config) self.updater = Updater(self.dvc_dir) self.files_to_git_add = [] self._ignore() self.updater.check()
def _find_or_create_user_id(): """ The user's ID is stored on a file under the global config directory. The file should contain a JSON with a "user_id" key: {"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"} IDs are generated randomly with UUID. """ config_dir = Config.get_global_config_dir() fname = os.path.join(config_dir, "user_id") lockfile = os.path.join(config_dir, "user_id.lock") # Since the `fname` and `lockfile` are under the global config, # we need to make sure such directory exist already. makedirs(config_dir, exist_ok=True) try: with Lock(lockfile): try: with open(fname, "r") as fobj: user_id = json.load(fobj)["user_id"] except (FileNotFoundError, ValueError, KeyError): user_id = str(uuid.uuid4()) with open(fname, "w") as fobj: json.dump({"user_id": user_id}, fobj) return user_id except LockError: logger.debug( "Failed to acquire '{lockfile}'".format(lockfile=lockfile))
def is_enabled(self): from dvc.config import Config, to_bool enabled = to_bool( Config(validate=False).get("core", {}).get("check_update", "true")) logger.debug( "Check for update is {}abled.".format("en" if enabled else "dis")) return enabled
def test_remote_config_no_traverse(): d = Config.COMPILED_SCHEMA({'remote "myremote"': {"url": "url"}}) assert "no_traverse" not in d['remote "myremote"'] d = Config.COMPILED_SCHEMA( {'remote "myremote"': { "url": "url", "no_traverse": "fAlSe" }}) assert not d['remote "myremote"']["no_traverse"] d = Config.COMPILED_SCHEMA( {'remote "myremote"': { "url": "url", "no_traverse": "tRuE" }}) assert d['remote "myremote"']["no_traverse"]
def init(root_dir=os.curdir, no_scm=False, force=False): """ Initiate dvc project in directory. Args: root_dir: Path to project's root directory. Returns: Project instance. Raises: KeyError: Raises an exception. """ import colorama import shutil from dvc.scm import SCM, Base from dvc.config import Config from dvc.logger import logger root_dir = os.path.abspath(root_dir) dvc_dir = os.path.join(root_dir, Project.DVC_DIR) scm = SCM(root_dir) if type(scm) == Base and not no_scm: msg = "{} is not tracked by any supported scm tool(e.g. git)." raise InitError(msg.format(root_dir)) if os.path.isdir(dvc_dir): if not force: msg = "'{}' exists. Use '-f' to force." raise InitError(msg.format(os.path.relpath(dvc_dir))) shutil.rmtree(dvc_dir) os.mkdir(dvc_dir) config = Config.init(dvc_dir) proj = Project(root_dir) scm.add([config.config_file]) if scm.ignore_file(): scm.add([os.path.join(dvc_dir, scm.ignore_file())]) logger.info('\nYou can now commit the changes to git.') logger.info( "\n" "{yellow}What's next?{nc}\n" "{yellow}------------{nc}\n" "- Check out the documentation: {blue}https://dvc.org/doc{nc}\n" "- Get help and share ideas: {blue}https://dvc.org/chat{nc}\n" "- Star us on GitHub: {blue}https://github.com/iterative/dvc{nc}" .format(yellow=colorama.Fore.YELLOW, blue=colorama.Fore.BLUE, green=colorama.Fore.GREEN, nc=colorama.Fore.RESET) ) return proj
def init(root_dir=os.curdir, no_scm=False, force=False): """ Creates an empty project on the given directory -- basically a `.dvc` directory with subdirectories for configuration and cache. It should be tracked by a SCM or use the `--no-scm` flag. If the given directory is not empty, you must use the `--force` flag to override it. Args: root_dir: Path to project's root directory. Returns: Project instance. Raises: KeyError: Raises an exception. """ import shutil from dvc.scm import SCM, Base from dvc.config import Config root_dir = os.path.abspath(root_dir) dvc_dir = os.path.join(root_dir, Project.DVC_DIR) scm = SCM(root_dir) if type(scm) == Base and not no_scm: raise InitError( "{project} is not tracked by any supported scm tool" " (e.g. git). Use '--no-scm' if you don't want to use any scm.".format( project=root_dir ) ) if os.path.isdir(dvc_dir): if not force: raise InitError( "'{project}' exists. Use '-f' to force.".format( project=os.path.relpath(dvc_dir) ) ) shutil.rmtree(dvc_dir) os.mkdir(dvc_dir) config = Config.init(dvc_dir) proj = Project(root_dir) scm.add([config.config_file]) if scm.ignore_file: scm.add([os.path.join(dvc_dir, scm.ignore_file)]) logger.info("\nYou can now commit the changes to git.\n") proj._welcome_message() return proj
def is_enabled(): if env2bool("DVC_TEST"): return False enabled = to_bool( Config(validate=False).get("core", {}).get("analytics", "true")) logger.debug("Analytics is {}abled.".format("en" if enabled else "dis")) return enabled
class CmdConfig(CmdBaseNoRepo): def __init__(self, args): super(CmdConfig, self).__init__(args) self.config = Config(validate=False) def run(self): section, opt = self.args.name.lower().strip().split(".", 1) if self.args.unset: self.config.unset(section, opt, level=self.args.level) elif self.args.value is None: logger.info(self.config.get(section, opt, level=self.args.level)) else: self.config.set(section, opt, self.args.value, level=self.args.level) return 0
def is_enabled(): if env2bool("DVC_TEST"): return False enabled = to_bool( Config(validate=False).config.get(Config.SECTION_CORE, {}).get( Config.SECTION_CORE_ANALYTICS, "true")) logger.debug("Analytics is {}abled.".format("en" if enabled else "dis")) return enabled