def test_with(self): lock = Lock(self.dvc.dvc_dir) with lock: with self.assertRaises(LockError): lock2 = Lock(self.dvc.dvc_dir) with lock2: self.assertTrue(False)
def test_with(tmp_dir, dvc, mocker): # patching to speedup tests mocker.patch("dvc.lock.DEFAULT_TIMEOUT", 0.01) lockfile = tmp_dir / dvc.tmp_dir / "lock" with Lock(lockfile): with pytest.raises(LockError), Lock(lockfile): pass
def test_with(self): lockfile = os.path.join(self.dvc.dvc_dir, "lock") lock = Lock(lockfile) with lock: with self.assertRaises(LockError): lock2 = Lock(lockfile) with lock2: self.assertTrue(False)
def init(root_dir=os.curdir): """ Initiate dvc project in directory. Args: root_dir: Path to project's root directory. Returns: Project instance. Raises: KeyError: Raises an exception. """ root_dir = os.path.abspath(root_dir) dvc_dir = os.path.join(root_dir, Project.DVC_DIR) os.mkdir(dvc_dir) config = Config.init(dvc_dir) cache = Cache.init(dvc_dir) state = State.init(root_dir, dvc_dir) lock = Lock(dvc_dir) scm = SCM(root_dir) scm.ignore_list([cache.cache_dir, state.state_file, lock.lock_file]) ignore_file = os.path.join(dvc_dir, scm.ignore_file()) scm.add([config.config_file, ignore_file]) return Project(root_dir)
def __init__(self, dvc_dir): self.dvc_dir = dvc_dir self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE) self.lock = Lock( self.updater_file + ".lock", tmp_dir=os.path.join(dvc_dir, "tmp") ) self.current = version.parse(__version__).base_version
def __init__(self, root_dir=None): from dvc.config import Config from dvc.state import State from dvc.lock import Lock from dvc.scm import SCM from dvc.cache import Cache from dvc.data_cloud import DataCloud from dvc.updater import Updater root_dir = self.find_root(root_dir) self.root_dir = os.path.abspath(os.path.realpath(root_dir)) self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) self.config = Config(self.dvc_dir) self.scm = SCM(self.root_dir, project=self) self.lock = Lock(self.dvc_dir) # NOTE: storing state and link_state in the repository itself to avoid # any possible state corruption in 'shared cache dir' scenario. self.state = State(self, self.config.config) core = self.config.config[Config.SECTION_CORE] logger.set_level(core.get(Config.SECTION_CORE_LOGLEVEL)) self.cache = Cache(self) self.cloud = DataCloud(self, config=self.config.config) self.updater = Updater(self.dvc_dir) self.files_to_git_add = [] self._ignore() self.updater.check()
def _find_or_create_user_id(): """ The user's ID is stored on a file under the global config directory. The file should contain a JSON with a "user_id" key: {"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"} IDs are generated randomly with UUID. """ config_dir = Config.get_global_config_dir() fname = os.path.join(config_dir, "user_id") lockfile = os.path.join(config_dir, "user_id.lock") # Since the `fname` and `lockfile` are under the global config, # we need to make sure such directory exist already. makedirs(config_dir, exist_ok=True) try: with Lock(lockfile): try: with open(fname, "r") as fobj: user_id = json.load(fobj)["user_id"] except (FileNotFoundError, ValueError, KeyError): user_id = str(uuid.uuid4()) with open(fname, "w") as fobj: json.dump({"user_id": user_id}, fobj) return user_id except LockError: logger.debug( "Failed to acquire '{lockfile}'".format(lockfile=lockfile))
def __init__(self, root_dir): from dvc.logger import Logger from dvc.config import Config from dvc.state import LinkState, State from dvc.lock import Lock from dvc.scm import SCM from dvc.cache import Cache from dvc.data_cloud import DataCloud from dvc.updater import Updater from dvc.prompt import Prompt self.root_dir = os.path.abspath(os.path.realpath(root_dir)) self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) self.config = Config(self.dvc_dir) self.scm = SCM(self.root_dir) self.lock = Lock(self.dvc_dir) # NOTE: storing state and link_state in the repository itself to avoid # any possible state corruption in 'shared cache dir' scenario. self.state = State(self) self.link_state = LinkState(self) core = self.config._config[Config.SECTION_CORE] self.logger = Logger(core.get(Config.SECTION_CORE_LOGLEVEL, None)) self.cache = Cache(self) self.cloud = DataCloud(self, config=self.config._config) self.updater = Updater(self.dvc_dir) self.prompt = Prompt() self._ignore() self.updater.check()
def test_cli(tmp_dir, dvc, mocker, caplog): # patching to speedup tests mocker.patch("dvc.lock.DEFAULT_TIMEOUT", 0.01) with Lock(tmp_dir / dvc.tmp_dir / "lock"): assert main(["add", "foo"]) == 1 assert FAILED_TO_LOCK_MESSAGE in caplog.text
def __init__(self, dvc_dir): self.dvc_dir = dvc_dir self._lock = threading.Lock() if dvc_dir: self.state_file = os.path.join(dvc_dir, self.STATE_FILE) self._lock_file = Lock(dvc_dir, self.STATE_LOCK_FILE) else: self.state_file = None self._lock_file = threading.Lock() self._db = self.load()
def __init__(self, root_dir): self.root_dir = os.path.abspath(os.path.realpath(root_dir)) self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) self.scm = SCM(self.root_dir) self.lock = Lock(self.dvc_dir) self.cache = Cache(self.dvc_dir) self.state = State(self.root_dir, self.dvc_dir) self.config = Config(self.dvc_dir) self.logger = Logger(self.config._config) self.cloud = DataCloud(self.cache, self.config._config)
def __init__(self, project): self.project = project self.dvc_dir = project.dvc_dir self._lock = threading.Lock() if self.dvc_dir: self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE) self._lock_file = Lock(self.dvc_dir, self.STATE_LOCK_FILE) else: self.state_file = None self._lock_file = threading.Lock() self._db = self.load()
def test_cli(tmp_dir, dvc, mocker, caplog): # patching to speedup tests mocker.patch("dvc.lock.DEFAULT_TIMEOUT", 0.01) expected_error_msg = ( "Unable to acquire lock. Most likely another DVC process is " "running or was terminated abruptly. Check the page " "<https://dvc.org/doc/user-guide/troubleshooting#lock-issue> " "for other possible reasons and to learn how to resolve this.") with Lock(tmp_dir / dvc.tmp_dir / "lock"): assert main(["add", "foo"]) == 1 assert expected_error_msg in caplog.text
def test_lock(self): lock = Lock(self.dvc.dvc_dir) lock.lock() self.assertTrue(lock._lock.exists()) self.assertTrue(lock._lock.acquired) lock.unlock() self.assertFalse(lock._lock.acquired)
def __init__(self, root_dir): self.root_dir = os.path.abspath(os.path.realpath(root_dir)) self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) self.scm = SCM(self.root_dir) self.lock = Lock(self.dvc_dir) self.cache = Cache(self.dvc_dir) self.state = State(self.root_dir, self.dvc_dir) self.config = Config(self.dvc_dir) self.logger = Logger(self.config._config[Config.SECTION_CORE].get( Config.SECTION_CORE_LOGLEVEL, None)) self.cloud = DataCloud(cache=self.cache, state=self.state, config=self.config._config)
def __init__(self, info={}): from dvc.config import Config from dvc.lock import Lock self.info = info cdir = Config.get_global_config_dir() try: os.makedirs(cdir) except OSError as e: if e.errno != errno.EEXIST: raise self.user_id_file = os.path.join(cdir, self.USER_ID_FILE) self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + '.lock')
def __init__(self, root_dir, dvc_dir, cache_dir=None, cache_type=None): self.cache_type = cache_type cache_dir = cache_dir if cache_dir else self.CACHE_DIR if os.path.isabs(cache_dir): self.cache_dir = cache_dir else: self.cache_dir = os.path.abspath( os.path.realpath(os.path.join(dvc_dir, cache_dir))) if not os.path.exists(self.cache_dir): os.mkdir(self.cache_dir) self.state = State(self.cache_dir) self.link_state = LinkState(root_dir, dvc_dir) self.lock = Lock(self.cache_dir, name=self.CACHE_DIR_LOCK)
def __init__(self, root_dir): self.root_dir = os.path.abspath(os.path.realpath(root_dir)) self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) self.config = Config(self.dvc_dir) self.scm = SCM(self.root_dir) self.lock = Lock(self.dvc_dir) self.link_state = LinkState(self.root_dir, self.dvc_dir) self.logger = Logger(self.config._config[Config.SECTION_CORE].get( Config.SECTION_CORE_LOGLEVEL, None)) self.cache = Cache(self) self.cloud = DataCloud(self, config=self.config._config) self.updater = Updater(self.dvc_dir) self._ignore() self.updater.check()
def __init__(self, root_dir=None): from dvc.config import Config from dvc.state import State from dvc.lock import Lock from dvc.scm import SCM from dvc.cache import Cache from dvc.data_cloud import DataCloud from dvc.updater import Updater from dvc.repo.metrics import Metrics from dvc.scm.tree import WorkingTree from dvc.repo.tag import Tag from dvc.repo.pkg import Pkg root_dir = self.find_root(root_dir) self.root_dir = os.path.abspath(os.path.realpath(root_dir)) self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) self.config = Config(self.dvc_dir) self.tree = WorkingTree(self.root_dir) self.scm = SCM(self.root_dir, repo=self) self.lock = Lock(self.dvc_dir) # NOTE: storing state and link_state in the repository itself to avoid # any possible state corruption in 'shared cache dir' scenario. self.state = State(self, self.config.config) core = self.config.config[Config.SECTION_CORE] level = core.get(Config.SECTION_CORE_LOGLEVEL) if level: logger.setLevel(level.upper()) self.cache = Cache(self) self.cloud = DataCloud(self, config=self.config.config) self.updater = Updater(self.dvc_dir) self.metrics = Metrics(self) self.tag = Tag(self) self.pkg = Pkg(self) self._ignore() self.updater.check()
def __init__(self, root_dir): self.root_dir = os.path.abspath(os.path.realpath(root_dir)) self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) self.config = Config(self.dvc_dir) self.scm = SCM(self.root_dir) self.lock = Lock(self.dvc_dir) # NOTE: storing state and link_state in the repository itself to avoid # any possible state corruption in 'shared cache dir' scenario. self.state = State(self) self.link_state = LinkState(self) self.logger = Logger(self.config._config[Config.SECTION_CORE].get( Config.SECTION_CORE_LOGLEVEL, None)) self.cache = Cache(self) self.cloud = DataCloud(self, config=self.config._config) self.updater = Updater(self.dvc_dir) self._ignore() self.updater.check()
def test_cli(self): lockfile = os.path.join(self.dvc.dvc_dir, "lock") lock = Lock(lockfile) with lock: ret = main(["add", self.FOO]) self.assertEqual(ret, 1)
def test_with(self): lock = Lock(self.dvc.dvc_dir) with lock: self.assertTrue(lock._lock.acquired) self.assertFalse(lock._lock.acquired)
def __init__(self, dvc_dir): self.dvc_dir = dvc_dir self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE) self.lock = Lock(dvc_dir, self.updater_file + '.lock') self.current = VERSION_BASE
def test_cli(self): lock = Lock(self.dvc.dvc_dir) with lock: ret = main(["add", self.FOO]) self.assertEqual(ret, 1)