def __init__(self, lockfile, tmp_dir=None):
lifetime = timedelta(days=365) # Lock for good by default
self._tmp_dir = tmp_dir
if self._tmp_dir is not None:
makedirs(self._tmp_dir, exist_ok=True)
super(Lock, self).__init__(lockfile, lifetime=lifetime)
def __init__(self, lockfile, tmp_dir=None):
import socket
self._tmp_dir = tmp_dir
if self._tmp_dir is not None:
makedirs(self._tmp_dir, exist_ok=True)
# NOTE: this is basically Lock.__init__ copy-paste, except that
# instead of using `socket.getfqdn()` we use `socket.gethostname()`
# to speed this up. We've seen [1] `getfqdn()` take ~5sec to return
# anything, which is way too slow. `gethostname()` is actually a
# fallback for `getfqdn()` when it is not able to resolve a
# canonical hostname through network. The claimfile that uses
# `self._hostname` is still usable, as it uses `pid` and random
# number to generate the resulting lock file name, which is unique
# enough for our application.
#
# [1] https://github.com/iterative/dvc/issues/2582
self._hostname = socket.gethostname()
self._lockfile = lockfile
self._lifetime = timedelta(days=365) # Lock for good by default
self._separator = flufl.lock.SEP
self._set_claimfile()
self._owned = True
self._retry_errnos = []
def _find_or_create_user_id():
"""
The user's ID is stored on a file under the global config directory.
The file should contain a JSON with a "user_id" key:
{"user_id": "16fd2706-8baf-433b-82eb-8c7fada847da"}
IDs are generated randomly with UUID.
"""
config_dir = Config.get_global_config_dir()
fname = os.path.join(config_dir, "user_id")
lockfile = os.path.join(config_dir, "user_id.lock")
# Since the `fname` and `lockfile` are under the global config,
# we need to make sure such directory exist already.
makedirs(config_dir, exist_ok=True)
try:
with Lock(lockfile):
try:
with open(fname, "r") as fobj:
user_id = json.load(fobj)["user_id"]
except (FileNotFoundError, ValueError, KeyError):
user_id = str(uuid.uuid4())
with open(fname, "w") as fobj:
json.dump({"user_id": user_id}, fobj)
return user_id
except LockError:
logger.debug(
"Failed to acquire '{lockfile}'".format(lockfile=lockfile))
def test_get_to_dir(tmp_dir, erepo_dir, dname):
with erepo_dir.chdir():
erepo_dir.dvc_gen("file", "contents", commit="create file")
makedirs(dname, exist_ok=True)
Repo.get(fspath(erepo_dir), "file", dname)
assert (tmp_dir / dname).is_dir()
assert (tmp_dir / dname / "file").read_text() == "contents"
def _upload(
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
):
makedirs(to_info.parent, exist_ok=True)
tmp_file = tmp_fname(to_info)
copyfile(
from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
)
os.rename(tmp_file, fspath_py35(to_info))
def test_makedirs(repo_dir):
path = os.path.join(repo_dir.root_dir, "directory")
path_info = PathInfo(
os.path.join(repo_dir.root_dir, "another", "directory"))
makedirs(path)
assert os.path.isdir(path)
makedirs(path_info)
assert os.path.isdir(path_info.fspath)
def test_get_to_dir(dname, erepo):
src = erepo.FOO
makedirs(dname, exist_ok=True)
Repo.get(erepo.root_dir, src, dname)
dst = os.path.join(dname, os.path.basename(src))
assert os.path.isdir(dname)
assert filecmp.cmp(erepo.FOO, dst, shallow=False)
def test_makedirs_permissions(tmp_dir):
dir_mode = 0o755
intermediate_dir = "тестовая-директория"
test_dir = os.path.join(intermediate_dir, "data")
assert not os.path.exists(intermediate_dir)
utils.makedirs(test_dir, mode=dir_mode)
assert stat.S_IMODE(os.stat(test_dir).st_mode) == dir_mode
assert stat.S_IMODE(os.stat(intermediate_dir).st_mode) == dir_mode
def test_get_url_to_dir(dname, repo_dir):
src = repo_dir.DATA
makedirs(dname, exist_ok=True)
Repo.get_url(src, dname)
dst = os.path.join(dname, os.path.basename(src))
assert os.path.isdir(dname)
assert filecmp.cmp(repo_dir.DATA, dst, shallow=False)
def _gen(self, struct, prefix=None):
for name, contents in struct.items():
path = (prefix or self) / name
if isinstance(contents, dict):
self._gen(contents, prefix=path)
else:
makedirs(path.parent, exist_ok=True)
if is_py2 and isinstance(contents, str):
path.write_bytes(contents)
else:
path.write_text(contents)
def test_import_url_to_dir(dname, repo_dir, dvc_repo):
src = repo_dir.DATA
makedirs(dname, exist_ok=True)
stage = dvc_repo.imp_url(src, dname)
dst = os.path.join(dname, os.path.basename(src))
assert stage.outs[0].fspath == os.path.abspath(dst)
assert os.path.isdir(dname)
assert filecmp.cmp(repo_dir.DATA, dst, shallow=False)
def test_import_to_dir(dname, dvc_repo, erepo):
src = erepo.FOO
makedirs(dname, exist_ok=True)
stage = dvc_repo.imp(erepo.root_dir, src, dname)
dst = os.path.join(dname, os.path.basename(src))
assert stage.outs[0].fspath == os.path.abspath(dst)
assert os.path.isdir(dname)
assert filecmp.cmp(erepo.FOO, dst, shallow=False)
def test_import_to_dir(dname, tmp_dir, dvc, erepo_dir):
makedirs(dname, exist_ok=True)
with erepo_dir.chdir():
erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
stage = dvc.imp(fspath(erepo_dir), "foo", dname)
dst = os.path.join(dname, "foo")
assert stage.outs[0].fspath == os.path.abspath(dst)
assert os.path.isdir(dname)
assert (tmp_dir / dst).read_text() == "foo content"
def download(
self,
from_info,
to_info,
name=None,
no_progress_bar=False,
file_mode=None,
dir_mode=None,
):
if not hasattr(self, "_download"):
raise RemoteActionNotImplemented("download", self.scheme)
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme == self.scheme != "local":
self.copy(from_info, to_info)
return 0
if to_info.scheme != "local":
raise NotImplementedError
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
name = name or to_info.name
if not no_progress_bar:
# real progress is not always available,
# lets at least show start and finish
progress.update_target(name, 0, None)
makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
tmp_file = tmp_fname(to_info)
try:
self._download(from_info,
tmp_file,
name=name,
no_progress_bar=no_progress_bar)
except Exception:
msg = "failed to download '{}' to '{}'"
logger.exception(msg.format(from_info, to_info))
return 1 # 1 fail
move(tmp_file, to_info, mode=file_mode)
if not no_progress_bar:
progress.finish_target(name)
return 0
def __init__(self, root_dir=None):
from dvc.state import State
from dvc.lock import make_lock
from dvc.scm import SCM
from dvc.cache import Cache
from dvc.data_cloud import DataCloud
from dvc.repo.metrics import Metrics
from dvc.scm.tree import WorkingTree
from dvc.repo.tag import Tag
from dvc.utils import makedirs
root_dir = self.find_root(root_dir)
self.root_dir = os.path.abspath(os.path.realpath(root_dir))
self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
self.config = Config(self.dvc_dir)
self.scm = SCM(self.root_dir)
self.tree = CleanTree(WorkingTree(self.root_dir))
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
makedirs(self.tmp_dir, exist_ok=True)
hardlink_lock = self.config.config["core"].get("hardlink_lock", False)
self.lock = make_lock(
os.path.join(self.dvc_dir, "lock"),
tmp_dir=os.path.join(self.dvc_dir, "tmp"),
hardlink_lock=hardlink_lock,
friendly=True,
)
# NOTE: storing state and link_state in the repository itself to avoid
# any possible state corruption in 'shared cache dir' scenario.
self.state = State(self, self.config.config)
core = self.config.config[Config.SECTION_CORE]
level = core.get(Config.SECTION_CORE_LOGLEVEL)
if level:
logger.setLevel(level.upper())
self.cache = Cache(self)
self.cloud = DataCloud(self)
self.metrics = Metrics(self)
self.tag = Tag(self)
self._ignore()
def _gen(self, struct, prefix=None):
for name, contents in struct.items():
path = (prefix or self) / name
if isinstance(contents, dict):
if not contents:
makedirs(path, exist_ok=True)
else:
self._gen(contents, prefix=path)
else:
makedirs(path.parent, exist_ok=True)
if isinstance(contents, bytes):
path.write_bytes(contents)
else:
path.write_text(contents, encoding="utf-8")
def download(
self,
from_info,
to_info,
name=None,
no_progress_bar=False,
file_mode=None,
dir_mode=None,
):
if not hasattr(self, "_download"):
raise RemoteActionNotImplemented("download", self.scheme)
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme == self.scheme != "local":
self.copy(from_info, to_info)
return 0
if to_info.scheme != "local":
raise NotImplementedError
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
name = name or to_info.name
makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
tmp_file = tmp_fname(to_info)
try:
self._download(from_info,
tmp_file,
name=name,
no_progress_bar=no_progress_bar)
except Exception:
msg = "failed to download '{}' to '{}'"
logger.exception(msg.format(from_info, to_info))
return 1 # 1 fail
move(tmp_file, to_info, mode=file_mode)
return 0
def _download_file(self, from_info, to_info, name, no_progress_bar,
file_mode, dir_mode):
makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
name = name or to_info.name
tmp_file = tmp_fname(to_info)
try:
self._download(from_info,
tmp_file,
name=name,
no_progress_bar=no_progress_bar)
except Exception as e:
return self._handle_transfer_exception(from_info, to_info, e,
"download")
move(tmp_file, to_info, mode=file_mode)
return 0
def _download_file(
self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
):
makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
name = name or to_info.name
tmp_file = tmp_fname(to_info)
try:
self._download(
from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
)
except Exception:
msg = "failed to download '{}' to '{}'"
logger.exception(msg.format(from_info, to_info))
return 1 # 1 fail
move(tmp_file, to_info, mode=file_mode)
return 0
def makedirs(self, path_info):
makedirs(path_info, exist_ok=True, mode=self._dir_mode)