def temporary_windows_drive(tmp_path_factory):
import string
import win32api
from ctypes import windll
from win32con import DDD_REMOVE_DEFINITION
drives = [
s[0].upper() for s in win32api.GetLogicalDriveStrings().split("\000")
if len(s) > 0
]
new_drive_name = [
letter for letter in string.ascii_uppercase if letter not in drives
][0]
new_drive = "{}:".format(new_drive_name)
target_path = tmp_path_factory.mktemp("tmp_windows_drive")
set_up_result = windll.kernel32.DefineDosDeviceW(0, new_drive,
fspath(target_path))
if set_up_result == 0:
raise RuntimeError("Failed to mount windows drive!")
# NOTE: new_drive has form of `A:` and joining it with some relative
# path might result in non-existing path (A:path\\to)
yield os.path.join(new_drive, os.sep)
tear_down_result = windll.kernel32.DefineDosDeviceW(
DDD_REMOVE_DEFINITION, new_drive, fspath(target_path))
if tear_down_result == 0:
raise RuntimeError("Could not unmount windows drive!")
def relpath(path, start=os.curdir):
path = fspath(path)
start = os.path.abspath(fspath(start))
# Windows path on different drive than curdir doesn't have relpath
if os.name == "nt" and not os.path.commonprefix(
[start, os.path.abspath(path)]):
return path
return os.path.relpath(path, start)
def get(self, path_info):
"""Gets the checksum for the specified path info. Checksum will be
retrieved from the state database if available.
Args:
path_info (dict): path info to get the checksum for.
Returns:
str or None: checksum for the specified path info or None if it
doesn't exist in the state database.
"""
assert path_info.scheme == "local"
path = fspath(path_info)
if not os.path.exists(path):
return None
actual_mtime, actual_size = get_mtime_and_size(path)
actual_inode = get_inode(path)
existing_record = self.get_state_record_for_inode(actual_inode)
if not existing_record:
return None
mtime, size, checksum, _ = existing_record
if self._file_metadata_changed(actual_mtime, mtime, actual_size, size):
return None
self._update_state_record_timestamp_for_inode(actual_inode)
return checksum
def test_rwlock_reentrant(tmp_path):
path = fspath(tmp_path)
foo = PathInfo("foo")
with rwlock(path, "cmd1", [], [foo]):
with rwlock(path, "cmd1", [], [foo]):
pass
with _edit_rwlock(path) as lock:
assert lock == {
"read": {},
"write": {
"foo": {
"cmd": "cmd1",
"pid": os.getpid()
}
},
}
with rwlock(path, "cmd", [foo], []):
with rwlock(path, "cmd", [foo], []):
pass
with _edit_rwlock(path) as lock:
assert lock == {
"read": {
"foo": [{
"cmd": "cmd",
"pid": os.getpid()
}]
},
"write": {},
}
def inode(path):
path = fspath(path)
if System.is_unix():
import ctypes
inode = os.lstat(path).st_ino
# NOTE: See https://bugs.python.org/issue29619 and
# https://stackoverflow.com/questions/34643289/
# pythons-os-stat-is-returning-wrong-inode-value
inode = ctypes.c_ulong(inode).value
else:
# getdirinfo from ntfsutils works on both files and dirs
info = System._getdirinfo(path)
inode = abs(
hash(
(
info.dwVolumeSerialNumber,
info.nFileIndexHigh,
info.nFileIndexLow,
)
)
)
assert inode >= 0
assert inode < 2 ** 64
return inode
def dvc(tmp_dir, request):
from dvc.repo import Repo
if "scm" in request.fixturenames:
if not hasattr(tmp_dir, "scm"):
_git_init()
dvc = Repo.init(fspath(tmp_dir))
dvc.scm.commit("init dvc")
else:
dvc = Repo.init(fspath(tmp_dir), no_scm=True)
try:
tmp_dir.dvc = dvc
yield dvc
finally:
dvc.close()
def is_hardlink(path):
path = fspath(path)
if System.is_unix():
return os.stat(path).st_nlink > 1
info = System._getdirinfo(path)
return info.nNumberOfLinks > 1
def reflink(source, link_name):
import platform
from dvc.exceptions import DvcException
source, link_name = fspath(source), fspath(link_name)
system = platform.system()
try:
if system == "Windows":
ret = System._reflink_windows(source, link_name)
elif system == "Darwin":
ret = System._reflink_darwin(source, link_name)
elif system == "Linux":
ret = System._reflink_linux(source, link_name)
else:
ret = -1
except IOError:
ret = -1
if ret != 0:
raise DvcException("reflink is not supported")
def test_broken_rwlock(tmp_path):
dir_path = fspath(tmp_path)
path = tmp_path / "rwlock"
path.write_text('{"broken": "format"}', encoding="utf-8")
with pytest.raises(RWLockFileFormatError):
with _edit_rwlock(dir_path):
pass
path.write_text("{broken json", encoding="utf-8")
with pytest.raises(RWLockFileCorruptedError):
with _edit_rwlock(dir_path):
pass
def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
tmp_dir.gen({"dir": {"subdir": {}}})
top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE)
top_ignore_file.write_text(os.path.basename(dname))
ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE
ignore_file.write_text("foo")
assert dvc.dvcignore.ignores == {
DvcIgnoreDirs([".git", ".hg", ".dvc"]),
DvcIgnorePatterns(fspath(top_ignore_file), WorkingTree(dvc.root_dir)),
}
def is_symlink(path):
path = fspath(path)
if System.is_unix():
return os.path.islink(path)
# https://docs.microsoft.com/en-us/windows/desktop/fileio/
# file-attribute-constants
from winnt import FILE_ATTRIBUTE_REPARSE_POINT
if os.path.lexists(path):
info = System._getdirinfo(path)
return info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT
return False
def test_get_from_non_dvc_master(erepo, tmp_path, monkeypatch, caplog):
monkeypatch.chdir(fspath(tmp_path))
erepo.dvc.scm.repo.index.remove([".dvc"], r=True)
erepo.dvc.scm.commit("remove .dvc")
caplog.clear()
imported_file = "foo_imported"
with caplog.at_level(logging.INFO, logger="dvc"):
Repo.get(erepo._root_dir, erepo.FOO, out=imported_file, rev="branch")
assert caplog.text == ""
assert filecmp.cmp(os.path.join(erepo._root_dir, erepo.FOO),
imported_file,
shallow=False)
def hardlink(source, link_name):
import ctypes
from dvc.exceptions import DvcException
source, link_name = fspath(source), fspath(link_name)
if System.is_unix():
try:
os.link(source, link_name)
return
except Exception as exc:
raise DvcException("link", cause=exc)
CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
CreateHardLink.argtypes = [
ctypes.c_wchar_p,
ctypes.c_wchar_p,
ctypes.c_void_p,
]
CreateHardLink.restype = ctypes.wintypes.BOOL
res = CreateHardLink(link_name, source, None)
if res == 0:
raise DvcException("CreateHardLinkW", cause=ctypes.WinError())
def symlink(source, link_name):
import ctypes
from dvc.exceptions import DvcException
source, link_name = fspath(source), fspath(link_name)
if System.is_unix():
try:
os.symlink(source, link_name)
return
except Exception as exc:
msg = "failed to symlink '{}' -> '{}': {}"
raise DvcException(msg.format(source, link_name, str(exc)))
flags = 0
if source is not None and os.path.isdir(source):
flags = 1
func = ctypes.windll.kernel32.CreateSymbolicLinkW
func.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
func.restype = ctypes.c_ubyte
if func(link_name, source, flags) == 0:
raise DvcException("CreateSymbolicLinkW", cause=ctypes.WinError())
def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker):
storage = tmp_path_factory.mktemp("test_remote_base")
remote_config = RemoteConfig(dvc.config)
remote_config.add("local_remote", fspath(storage), default=True)
tmp_dir.dvc_gen({"file": "file content"})
mocker.patch.object(
RemoteLOCAL,
"_upload",
side_effect=OSError(errno.EMFILE, "Too many open files"),
)
with pytest.raises(OSError) as e:
dvc.push()
assert e.errno == errno.EMFILE
def scm(tmp_dir, request):
# Use dvc.scm if available
if "dvc" in request.fixturenames:
dvc = request.getfixturevalue("dvc")
tmp_dir.scm = dvc.scm
yield dvc.scm
else:
from dvc.scm.git import Git
_git_init()
try:
scm = tmp_dir.scm = Git(fspath(tmp_dir))
yield scm
finally:
scm.close()
def test_rwlock(tmp_path):
path = fspath(tmp_path)
foo = PathInfo("foo")
with rwlock(path, "cmd1", [foo], []):
with pytest.raises(LockError):
with rwlock(path, "cmd2", [], [foo]):
pass
with rwlock(path, "cmd1", [], [foo]):
with pytest.raises(LockError):
with rwlock(path, "cmd2", [foo], []):
pass
with rwlock(path, "cmd1", [], [foo]):
with pytest.raises(LockError):
with rwlock(path, "cmd2", [], [foo]):
pass
def test_destroy(tmp_dir, dvc):
dvc.config.set("cache", "type", "symlink")
dvc.cache = Cache(dvc)
tmp_dir.dvc_gen("file", "text")
tmp_dir.dvc_gen({"dir": {"file": "lorem", "subdir/file": "ipsum"}})
dvc.destroy()
# Remove all the files related to DVC
for path in [".dvc", "file.dvc", "dir.dvc"]:
assert not (tmp_dir / path).exists()
# Leave the rest of the files
for path in ["file", "dir/file", "dir/subdir/file"]:
assert (tmp_dir / path).is_file()
# Make sure that data was unprotected after `destroy`
for path in ["file", "dir", "dir/file", "dir/subdir", "dir/subdir/file"]:
assert not System.is_symlink(fspath(tmp_dir / path))
def save_link(self, path_info):
"""Adds the specified path to the list of links created by dvc. This
list is later used on `dvc checkout` to cleanup old links.
Args:
path_info (dict): path info to add to the list of links.
"""
assert path_info.scheme == "local"
path = fspath(path_info)
if not os.path.exists(path):
return
mtime, _ = get_mtime_and_size(path)
inode = get_inode(path)
relpath = os.path.relpath(path, self.root_dir)
cmd = ("REPLACE INTO {}(path, inode, mtime) "
'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE, relpath,
self._to_sqlite(inode), mtime))
self._execute(cmd)
def save(self, path_info, checksum):
"""Save checksum for the specified path info.
Args:
path_info (dict): path_info to save checksum for.
checksum (str): checksum to save.
"""
assert path_info.scheme == "local"
assert checksum is not None
path = fspath(path_info)
assert os.path.exists(path)
actual_mtime, actual_size = get_mtime_and_size(path)
actual_inode = get_inode(path)
existing_record = self.get_state_record_for_inode(actual_inode)
if not existing_record:
self._insert_new_state_record(actual_inode, actual_mtime,
actual_size, checksum)
return
self._update_state_for_path_changed(actual_inode, actual_mtime,
actual_size, checksum)
def test_daemon_analytics(mock_send, tmp_path):
report = fspath(tmp_path)
assert 0 == main(["daemon", "analytics", report])
mock_send.assert_called_with(report)
def tmp_fname(fname):
""" Temporary name for a partial download """
from shortuuid import uuid
return fspath(fname) + "." + str(uuid()) + ".tmp"
def stages():
return set(stage.relpath for stage in Repo(fspath(tmp_dir)).stages)
def copy(src, dest):
src, dest = fspath(src), fspath(dest)
return shutil.copyfile(src, dest)
