def temporary_windows_drive(tmp_path_factory): import string import win32api from ctypes import windll from win32con import DDD_REMOVE_DEFINITION drives = [ s[0].upper() for s in win32api.GetLogicalDriveStrings().split("\000") if len(s) > 0 ] new_drive_name = [ letter for letter in string.ascii_uppercase if letter not in drives ][0] new_drive = "{}:".format(new_drive_name) target_path = tmp_path_factory.mktemp("tmp_windows_drive") set_up_result = windll.kernel32.DefineDosDeviceW(0, new_drive, fspath(target_path)) if set_up_result == 0: raise RuntimeError("Failed to mount windows drive!") # NOTE: new_drive has form of `A:` and joining it with some relative # path might result in non-existing path (A:path\\to) yield os.path.join(new_drive, os.sep) tear_down_result = windll.kernel32.DefineDosDeviceW( DDD_REMOVE_DEFINITION, new_drive, fspath(target_path)) if tear_down_result == 0: raise RuntimeError("Could not unmount windows drive!")
def relpath(path, start=os.curdir): path = fspath(path) start = os.path.abspath(fspath(start)) # Windows path on different drive than curdir doesn't have relpath if os.name == "nt" and not os.path.commonprefix( [start, os.path.abspath(path)]): return path return os.path.relpath(path, start)
def get(self, path_info): """Gets the checksum for the specified path info. Checksum will be retrieved from the state database if available. Args: path_info (dict): path info to get the checksum for. Returns: str or None: checksum for the specified path info or None if it doesn't exist in the state database. """ assert path_info.scheme == "local" path = fspath(path_info) if not os.path.exists(path): return None actual_mtime, actual_size = get_mtime_and_size(path) actual_inode = get_inode(path) existing_record = self.get_state_record_for_inode(actual_inode) if not existing_record: return None mtime, size, checksum, _ = existing_record if self._file_metadata_changed(actual_mtime, mtime, actual_size, size): return None self._update_state_record_timestamp_for_inode(actual_inode) return checksum
def test_rwlock_reentrant(tmp_path): path = fspath(tmp_path) foo = PathInfo("foo") with rwlock(path, "cmd1", [], [foo]): with rwlock(path, "cmd1", [], [foo]): pass with _edit_rwlock(path) as lock: assert lock == { "read": {}, "write": { "foo": { "cmd": "cmd1", "pid": os.getpid() } }, } with rwlock(path, "cmd", [foo], []): with rwlock(path, "cmd", [foo], []): pass with _edit_rwlock(path) as lock: assert lock == { "read": { "foo": [{ "cmd": "cmd", "pid": os.getpid() }] }, "write": {}, }
def inode(path): path = fspath(path) if System.is_unix(): import ctypes inode = os.lstat(path).st_ino # NOTE: See https://bugs.python.org/issue29619 and # https://stackoverflow.com/questions/34643289/ # pythons-os-stat-is-returning-wrong-inode-value inode = ctypes.c_ulong(inode).value else: # getdirinfo from ntfsutils works on both files and dirs info = System._getdirinfo(path) inode = abs( hash( ( info.dwVolumeSerialNumber, info.nFileIndexHigh, info.nFileIndexLow, ) ) ) assert inode >= 0 assert inode < 2 ** 64 return inode
def dvc(tmp_dir, request): from dvc.repo import Repo if "scm" in request.fixturenames: if not hasattr(tmp_dir, "scm"): _git_init() dvc = Repo.init(fspath(tmp_dir)) dvc.scm.commit("init dvc") else: dvc = Repo.init(fspath(tmp_dir), no_scm=True) try: tmp_dir.dvc = dvc yield dvc finally: dvc.close()
def is_hardlink(path): path = fspath(path) if System.is_unix(): return os.stat(path).st_nlink > 1 info = System._getdirinfo(path) return info.nNumberOfLinks > 1
def reflink(source, link_name): import platform from dvc.exceptions import DvcException source, link_name = fspath(source), fspath(link_name) system = platform.system() try: if system == "Windows": ret = System._reflink_windows(source, link_name) elif system == "Darwin": ret = System._reflink_darwin(source, link_name) elif system == "Linux": ret = System._reflink_linux(source, link_name) else: ret = -1 except IOError: ret = -1 if ret != 0: raise DvcException("reflink is not supported")
def test_broken_rwlock(tmp_path): dir_path = fspath(tmp_path) path = tmp_path / "rwlock" path.write_text('{"broken": "format"}', encoding="utf-8") with pytest.raises(RWLockFileFormatError): with _edit_rwlock(dir_path): pass path.write_text("{broken json", encoding="utf-8") with pytest.raises(RWLockFileCorruptedError): with _edit_rwlock(dir_path): pass
def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname): tmp_dir.gen({"dir": {"subdir": {}}}) top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE) top_ignore_file.write_text(os.path.basename(dname)) ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE ignore_file.write_text("foo") assert dvc.dvcignore.ignores == { DvcIgnoreDirs([".git", ".hg", ".dvc"]), DvcIgnorePatterns(fspath(top_ignore_file), WorkingTree(dvc.root_dir)), }
def is_symlink(path): path = fspath(path) if System.is_unix(): return os.path.islink(path) # https://docs.microsoft.com/en-us/windows/desktop/fileio/ # file-attribute-constants from winnt import FILE_ATTRIBUTE_REPARSE_POINT if os.path.lexists(path): info = System._getdirinfo(path) return info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT return False
def test_get_from_non_dvc_master(erepo, tmp_path, monkeypatch, caplog): monkeypatch.chdir(fspath(tmp_path)) erepo.dvc.scm.repo.index.remove([".dvc"], r=True) erepo.dvc.scm.commit("remove .dvc") caplog.clear() imported_file = "foo_imported" with caplog.at_level(logging.INFO, logger="dvc"): Repo.get(erepo._root_dir, erepo.FOO, out=imported_file, rev="branch") assert caplog.text == "" assert filecmp.cmp(os.path.join(erepo._root_dir, erepo.FOO), imported_file, shallow=False)
def hardlink(source, link_name): import ctypes from dvc.exceptions import DvcException source, link_name = fspath(source), fspath(link_name) if System.is_unix(): try: os.link(source, link_name) return except Exception as exc: raise DvcException("link", cause=exc) CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW CreateHardLink.argtypes = [ ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p, ] CreateHardLink.restype = ctypes.wintypes.BOOL res = CreateHardLink(link_name, source, None) if res == 0: raise DvcException("CreateHardLinkW", cause=ctypes.WinError())
def symlink(source, link_name): import ctypes from dvc.exceptions import DvcException source, link_name = fspath(source), fspath(link_name) if System.is_unix(): try: os.symlink(source, link_name) return except Exception as exc: msg = "failed to symlink '{}' -> '{}': {}" raise DvcException(msg.format(source, link_name, str(exc))) flags = 0 if source is not None and os.path.isdir(source): flags = 1 func = ctypes.windll.kernel32.CreateSymbolicLinkW func.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32) func.restype = ctypes.c_ubyte if func(link_name, source, flags) == 0: raise DvcException("CreateSymbolicLinkW", cause=ctypes.WinError())
def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker): storage = tmp_path_factory.mktemp("test_remote_base") remote_config = RemoteConfig(dvc.config) remote_config.add("local_remote", fspath(storage), default=True) tmp_dir.dvc_gen({"file": "file content"}) mocker.patch.object( RemoteLOCAL, "_upload", side_effect=OSError(errno.EMFILE, "Too many open files"), ) with pytest.raises(OSError) as e: dvc.push() assert e.errno == errno.EMFILE
def scm(tmp_dir, request): # Use dvc.scm if available if "dvc" in request.fixturenames: dvc = request.getfixturevalue("dvc") tmp_dir.scm = dvc.scm yield dvc.scm else: from dvc.scm.git import Git _git_init() try: scm = tmp_dir.scm = Git(fspath(tmp_dir)) yield scm finally: scm.close()
def test_rwlock(tmp_path): path = fspath(tmp_path) foo = PathInfo("foo") with rwlock(path, "cmd1", [foo], []): with pytest.raises(LockError): with rwlock(path, "cmd2", [], [foo]): pass with rwlock(path, "cmd1", [], [foo]): with pytest.raises(LockError): with rwlock(path, "cmd2", [foo], []): pass with rwlock(path, "cmd1", [], [foo]): with pytest.raises(LockError): with rwlock(path, "cmd2", [], [foo]): pass
def test_destroy(tmp_dir, dvc): dvc.config.set("cache", "type", "symlink") dvc.cache = Cache(dvc) tmp_dir.dvc_gen("file", "text") tmp_dir.dvc_gen({"dir": {"file": "lorem", "subdir/file": "ipsum"}}) dvc.destroy() # Remove all the files related to DVC for path in [".dvc", "file.dvc", "dir.dvc"]: assert not (tmp_dir / path).exists() # Leave the rest of the files for path in ["file", "dir/file", "dir/subdir/file"]: assert (tmp_dir / path).is_file() # Make sure that data was unprotected after `destroy` for path in ["file", "dir", "dir/file", "dir/subdir", "dir/subdir/file"]: assert not System.is_symlink(fspath(tmp_dir / path))
def save_link(self, path_info): """Adds the specified path to the list of links created by dvc. This list is later used on `dvc checkout` to cleanup old links. Args: path_info (dict): path info to add to the list of links. """ assert path_info.scheme == "local" path = fspath(path_info) if not os.path.exists(path): return mtime, _ = get_mtime_and_size(path) inode = get_inode(path) relpath = os.path.relpath(path, self.root_dir) cmd = ("REPLACE INTO {}(path, inode, mtime) " 'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE, relpath, self._to_sqlite(inode), mtime)) self._execute(cmd)
def save(self, path_info, checksum): """Save checksum for the specified path info. Args: path_info (dict): path_info to save checksum for. checksum (str): checksum to save. """ assert path_info.scheme == "local" assert checksum is not None path = fspath(path_info) assert os.path.exists(path) actual_mtime, actual_size = get_mtime_and_size(path) actual_inode = get_inode(path) existing_record = self.get_state_record_for_inode(actual_inode) if not existing_record: self._insert_new_state_record(actual_inode, actual_mtime, actual_size, checksum) return self._update_state_for_path_changed(actual_inode, actual_mtime, actual_size, checksum)
def test_daemon_analytics(mock_send, tmp_path): report = fspath(tmp_path) assert 0 == main(["daemon", "analytics", report]) mock_send.assert_called_with(report)
def tmp_fname(fname): """ Temporary name for a partial download """ from shortuuid import uuid return fspath(fname) + "." + str(uuid()) + ".tmp"
def stages(): return set(stage.relpath for stage in Repo(fspath(tmp_dir)).stages)
def copy(src, dest): src, dest = fspath(src), fspath(dest) return shutil.copyfile(src, dest)