def collect(self, targets=None, revs=None):
"""Collects all props and data for plots.
Returns a structure like:
{rev: {plots.csv: {
props: {x: ..., "header": ..., ...},
data: "...data as a string...",
}}}
Data parsing is postponed, since it's affected by props.
"""
targets = [targets] if isinstance(targets, str) else targets or []
data = {}
for rev in self.repo.brancher(revs=revs):
# .brancher() adds unwanted workspace
if revs is not None and rev not in revs:
continue
rev = rev or "workspace"
tree = RepoTree(self.repo)
plots = _collect_plots(self.repo, targets, rev)
for path_info, props in plots.items():
datafile = relpath(path_info, self.repo.root_dir)
if rev not in data:
data[rev] = {}
data[rev].update({datafile: {"props": props}})
# Load data from git or dvc cache
try:
with tree.open(path_info) as fd:
data[rev][datafile]["data"] = fd.read()
except FileNotFoundError:
# This might happen simply because cache is absent
pass
return data
def test_open_dirty_no_hash(tmp_dir, dvc):
tmp_dir.gen("file", "file")
(tmp_dir / "file.dvc").write_text("outs:\n- path: file\n")
tree = RepoTree(dvc)
with tree.open("file", "r") as fobj:
assert fobj.read() == "file"
def test_open_dirty_hash(tmp_dir, dvc):
tmp_dir.dvc_gen("file", "file")
(tmp_dir / "file").write_text("something")
tree = RepoTree(dvc)
with tree.open("file", "r") as fobj:
assert fobj.read() == "something"
def test_open(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
dvc.add("foo")
(tmp_dir / "foo").unlink()
tree = RepoTree(dvc)
with tree.open("foo", "r") as fobj:
assert fobj.read() == "foo"
def open_by_relpath(self, path, mode="r", encoding=None, **kwargs):
"""Opens a specified resource as a file object."""
tree = RepoTree(self)
try:
with tree.open(path, mode=mode, encoding=encoding,
**kwargs) as fobj:
yield fobj
except FileNotFoundError:
raise PathMissingError(path, self.url)
def _load_from_revision(repo, datafile, revision):
from dvc.repo.tree import RepoTree
tree = RepoTree(repo)
try:
with tree.open(datafile) as fobj:
datafile_content = fobj.read()
except (FileNotFoundError, PathMissingError):
raise NoMetricOnRevisionError(datafile, revision)
return plot_data(datafile, revision, datafile_content)
def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
"""Opens a specified resource as a file descriptor"""
tree = RepoTree(self, stream=True)
path = os.path.join(self.root_dir, path)
try:
with tree.open(
os.path.join(self.root_dir, path),
mode=mode,
encoding=encoding,
remote=remote,
) as fobj:
yield fobj
except FileNotFoundError as exc:
raise FileMissingError(path) from exc
except IsADirectoryError as exc:
raise DvcIsADirectoryError from exc
def test_open_in_history(tmp_dir, scm, dvc):
tmp_dir.gen("foo", "foo")
dvc.add("foo")
dvc.scm.add(["foo.dvc", ".gitignore"])
dvc.scm.commit("foo")
tmp_dir.gen("foo", "foofoo")
dvc.add("foo")
dvc.scm.add(["foo.dvc", ".gitignore"])
dvc.scm.commit("foofoo")
for rev in dvc.brancher(revs=["HEAD~1"]):
if rev == "working tree":
continue
tree = RepoTree(dvc)
with tree.open("foo", "r") as fobj:
assert fobj.read() == "foo"
def _read_metrics(repo, metrics, rev):
tree = RepoTree(repo)
res = {}
for metric in metrics:
if not tree.exists(fspath_py35(metric)):
continue
with tree.open(fspath_py35(metric), "r") as fobj:
try:
# NOTE this also supports JSON
val = yaml.safe_load(fobj)
except yaml.YAMLError:
logger.debug(
"failed to read '%s' on '%s'", metric, rev, exc_info=True
)
continue
val = _extract_metrics(val)
if val:
res[str(metric)] = val
return res
