def _collect_specific_target(
loader: "StageLoad",
target: str,
with_deps: bool,
recursive: bool,
accept_group: bool,
) -> Tuple[StageIter, "OptStr", "OptStr"]:
from dvc.dvcfile import is_valid_filename
# Optimization: do not collect the graph for a specific target
file, name = parse_target(target)
# if the target has a file, we can load directly from it.
if not file:
# but, if there's no file, parsing is ambiguous as it can be a
# stage name in `dvc.yaml` file or an output. We prioritize
# `dvc.yaml` stage name here. If it exists, then we move on.
# else, we assume it's a output name in the `collect_granular()` below
msg = "Checking if stage '%s' is in '%s'"
logger.debug(msg, target, PIPELINE_FILE)
if not (recursive and loader.tree.isdir(target)):
stages = _maybe_collect_from_dvc_yaml(
loader,
target,
with_deps,
accept_group=accept_group,
)
if stages:
return stages, file, name
elif not with_deps and is_valid_filename(file):
stages = loader.load_all(file, name, accept_group=accept_group)
return stages, file, name
return [], file, name
def _build_graph(self, target, commands=False, outs=False):
import networkx
from dvc import dvcfile
from dvc.repo.graph import get_pipeline
from dvc.utils import parse_target
path, name = parse_target(target)
target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
G = get_pipeline(self.repo.pipelines, target_stage)
nodes = set()
for stage in networkx.dfs_preorder_nodes(G, target_stage):
if commands:
if stage.cmd is None:
continue
nodes.add(stage.cmd)
elif not outs:
nodes.add(stage.addressing)
edges = []
for from_stage, to_stage in networkx.edge_dfs(G, target_stage):
if commands:
if to_stage.cmd is None:
continue
edges.append((from_stage.cmd, to_stage.cmd))
elif not outs:
edges.append((from_stage.addressing, to_stage.addressing))
if outs:
nodes, edges = self._build_output_graph(G, target_stage)
return list(nodes), edges, networkx.is_tree(G)
def get_target(self, target: str) -> "Stage":
"""
Returns a stage from the provided target.
(see load_one method for further details)
"""
path, name = parse_target(target)
return self.load_one(path=path, name=name)
def _set(repo, target, frozen):
from dvc.utils import parse_target
path, name = parse_target(target)
stage = repo.get_stage(path, name)
stage.frozen = frozen
stage.dvcfile.dump(stage, update_pipeline=True)
return stage
def lock(self, target, unlock=False):
from dvc.utils import parse_target
path, name = parse_target(target)
stage = self.get_stage(path, name)
stage.locked = False if unlock else True
stage.dvcfile.dump(stage, update_pipeline=True)
return stage
def lock(self, target, unlock=False):
from .. import dvcfile
from dvc.utils import parse_target
path, name = parse_target(target)
dvcfile = dvcfile.Dvcfile(self, path)
stage = dvcfile.stages[name]
stage.locked = False if unlock else True
dvcfile.dump(stage, update_pipeline=True)
return stage
def from_target(
self, target: str, accept_group: bool = False, glob: bool = False,
) -> StageList:
"""
Returns a list of stage from the provided target.
(see load method below for further details)
"""
path, name = parse_target(target, isa_glob=glob)
return self.load_all(
path=path, name=name, accept_group=accept_group, glob=glob,
)
def reproduce(
self,
target=None,
recursive=False,
pipeline=False,
all_pipelines=False,
**kwargs
):
from dvc.utils import parse_target
assert target is None or isinstance(target, str)
if not target and not all_pipelines:
raise InvalidArgumentError(
"Neither `target` nor `--all-pipelines` are specified."
)
experiment = kwargs.pop("experiment", False)
if experiment and self.experiments:
try:
return self.experiments.new(
target=target,
recursive=recursive,
all_pipelines=all_pipelines,
**kwargs
)
except UnchangedExperimentError:
# If experiment contains no changes, just run regular repro
pass
interactive = kwargs.get("interactive", False)
if not interactive:
kwargs["interactive"] = self.config["core"].get("interactive", False)
active_graph = _get_active_graph(self.graph)
active_pipelines = get_pipelines(active_graph)
path, name = parse_target(target)
if pipeline or all_pipelines:
if all_pipelines:
pipelines = active_pipelines
else:
stage = self.get_stage(path, name)
pipelines = [get_pipeline(active_pipelines, stage)]
targets = []
for pipeline in pipelines:
for stage in pipeline:
if pipeline.in_degree(stage) == 0:
targets.append(stage)
else:
targets = self.collect(target, recursive=recursive, graph=active_graph)
return _reproduce_stages(active_graph, targets, **kwargs)
def _show(self, target, commands, outs, locked):
import networkx
from dvc.utils import parse_target
path, name = parse_target(target)
stage = self.repo.get_stage(path, name)
G = self.repo.graph
stages = networkx.dfs_postorder_nodes(G, stage)
if locked:
stages = [s for s in stages if s.locked]
for stage in stages:
if commands:
if stage.cmd is None:
continue
logger.info(stage.cmd)
elif outs:
for out in stage.outs:
logger.info(str(out))
else:
logger.info(stage.addressing)
def reproduce(self,
target=None,
recursive=False,
pipeline=False,
all_pipelines=False,
**kwargs):
from ..dvcfile import Dvcfile
from dvc.utils import parse_target
if not target and not all_pipelines:
raise InvalidArgumentError(
"Neither `target` nor `--all-pipelines` are specified.")
interactive = kwargs.get("interactive", False)
if not interactive:
kwargs["interactive"] = self.config["core"].get("interactive", False)
active_graph = _get_active_graph(self.graph)
active_pipelines = get_pipelines(active_graph)
path, name = parse_target(target)
if pipeline or all_pipelines:
if all_pipelines:
pipelines = active_pipelines
else:
dvcfile = Dvcfile(self, path)
stage = dvcfile.stages[name]
pipelines = [get_pipeline(active_pipelines, stage)]
targets = []
for pipeline in pipelines:
for stage in pipeline:
if pipeline.in_degree(stage) == 0:
targets.append(stage)
else:
targets = self.collect(target, recursive=recursive, graph=active_graph)
return _reproduce_stages(active_graph, targets, **kwargs)
def test_hint_on_lockfile():
with pytest.raises(Exception) as exc:
assert parse_target("pipelines.lock:name@v223")
assert "pipelines.yaml:name@v223" in str(exc.value)
def test_parse_target(inp, out, default):
assert parse_target(inp, default) == out
def test_hint_on_lockfile():
with pytest.raises(Exception) as exc:
assert parse_target("dvc.lock:name")
assert "dvc.yaml:name" in str(exc.value)
