def _collect_specific_target( loader: "StageLoad", target: str, with_deps: bool, recursive: bool, accept_group: bool, ) -> Tuple[StageIter, "OptStr", "OptStr"]: from dvc.dvcfile import is_valid_filename # Optimization: do not collect the graph for a specific target file, name = parse_target(target) # if the target has a file, we can load directly from it. if not file: # but, if there's no file, parsing is ambiguous as it can be a # stage name in `dvc.yaml` file or an output. We prioritize # `dvc.yaml` stage name here. If it exists, then we move on. # else, we assume it's a output name in the `collect_granular()` below msg = "Checking if stage '%s' is in '%s'" logger.debug(msg, target, PIPELINE_FILE) if not (recursive and loader.tree.isdir(target)): stages = _maybe_collect_from_dvc_yaml( loader, target, with_deps, accept_group=accept_group, ) if stages: return stages, file, name elif not with_deps and is_valid_filename(file): stages = loader.load_all(file, name, accept_group=accept_group) return stages, file, name return [], file, name
def _build_graph(self, target, commands=False, outs=False): import networkx from dvc import dvcfile from dvc.repo.graph import get_pipeline from dvc.utils import parse_target path, name = parse_target(target) target_stage = dvcfile.Dvcfile(self.repo, path).stages[name] G = get_pipeline(self.repo.pipelines, target_stage) nodes = set() for stage in networkx.dfs_preorder_nodes(G, target_stage): if commands: if stage.cmd is None: continue nodes.add(stage.cmd) elif not outs: nodes.add(stage.addressing) edges = [] for from_stage, to_stage in networkx.edge_dfs(G, target_stage): if commands: if to_stage.cmd is None: continue edges.append((from_stage.cmd, to_stage.cmd)) elif not outs: edges.append((from_stage.addressing, to_stage.addressing)) if outs: nodes, edges = self._build_output_graph(G, target_stage) return list(nodes), edges, networkx.is_tree(G)
def get_target(self, target: str) -> "Stage": """ Returns a stage from the provided target. (see load_one method for further details) """ path, name = parse_target(target) return self.load_one(path=path, name=name)
def _set(repo, target, frozen): from dvc.utils import parse_target path, name = parse_target(target) stage = repo.get_stage(path, name) stage.frozen = frozen stage.dvcfile.dump(stage, update_pipeline=True) return stage
def lock(self, target, unlock=False): from dvc.utils import parse_target path, name = parse_target(target) stage = self.get_stage(path, name) stage.locked = False if unlock else True stage.dvcfile.dump(stage, update_pipeline=True) return stage
def lock(self, target, unlock=False): from .. import dvcfile from dvc.utils import parse_target path, name = parse_target(target) dvcfile = dvcfile.Dvcfile(self, path) stage = dvcfile.stages[name] stage.locked = False if unlock else True dvcfile.dump(stage, update_pipeline=True) return stage
def from_target( self, target: str, accept_group: bool = False, glob: bool = False, ) -> StageList: """ Returns a list of stage from the provided target. (see load method below for further details) """ path, name = parse_target(target, isa_glob=glob) return self.load_all( path=path, name=name, accept_group=accept_group, glob=glob, )
def reproduce( self, target=None, recursive=False, pipeline=False, all_pipelines=False, **kwargs ): from dvc.utils import parse_target assert target is None or isinstance(target, str) if not target and not all_pipelines: raise InvalidArgumentError( "Neither `target` nor `--all-pipelines` are specified." ) experiment = kwargs.pop("experiment", False) if experiment and self.experiments: try: return self.experiments.new( target=target, recursive=recursive, all_pipelines=all_pipelines, **kwargs ) except UnchangedExperimentError: # If experiment contains no changes, just run regular repro pass interactive = kwargs.get("interactive", False) if not interactive: kwargs["interactive"] = self.config["core"].get("interactive", False) active_graph = _get_active_graph(self.graph) active_pipelines = get_pipelines(active_graph) path, name = parse_target(target) if pipeline or all_pipelines: if all_pipelines: pipelines = active_pipelines else: stage = self.get_stage(path, name) pipelines = [get_pipeline(active_pipelines, stage)] targets = [] for pipeline in pipelines: for stage in pipeline: if pipeline.in_degree(stage) == 0: targets.append(stage) else: targets = self.collect(target, recursive=recursive, graph=active_graph) return _reproduce_stages(active_graph, targets, **kwargs)
def _show(self, target, commands, outs, locked): import networkx from dvc.utils import parse_target path, name = parse_target(target) stage = self.repo.get_stage(path, name) G = self.repo.graph stages = networkx.dfs_postorder_nodes(G, stage) if locked: stages = [s for s in stages if s.locked] for stage in stages: if commands: if stage.cmd is None: continue logger.info(stage.cmd) elif outs: for out in stage.outs: logger.info(str(out)) else: logger.info(stage.addressing)
def reproduce(self, target=None, recursive=False, pipeline=False, all_pipelines=False, **kwargs): from ..dvcfile import Dvcfile from dvc.utils import parse_target if not target and not all_pipelines: raise InvalidArgumentError( "Neither `target` nor `--all-pipelines` are specified.") interactive = kwargs.get("interactive", False) if not interactive: kwargs["interactive"] = self.config["core"].get("interactive", False) active_graph = _get_active_graph(self.graph) active_pipelines = get_pipelines(active_graph) path, name = parse_target(target) if pipeline or all_pipelines: if all_pipelines: pipelines = active_pipelines else: dvcfile = Dvcfile(self, path) stage = dvcfile.stages[name] pipelines = [get_pipeline(active_pipelines, stage)] targets = [] for pipeline in pipelines: for stage in pipeline: if pipeline.in_degree(stage) == 0: targets.append(stage) else: targets = self.collect(target, recursive=recursive, graph=active_graph) return _reproduce_stages(active_graph, targets, **kwargs)
def test_hint_on_lockfile(): with pytest.raises(Exception) as exc: assert parse_target("pipelines.lock:name@v223") assert "pipelines.yaml:name@v223" in str(exc.value)
def test_parse_target(inp, out, default): assert parse_target(inp, default) == out
def test_hint_on_lockfile(): with pytest.raises(Exception) as exc: assert parse_target("dvc.lock:name") assert "dvc.yaml:name" in str(exc.value)