def init_interactive( defaults: Dict[str, str], provided: Dict[str, str], validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None, stream: Optional[TextIO] = None, ) -> Dict[str, str]: command_prompts = lremove(provided.keys(), ["cmd"]) dependencies_prompts = lremove(provided.keys(), ["code", "data", "params"]) output_keys = ["models"] if "live" not in provided: output_keys.extend(["metrics", "plots"]) outputs_prompts = lremove(provided.keys(), output_keys) ret: Dict[str, str] = {} if "cmd" in provided: ret["cmd"] = provided["cmd"] for heading, prompts, allow_omission in ( ("", command_prompts, False), ("Enter experiment dependencies.", dependencies_prompts, True), ("Enter experiment outputs.", outputs_prompts, True), ): if prompts and heading: ui.error_write(heading, styled=True) response = _prompts( prompts, defaults=defaults, allow_omission=allow_omission, validator=validator, stream=stream, ) ret.update(compact(response)) if prompts: ui.error_write(styled=True) return ret
def show( self, targets: List[str] = None, revs=None, props=None, recursive=False, onerror=None, ): if onerror is None: onerror = onerror_collect data: Dict[str, Dict] = {} for rev_data in self.collect(targets, revs, recursive, onerror=onerror, props=props): data.update(rev_data) errored = errored_revisions(data) if errored: from dvc.ui import ui ui.error_write( "DVC failed to load some plots for following revisions: " f"'{', '.join(errored)}'.") return data
def show( self, targets: List[str] = None, revs=None, props=None, recursive=False, onerror=None, ): if onerror is None: onerror = onerror_collect result: Dict[str, Dict] = {} for data in self.collect(targets, revs, recursive, onerror=onerror, props=props): assert len(data) == 1 revision_data = first(data.values()) if "data" in revision_data: for path_data in revision_data["data"].values(): result_source = path_data.pop("data_source", None) if result_source: path_data.update(result_source()) result.update(data) errored = errored_revisions(result) if errored: from dvc.ui import ui ui.error_write( "DVC failed to load some plots for following revisions: " f"'{', '.join(errored)}'.") return result
def show(repo, revs=None, targets=None, deps=False, onerror: Callable = None): if onerror is None: onerror = onerror_collect res = {} for branch in repo.brancher(revs=revs): params = error_handler(_gather_params)(repo=repo, rev=branch, targets=targets, deps=deps, onerror=onerror) if params: res[branch] = params # Hide workspace params if they are the same as in the active branch try: active_branch = repo.scm.active_branch() except (SCMError, NoSCMError): # SCMError - detached head # NoSCMError - no repo case pass else: if res.get("workspace") == res.get(active_branch): res.pop("workspace", None) errored = errored_revisions(res) if errored: ui.error_write( "DVC failed to load some parameters for following revisions:" f" '{', '.join(errored)}'.") return res
def __pretty_exc__(self, **kwargs: Any) -> None: from ruamel.yaml.error import MarkedYAMLError exc = self.exc.__cause__ if not isinstance(exc, MarkedYAMLError): raise ValueError("nothing to pretty-print here. :)") source = self.yaml_text.splitlines() def prepare_linecol(mark: "StreamMark") -> str: return f"in line {mark.line + 1}, column {mark.column + 1}" def prepare_message( message: str, mark: "StreamMark" = None ) -> "RichText": cause = ", ".join( [message.capitalize(), prepare_linecol(mark) if mark else ""] ) return _prepare_cause(cause) def prepare_code(mark: "StreamMark") -> "Syntax": line = mark.line + 1 code = "" if line > len(source) else source[line - 1] return _prepare_code_snippets(code, line) lines: List[object] = [] if hasattr(exc, "context"): if exc.context_mark is not None: lines.append( prepare_message(str(exc.context), exc.context_mark) ) if exc.context_mark is not None and ( exc.problem is None or exc.problem_mark is None or exc.context_mark.name != exc.problem_mark.name or exc.context_mark.line != exc.problem_mark.line or exc.context_mark.column != exc.problem_mark.column ): lines.extend([prepare_code(exc.context_mark), ""]) if exc.problem is not None: lines.append( prepare_message(str(exc.problem), exc.problem_mark) ) if exc.problem_mark is not None: lines.append(prepare_code(exc.problem_mark)) if lines: # we should not add a newline after the main message # if there are no other outputs lines.insert(0, "") rel = make_relpath(self.path) rev_msg = f" in revision '{self.rev[:7]}'" if self.rev else "" msg_fmt = f"'{rel}' is invalid{self.hint}{rev_msg}." lines.insert(0, _prepare_message(msg_fmt)) for line in lines: ui.error_write(line, styled=True)
def run(self): from pathlib import Path if self.args.show_vega: if not self.args.targets: logger.error("please specify a target for `--show-vega`") return 1 if len(self.args.targets) > 1: logger.error( "you can only specify one target for `--show-vega`" ) return 1 try: plots_data = self._func( targets=self.args.targets, props=self._props() ) if not plots_data: ui.error_write( "No plots were loaded, " "visualization file will not be created." ) if self.args.show_vega: target = self.args.targets[0] plot_json = find_vega(self.repo, plots_data, target) if plot_json: ui.write(plot_json) return 0 rel: str = self.args.out or "dvc_plots" path: Path = (Path.cwd() / rel).resolve() index_path = render( self.repo, plots_data, path=path, html_template_path=self.args.html_template, ) assert index_path.is_absolute() url = index_path.as_uri() ui.write(url) if self.args.open: import webbrowser opened = webbrowser.open(index_path) if not opened: ui.error_write( "Failed to open. Please try opening it manually." ) return 1 return 0 except DvcException: logger.exception("") return 1
def transform_targets(args): from funcy import count_reps counts = count_reps(ensure_list(args.targets)) dupes = [key for key, count in counts.items() if count > 1] if dupes: msg = ", ".join(f"[b]{key}[/]" for key in dupes) ui.error_write(f"ignoring duplicated targets: {msg}", styled=True) args.targets = list(counts)
def init_interactive( name: str, defaults: Dict[str, str], provided: Dict[str, str], validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None, live: bool = False, stream: Optional[TextIO] = None, ) -> Dict[str, str]: command = provided.pop("cmd", None) primary = lremove(provided.keys(), ["code", "data", "models", "params"]) secondary = lremove(provided.keys(), ["live"] if live else ["metrics", "plots"]) prompts = primary + secondary workspace = {**defaults, **provided} if not live and "live" not in provided: workspace.pop("live", None) for key in ("plots", "metrics"): if live and key not in provided: workspace.pop(key, None) ret: Dict[str, str] = {} if command: ret["cmd"] = command if not prompts and command: return ret ui.error_write( f"This command will guide you to set up a [bright_blue]{name}[/]", "stage in [green]dvc.yaml[/].", f"\nSee [repr.url]{PIPELINE_FILE_LINK}[/].\n", styled=True, ) if not command: ret.update( compact(_prompts(["cmd"], allow_omission=False, stream=stream))) if prompts: ui.error_write(styled=True) if not prompts: return ret ui.error_write( "Enter the paths for dependencies and outputs of the command.", styled=True, ) if workspace: ui.error_write(build_workspace_tree(workspace), styled=True) ui.error_write(styled=True) ret.update( compact(_prompts(prompts, defaults, validator=validator, stream=stream))) return ret
def test_capsys_works(capsys: CaptureFixture[str]): """Sanity check that capsys can capture outputs from a global ui.""" from dvc.ui import ui message = "hello world" ui.write(message) ui.error_write(message) captured = capsys.readouterr() assert captured.out == f"{message}\n" assert captured.err == f"{message}\n"
def warn_link_failures() -> Iterator[List[str]]: link_failures: List[str] = [] try: yield link_failures finally: if link_failures: msg = LINK_FAILURE_MESSAGE.format( CacheLinkError.SUPPORT_LINK, " ".join(link_failures), ) ui.error_write(msg)
def run(self): from pathlib import Path if self.args.show_vega: if not self.args.targets: logger.error("please specify a target for `--show-vega`") return 1 if len(self.args.targets) > 1: logger.error( "you can only specify one target for `--show-vega`" ) return 1 try: plots = self._func(targets=self.args.targets, props=self._props()) if self.args.show_vega: target = self.args.targets[0] ui.write(plots[target]) return 0 except DvcException: logger.exception("") return 1 if plots: rel: str = self.args.out or "plots.html" path: Path = (Path.cwd() / rel).resolve() self.repo.plots.write_html( path, plots=plots, html_template_path=self.args.html_template ) assert ( path.is_absolute() ) # as_uri throws ValueError if not absolute url = path.as_uri() ui.write(url) if self.args.open: import webbrowser opened = webbrowser.open(rel) if not opened: ui.error_write( "Failed to open. Please try opening it manually." ) return 1 else: ui.error_write( "No plots were loaded, visualization file will not be created." ) return 0
def show( repo, targets=None, all_branches=False, all_tags=False, recursive=False, revs=None, all_commits=False, onerror=None, ): if onerror is None: onerror = onerror_collect res = {} for rev in repo.brancher( revs=revs, all_branches=all_branches, all_tags=all_tags, all_commits=all_commits, ): res[rev] = error_handler(_gather_metrics)(repo, targets, rev, recursive, onerror=onerror) # Hide workspace metrics if they are the same as in the active branch try: active_branch = repo.scm.active_branch() except (SCMError, NoSCMError): # SCMError - detached head # NoSCMError - no repo case pass else: if res.get("workspace") == res.get(active_branch): res.pop("workspace", None) errored = errored_revisions(res) if errored: from dvc.ui import ui ui.error_write( "DVC failed to load some metrics for following revisions:" f" '{', '.join(errored)}'.") return res
def _notify(self, latest: str, pkg: Optional[str] = PKG) -> None: from dvc.ui import ui if not sys.stdout.isatty(): return message = self._get_message(latest, pkg=pkg) return ui.error_write(message, styled=True)
def collect_targets( repo: "Repo", targets: "TargetType", recursive: bool = False, glob: bool = False, ) -> Iterator[str]: for target in glob_targets(ensure_list(targets), glob=glob): expanded_targets = _find_all_targets(repo, target, recursive=recursive) for index, path in enumerate(expanded_targets): if index == LARGE_DIR_SIZE: msg = LARGE_DIR_RECURSIVE_ADD_WARNING.format( cyan=colorama.Fore.CYAN, nc=colorama.Style.RESET_ALL, target=target, ) ui.error_write(msg) yield path
def collect( self, targets: List[str] = None, revs: List[str] = None, recursive: bool = False, onerror: Optional[Callable] = None, props: Optional[Dict] = None, ) -> Dict[str, Dict]: """Collects all props and data for plots. Returns a structure like: {rev: {plots.csv: { props: {x: ..., "header": ..., ...}, data: "unstructured data (as stored for given extension)", }}} """ from dvc.utils.collections import ensure_list targets = ensure_list(targets) data: Dict[str, Dict] = {} for rev in self.repo.brancher(revs=revs): # .brancher() adds unwanted workspace if revs is not None and rev not in revs: continue rev = rev or "workspace" data[rev] = self._collect_from_revision( revision=rev, targets=targets, recursive=recursive, onerror=onerror, props=props, ) errored = errored_revisions(data) if errored: from dvc.ui import ui ui.error_write( "DVC failed to load some plots for following revisions: " f"'{', '.join(errored)}'.") return data
def init_interactive( defaults: Dict[str, str], provided: Iterable[str], show_heading: bool = False, live: bool = False, ) -> Dict[str, str]: primary = lremove(provided, ["cmd", "code", "data", "models", "params"]) secondary = lremove(provided, ["live"] if live else ["metrics", "plots"]) if not (primary or secondary): return {} message = ("This command will guide you to set up your first stage in " "[green]dvc.yaml[/green].\n") if show_heading: ui.error_write(message, styled=True) return compact({ **_prompts(primary, defaults), **_prompts(secondary, defaults), })
def __pretty_exc__(self, **kwargs: Any) -> None: """Prettify exception message.""" from collections.abc import Mapping lines: List[object] = [] data = parse_yaml_for_update(self.text, self.path) if isinstance(data, Mapping): lines.extend(self._prepare_context(data)) cause = "" if lines: # we should not add a newline after the main message # if there are no other outputs lines.insert(0, "") else: # if we don't have any context to show, we'll fallback to what we # got from voluptuous and print them in the same line. cause = f": {self.exc}" lines.insert(0, _prepare_message(f"{self}{cause}.")) for line in lines: ui.error_write(line, styled=True)
def init_interactive( self, defaults=None, show_heading: bool = False, live: bool = False, ): message = ( "This command will guide you to set up your first stage in " "[green]dvc.yaml[/green].\n" ) if show_heading: ui.error_write(message, styled=True) yield from self._prompt( { "cmd": "[b]Command[/b] to execute", "code": "Path to a [b]code[/b] file/directory", "data": "Path to a [b]data[/b] file/directory", "models": "Path to a [b]model[/b] file/directory", "params": "Path to a [b]parameters[/b] file", }, defaults=defaults, ) if not live: yield from self._prompt( { "metrics": "Path to a [b]metrics[/b] file", "plots": "Path to a [b]plots[/b] file/directory", }, defaults=defaults, ) return yield from self._prompt( {"live": "Path to log [b]dvclive[/b] outputs"}, defaults=defaults, )
def init_interactive( name: str, defaults: Dict[str, str], provided: Dict[str, str], show_tree: bool = False, live: bool = False, ) -> Dict[str, str]: primary = lremove(provided.keys(), ["cmd", "code", "data", "models", "params"]) secondary = lremove(provided.keys(), ["live"] if live else ["metrics", "plots"]) if not (primary or secondary): return {} message = ui.rich_text.assemble( "This command will guide you to set up a ", (name, "bright_blue"), " stage in ", ("dvc.yaml", "green"), ".", ) doc_link = ui.rich_text.assemble("See ", (PIPELINE_FILE_LINK, "repr.url"), ".") ui.error_write(message, doc_link, "", sep="\n", styled=True) if show_tree: from rich.tree import Tree tree = Tree( "DVC assumes the following workspace structure:", highlight=True, ) workspace = {**defaults, **provided} workspace.pop("cmd", None) if not live and "live" not in provided: workspace.pop("live", None) for value in sorted(workspace.values()): tree.add(f"[green]{value}[/green]") ui.error_write(tree, styled=True) ui.error_write() return compact({ **_prompts(primary, defaults), **_prompts(secondary, defaults), })
def run(self): from pathlib import Path from dvc.render.match import match_renderers from dvc_render import render_html if self.args.show_vega: if not self.args.targets: logger.error("please specify a target for `--show-vega`") return 1 if len(self.args.targets) > 1: logger.error( "you can only specify one target for `--show-vega`") return 1 if self.args.json: logger.error( "'--show-vega' and '--json' are mutually exclusive " "options.") return 1 try: plots_data = self._func(targets=self.args.targets, props=self._props()) if not plots_data: ui.error_write("No plots were loaded, " "visualization file will not be created.") out: str = self.args.out or self.repo.config.get("plots", {}).get( "out_dir", "dvc_plots") renderers_out = (out if self.args.json else os.path.join( out, "static")) renderers = match_renderers( plots_data=plots_data, out=renderers_out, templates_dir=self.repo.plots.templates_dir, ) if self.args.show_vega: renderer = first(filter(lambda r: r.TYPE == "vega", renderers)) if renderer: ui.write_json(json.loads(renderer.get_filled_template())) return 0 if self.args.json: _show_json(renderers, self.args.split) return 0 html_template_path = self.args.html_template if not html_template_path: html_template_path = self.repo.config.get("plots", {}).get( "html_template", None) if html_template_path and not os.path.isabs( html_template_path): html_template_path = os.path.join(self.repo.dvc_dir, html_template_path) output_file: Path = (Path.cwd() / out).resolve() / "index.html" render_html( renderers=renderers, output_file=output_file, template_path=html_template_path, ) ui.write(output_file.as_uri()) auto_open = self.repo.config["plots"].get("auto_open", False) if self.args.open or auto_open: if not auto_open: ui.write("To enable auto opening, you can run:\n" "\n" "\tdvc config plots.auto_open true") return ui.open_browser(output_file) return 0 except DvcException: logger.exception("") return 1
def run(self): from dvc.command.stage import parse_cmd cmd = parse_cmd(self.args.cmd) if not self.args.interactive and not cmd: raise InvalidArgumentError("command is not specified") from dvc.dvcfile import make_dvcfile global_defaults = { "code": self.CODE, "data": self.DATA, "models": self.MODELS, "metrics": self.DEFAULT_METRICS, "params": self.DEFAULT_PARAMS, "plots": self.PLOTS, "live": self.DVCLIVE, } dvcfile = make_dvcfile(self.repo, "dvc.yaml") name = self.args.name or self.args.type dvcfile_exists = dvcfile.exists() if not self.args.force and dvcfile_exists and name in dvcfile.stages: from dvc.stage.exceptions import DuplicateStageName hint = "Use '--force' to overwrite." raise DuplicateStageName( f"Stage '{name}' already exists in 'dvc.yaml'. {hint}" ) context = ChainMap() if not self.args.explicit: config = {} # TODO context.maps.extend([config, global_defaults]) with_live = self.args.type == "live" if self.args.interactive: try: context = self.init_interactive( defaults=context, show_heading=not dvcfile_exists, live=with_live, ) except (KeyboardInterrupt, EOFError): ui.error_write() raise elif with_live: # suppress `metrics`/`params` if live is selected, unless # also provided via cli, also make output to be a checkpoint. context = context.new_child({"metrics": None, "params": None}) else: # suppress live otherwise context = context.new_child({"live": None}) if not self.args.interactive: d = compact( { "cmd": cmd, "code": self.args.code, "data": self.args.data, "models": self.args.models, "metrics": self.args.metrics, "params": self.args.params, "plots": self.args.plots, "live": self.args.live, } ) context = context.new_child(d) assert "cmd" in context command = context["cmd"] code = context.get("code") data = context.get("data") models = context.get("models") metrics = context.get("metrics") plots = context.get("plots") live = context.get("live") params_kv = [] if context.get("params"): from dvc.utils.serialize import LOADERS path = context["params"] _, ext = os.path.splitext(path) params_kv = [{path: list(LOADERS[ext](path))}] checkpoint_out = bool(context.get("live")) stage = self.repo.stage.add( name=name, cmd=command, deps=compact([code, data]), params=params_kv, metrics_no_cache=compact([metrics]), plots_no_cache=compact([plots]), live=live, force=self.args.force, **{"checkpoints" if checkpoint_out else "outs": compact([models])}, ) if self.args.run: return self.repo.experiments.run(targets=[stage.addressing]) return 0
def init( repo: "Repo", name: str = None, type: str = "default", # pylint: disable=redefined-builtin defaults: Dict[str, str] = None, overrides: Dict[str, str] = None, interactive: bool = False, force: bool = False, ) -> "Stage": from dvc.dvcfile import make_dvcfile dvcfile = make_dvcfile(repo, "dvc.yaml") name = name or type _check_stage_exists(dvcfile, name, force=force) defaults = defaults or {} overrides = overrides or {} with_live = type == "live" if interactive: defaults = init_interactive( name, defaults=defaults, live=with_live, provided=overrides, show_tree=True, ) else: if with_live: # suppress `metrics`/`params` if live is selected, unless # it is also provided via overrides/cli. # This makes output to be a checkpoint as well. defaults.pop("metrics") defaults.pop("params") else: defaults.pop("live") # suppress live otherwise context: Dict[str, str] = {**defaults, **overrides} assert "cmd" in context params_kv = [] if context.get("params"): from dvc.utils.serialize import LOADERS path = context["params"] assert isinstance(path, str) _, ext = os.path.splitext(path) params_kv = [{path: list(LOADERS[ext](path))}] checkpoint_out = bool(context.get("live")) models = context.get("models") stage = repo.stage.create( name=name, cmd=context["cmd"], deps=compact([context.get("code"), context.get("data")]), params=params_kv, metrics_no_cache=compact([context.get("metrics")]), plots_no_cache=compact([context.get("plots")]), live=context.get("live"), force=force, **{"checkpoints" if checkpoint_out else "outs": compact([models])}, ) if interactive: ui.write(Rule(style="green"), styled=True) _yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage))) syn = Syntax(_yaml, "yaml", theme="ansi_dark") ui.error_write(syn, styled=True) if not interactive or ui.confirm( "Do you want to add the above contents to dvc.yaml?"): with _disable_logging(): stage.dump(update_lock=False) stage.ignore_outs() else: raise DvcException("Aborting ...") return stage
def get_input(cls, *args, **kwargs) -> str: try: return super().get_input(*args, **kwargs) except (KeyboardInterrupt, EOFError): ui.error_write() raise
def init( repo: "Repo", name: str = None, type: str = "default", # pylint: disable=redefined-builtin defaults: Dict[str, str] = None, overrides: Dict[str, str] = None, interactive: bool = False, force: bool = False, stream: Optional[TextIO] = None, ) -> "Stage": from dvc.dvcfile import make_dvcfile dvcfile = make_dvcfile(repo, "dvc.yaml") name = name or type _check_stage_exists(dvcfile, name, force=force) defaults = defaults.copy() if defaults else {} overrides = overrides.copy() if overrides else {} with_live = type == "live" if interactive: defaults = init_interactive( name, validator=validate_prompts, defaults=defaults, live=with_live, provided=overrides, stream=stream, ) else: if with_live: # suppress `metrics`/`plots` if live is selected, unless # it is also provided via overrides/cli. # This makes output to be a checkpoint as well. defaults.pop("metrics", None) defaults.pop("plots", None) else: defaults.pop("live", None) # suppress live otherwise context: Dict[str, str] = {**defaults, **overrides} assert "cmd" in context params_kv = [] params = context.get("params") if params: params_kv.append(loadd_params(params)) checkpoint_out = bool(context.get("live")) models = context.get("models") stage = repo.stage.create( name=name, cmd=context["cmd"], deps=compact([context.get("code"), context.get("data")]), params=params_kv, metrics_no_cache=compact([context.get("metrics")]), plots_no_cache=compact([context.get("plots")]), live=context.get("live"), force=force, **{"checkpoints" if checkpoint_out else "outs": compact([models])}, ) if interactive: ui.error_write(Rule(style="green"), styled=True) _yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage))) syn = Syntax(_yaml, "yaml", theme="ansi_dark") ui.error_write(syn, styled=True) from dvc.ui.prompt import Confirm if not interactive or Confirm.ask( "Do you want to add the above contents to dvc.yaml?", console=ui.error_console, default=True, stream=stream, ): with _disable_logging(), repo.scm_context(autostage=True, quiet=True): stage.dump(update_lock=False) stage.ignore_outs() if params: repo.scm_context.track_file(params) else: raise DvcException("Aborting ...") return stage
def run(self): from pathlib import Path from dvc.render.utils import match_renderers, render from dvc.render.vega import VegaRenderer if self.args.show_vega: if not self.args.targets: logger.error("please specify a target for `--show-vega`") return 1 if len(self.args.targets) > 1: logger.error( "you can only specify one target for `--show-vega`") return 1 if self.args.json: logger.error( "'--show-vega' and '--json' are mutually exclusive " "options.") return 1 try: plots_data = self._func(targets=self.args.targets, props=self._props()) if not plots_data: ui.error_write("No plots were loaded, " "visualization file will not be created.") renderers = match_renderers(plots_data=plots_data, templates=self.repo.plots.templates) if self.args.show_vega: renderer = first( filter(lambda r: isinstance(r, VegaRenderer), renderers)) if renderer: ui.write_json(renderer.asdict()) return 0 if self.args.json: _show_json(renderers, self.args.out) return 0 rel: str = self.args.out or "dvc_plots" path: Path = (Path.cwd() / rel).resolve() index_path = render( self.repo, renderers, path=path, html_template_path=self.args.html_template, ) ui.write(index_path.as_uri()) auto_open = self.repo.config["plots"].get("auto_open", False) if self.args.open or auto_open: if not auto_open: ui.write("To enable auto opening, you can run:\n" "\n" "\tdvc config plots.auto_open true") return ui.open_browser(index_path) return 0 except DvcException: logger.exception("") return 1