Пример #1
0
def init_interactive(
    defaults: Dict[str, str],
    provided: Dict[str, str],
    validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None,
    stream: Optional[TextIO] = None,
) -> Dict[str, str]:
    command_prompts = lremove(provided.keys(), ["cmd"])
    dependencies_prompts = lremove(provided.keys(), ["code", "data", "params"])
    output_keys = ["models"]
    if "live" not in provided:
        output_keys.extend(["metrics", "plots"])
    outputs_prompts = lremove(provided.keys(), output_keys)

    ret: Dict[str, str] = {}
    if "cmd" in provided:
        ret["cmd"] = provided["cmd"]

    for heading, prompts, allow_omission in (
        ("", command_prompts, False),
        ("Enter experiment dependencies.", dependencies_prompts, True),
        ("Enter experiment outputs.", outputs_prompts, True),
    ):
        if prompts and heading:
            ui.error_write(heading, styled=True)
        response = _prompts(
            prompts,
            defaults=defaults,
            allow_omission=allow_omission,
            validator=validator,
            stream=stream,
        )
        ret.update(compact(response))
        if prompts:
            ui.error_write(styled=True)
    return ret
Пример #2
0
    def show(
        self,
        targets: List[str] = None,
        revs=None,
        props=None,
        recursive=False,
        onerror=None,
    ):
        if onerror is None:
            onerror = onerror_collect

        data: Dict[str, Dict] = {}
        for rev_data in self.collect(targets,
                                     revs,
                                     recursive,
                                     onerror=onerror,
                                     props=props):
            data.update(rev_data)

        errored = errored_revisions(data)
        if errored:
            from dvc.ui import ui

            ui.error_write(
                "DVC failed to load some plots for following revisions: "
                f"'{', '.join(errored)}'.")

        return data
Пример #3
0
    def show(
        self,
        targets: List[str] = None,
        revs=None,
        props=None,
        recursive=False,
        onerror=None,
    ):
        if onerror is None:
            onerror = onerror_collect

        result: Dict[str, Dict] = {}
        for data in self.collect(targets,
                                 revs,
                                 recursive,
                                 onerror=onerror,
                                 props=props):
            assert len(data) == 1
            revision_data = first(data.values())
            if "data" in revision_data:
                for path_data in revision_data["data"].values():
                    result_source = path_data.pop("data_source", None)
                    if result_source:
                        path_data.update(result_source())
            result.update(data)

        errored = errored_revisions(result)
        if errored:
            from dvc.ui import ui

            ui.error_write(
                "DVC failed to load some plots for following revisions: "
                f"'{', '.join(errored)}'.")

        return result
Пример #4
0
def show(repo, revs=None, targets=None, deps=False, onerror: Callable = None):
    if onerror is None:
        onerror = onerror_collect
    res = {}

    for branch in repo.brancher(revs=revs):
        params = error_handler(_gather_params)(repo=repo,
                                               rev=branch,
                                               targets=targets,
                                               deps=deps,
                                               onerror=onerror)

        if params:
            res[branch] = params

    # Hide workspace params if they are the same as in the active branch
    try:
        active_branch = repo.scm.active_branch()
    except (SCMError, NoSCMError):
        # SCMError - detached head
        # NoSCMError - no repo case
        pass
    else:
        if res.get("workspace") == res.get(active_branch):
            res.pop("workspace", None)

    errored = errored_revisions(res)
    if errored:
        ui.error_write(
            "DVC failed to load some parameters for following revisions:"
            f" '{', '.join(errored)}'.")

    return res
Пример #5
0
    def __pretty_exc__(self, **kwargs: Any) -> None:
        from ruamel.yaml.error import MarkedYAMLError

        exc = self.exc.__cause__

        if not isinstance(exc, MarkedYAMLError):
            raise ValueError("nothing to pretty-print here. :)")

        source = self.yaml_text.splitlines()

        def prepare_linecol(mark: "StreamMark") -> str:
            return f"in line {mark.line + 1}, column {mark.column + 1}"

        def prepare_message(
            message: str, mark: "StreamMark" = None
        ) -> "RichText":
            cause = ", ".join(
                [message.capitalize(), prepare_linecol(mark) if mark else ""]
            )
            return _prepare_cause(cause)

        def prepare_code(mark: "StreamMark") -> "Syntax":
            line = mark.line + 1
            code = "" if line > len(source) else source[line - 1]
            return _prepare_code_snippets(code, line)

        lines: List[object] = []
        if hasattr(exc, "context"):
            if exc.context_mark is not None:
                lines.append(
                    prepare_message(str(exc.context), exc.context_mark)
                )
            if exc.context_mark is not None and (
                exc.problem is None
                or exc.problem_mark is None
                or exc.context_mark.name != exc.problem_mark.name
                or exc.context_mark.line != exc.problem_mark.line
                or exc.context_mark.column != exc.problem_mark.column
            ):
                lines.extend([prepare_code(exc.context_mark), ""])
            if exc.problem is not None:
                lines.append(
                    prepare_message(str(exc.problem), exc.problem_mark)
                )
            if exc.problem_mark is not None:
                lines.append(prepare_code(exc.problem_mark))

        if lines:
            # we should not add a newline after the main message
            # if there are no other outputs
            lines.insert(0, "")

        rel = make_relpath(self.path)
        rev_msg = f" in revision '{self.rev[:7]}'" if self.rev else ""
        msg_fmt = f"'{rel}' is invalid{self.hint}{rev_msg}."
        lines.insert(0, _prepare_message(msg_fmt))
        for line in lines:
            ui.error_write(line, styled=True)
Пример #6
0
    def run(self):
        from pathlib import Path

        if self.args.show_vega:
            if not self.args.targets:
                logger.error("please specify a target for `--show-vega`")
                return 1
            if len(self.args.targets) > 1:
                logger.error(
                    "you can only specify one target for `--show-vega`"
                )
                return 1

        try:
            plots_data = self._func(
                targets=self.args.targets, props=self._props()
            )

            if not plots_data:
                ui.error_write(
                    "No plots were loaded, "
                    "visualization file will not be created."
                )

            if self.args.show_vega:
                target = self.args.targets[0]
                plot_json = find_vega(self.repo, plots_data, target)
                if plot_json:
                    ui.write(plot_json)
                return 0

            rel: str = self.args.out or "dvc_plots"
            path: Path = (Path.cwd() / rel).resolve()
            index_path = render(
                self.repo,
                plots_data,
                path=path,
                html_template_path=self.args.html_template,
            )

            assert index_path.is_absolute()
            url = index_path.as_uri()
            ui.write(url)

            if self.args.open:
                import webbrowser

                opened = webbrowser.open(index_path)
                if not opened:
                    ui.error_write(
                        "Failed to open. Please try opening it manually."
                    )
                    return 1
            return 0

        except DvcException:
            logger.exception("")
            return 1
Пример #7
0
def transform_targets(args):
    from funcy import count_reps

    counts = count_reps(ensure_list(args.targets))
    dupes = [key for key, count in counts.items() if count > 1]
    if dupes:
        msg = ", ".join(f"[b]{key}[/]" for key in dupes)
        ui.error_write(f"ignoring duplicated targets: {msg}", styled=True)
    args.targets = list(counts)
Пример #8
0
def init_interactive(
    name: str,
    defaults: Dict[str, str],
    provided: Dict[str, str],
    validator: Callable[[str, str], Union[str, Tuple[str, str]]] = None,
    live: bool = False,
    stream: Optional[TextIO] = None,
) -> Dict[str, str]:
    command = provided.pop("cmd", None)
    primary = lremove(provided.keys(), ["code", "data", "models", "params"])
    secondary = lremove(provided.keys(),
                        ["live"] if live else ["metrics", "plots"])
    prompts = primary + secondary

    workspace = {**defaults, **provided}
    if not live and "live" not in provided:
        workspace.pop("live", None)
    for key in ("plots", "metrics"):
        if live and key not in provided:
            workspace.pop(key, None)

    ret: Dict[str, str] = {}
    if command:
        ret["cmd"] = command

    if not prompts and command:
        return ret

    ui.error_write(
        f"This command will guide you to set up a [bright_blue]{name}[/]",
        "stage in [green]dvc.yaml[/].",
        f"\nSee [repr.url]{PIPELINE_FILE_LINK}[/].\n",
        styled=True,
    )

    if not command:
        ret.update(
            compact(_prompts(["cmd"], allow_omission=False, stream=stream)))
        if prompts:
            ui.error_write(styled=True)

    if not prompts:
        return ret

    ui.error_write(
        "Enter the paths for dependencies and outputs of the command.",
        styled=True,
    )
    if workspace:
        ui.error_write(build_workspace_tree(workspace), styled=True)
    ui.error_write(styled=True)
    ret.update(
        compact(_prompts(prompts, defaults, validator=validator,
                         stream=stream)))
    return ret
Пример #9
0
def test_capsys_works(capsys: CaptureFixture[str]):
    """Sanity check that capsys can capture outputs from a global ui."""
    from dvc.ui import ui

    message = "hello world"
    ui.write(message)
    ui.error_write(message)

    captured = capsys.readouterr()
    assert captured.out == f"{message}\n"
    assert captured.err == f"{message}\n"
Пример #10
0
def warn_link_failures() -> Iterator[List[str]]:
    link_failures: List[str] = []
    try:
        yield link_failures
    finally:
        if link_failures:
            msg = LINK_FAILURE_MESSAGE.format(
                CacheLinkError.SUPPORT_LINK,
                " ".join(link_failures),
            )
            ui.error_write(msg)
Пример #11
0
    def run(self):
        from pathlib import Path

        if self.args.show_vega:
            if not self.args.targets:
                logger.error("please specify a target for `--show-vega`")
                return 1
            if len(self.args.targets) > 1:
                logger.error(
                    "you can only specify one target for `--show-vega`"
                )
                return 1

        try:
            plots = self._func(targets=self.args.targets, props=self._props())

            if self.args.show_vega:
                target = self.args.targets[0]
                ui.write(plots[target])
                return 0

        except DvcException:
            logger.exception("")
            return 1

        if plots:
            rel: str = self.args.out or "plots.html"
            path: Path = (Path.cwd() / rel).resolve()
            self.repo.plots.write_html(
                path, plots=plots, html_template_path=self.args.html_template
            )

            assert (
                path.is_absolute()
            )  # as_uri throws ValueError if not absolute
            url = path.as_uri()
            ui.write(url)
            if self.args.open:
                import webbrowser

                opened = webbrowser.open(rel)
                if not opened:
                    ui.error_write(
                        "Failed to open. Please try opening it manually."
                    )
                    return 1
        else:
            ui.error_write(
                "No plots were loaded, visualization file will not be created."
            )
        return 0
Пример #12
0
def show(
    repo,
    targets=None,
    all_branches=False,
    all_tags=False,
    recursive=False,
    revs=None,
    all_commits=False,
    onerror=None,
):
    if onerror is None:
        onerror = onerror_collect

    res = {}
    for rev in repo.brancher(
            revs=revs,
            all_branches=all_branches,
            all_tags=all_tags,
            all_commits=all_commits,
    ):
        res[rev] = error_handler(_gather_metrics)(repo,
                                                  targets,
                                                  rev,
                                                  recursive,
                                                  onerror=onerror)

    # Hide workspace metrics if they are the same as in the active branch
    try:
        active_branch = repo.scm.active_branch()
    except (SCMError, NoSCMError):
        # SCMError - detached head
        # NoSCMError - no repo case
        pass
    else:
        if res.get("workspace") == res.get(active_branch):
            res.pop("workspace", None)

    errored = errored_revisions(res)
    if errored:
        from dvc.ui import ui

        ui.error_write(
            "DVC failed to load some metrics for following revisions:"
            f" '{', '.join(errored)}'.")

    return res
Пример #13
0
    def _notify(self, latest: str, pkg: Optional[str] = PKG) -> None:
        from dvc.ui import ui

        if not sys.stdout.isatty():
            return

        message = self._get_message(latest, pkg=pkg)
        return ui.error_write(message, styled=True)
Пример #14
0
def collect_targets(
    repo: "Repo",
    targets: "TargetType",
    recursive: bool = False,
    glob: bool = False,
) -> Iterator[str]:
    for target in glob_targets(ensure_list(targets), glob=glob):
        expanded_targets = _find_all_targets(repo, target, recursive=recursive)
        for index, path in enumerate(expanded_targets):
            if index == LARGE_DIR_SIZE:
                msg = LARGE_DIR_RECURSIVE_ADD_WARNING.format(
                    cyan=colorama.Fore.CYAN,
                    nc=colorama.Style.RESET_ALL,
                    target=target,
                )
                ui.error_write(msg)
            yield path
Пример #15
0
    def collect(
        self,
        targets: List[str] = None,
        revs: List[str] = None,
        recursive: bool = False,
        onerror: Optional[Callable] = None,
        props: Optional[Dict] = None,
    ) -> Dict[str, Dict]:
        """Collects all props and data for plots.

        Returns a structure like:
            {rev: {plots.csv: {
                props: {x: ..., "header": ..., ...},
                data: "unstructured data (as stored for given extension)",
            }}}
        """
        from dvc.utils.collections import ensure_list

        targets = ensure_list(targets)
        data: Dict[str, Dict] = {}
        for rev in self.repo.brancher(revs=revs):
            # .brancher() adds unwanted workspace
            if revs is not None and rev not in revs:
                continue
            rev = rev or "workspace"
            data[rev] = self._collect_from_revision(
                revision=rev,
                targets=targets,
                recursive=recursive,
                onerror=onerror,
                props=props,
            )

        errored = errored_revisions(data)
        if errored:
            from dvc.ui import ui

            ui.error_write(
                "DVC failed to load some plots for following revisions: "
                f"'{', '.join(errored)}'.")

        return data
Пример #16
0
def init_interactive(
    defaults: Dict[str, str],
    provided: Iterable[str],
    show_heading: bool = False,
    live: bool = False,
) -> Dict[str, str]:
    primary = lremove(provided, ["cmd", "code", "data", "models", "params"])
    secondary = lremove(provided, ["live"] if live else ["metrics", "plots"])

    if not (primary or secondary):
        return {}

    message = ("This command will guide you to set up your first stage in "
               "[green]dvc.yaml[/green].\n")
    if show_heading:
        ui.error_write(message, styled=True)

    return compact({
        **_prompts(primary, defaults),
        **_prompts(secondary, defaults),
    })
Пример #17
0
    def __pretty_exc__(self, **kwargs: Any) -> None:
        """Prettify exception message."""
        from collections.abc import Mapping

        lines: List[object] = []
        data = parse_yaml_for_update(self.text, self.path)
        if isinstance(data, Mapping):
            lines.extend(self._prepare_context(data))

        cause = ""
        if lines:
            # we should not add a newline after the main message
            # if there are no other outputs
            lines.insert(0, "")
        else:
            # if we don't have any context to show, we'll fallback to what we
            # got from voluptuous and print them in the same line.
            cause = f": {self.exc}"

        lines.insert(0, _prepare_message(f"{self}{cause}."))
        for line in lines:
            ui.error_write(line, styled=True)
Пример #18
0
    def init_interactive(
        self,
        defaults=None,
        show_heading: bool = False,
        live: bool = False,
    ):
        message = (
            "This command will guide you to set up your first stage in "
            "[green]dvc.yaml[/green].\n"
        )
        if show_heading:
            ui.error_write(message, styled=True)

        yield from self._prompt(
            {
                "cmd": "[b]Command[/b] to execute",
                "code": "Path to a [b]code[/b] file/directory",
                "data": "Path to a [b]data[/b] file/directory",
                "models": "Path to a [b]model[/b] file/directory",
                "params": "Path to a [b]parameters[/b] file",
            },
            defaults=defaults,
        )

        if not live:
            yield from self._prompt(
                {
                    "metrics": "Path to a [b]metrics[/b] file",
                    "plots": "Path to a [b]plots[/b] file/directory",
                },
                defaults=defaults,
            )
            return

        yield from self._prompt(
            {"live": "Path to log [b]dvclive[/b] outputs"},
            defaults=defaults,
        )
Пример #19
0
def init_interactive(
    name: str,
    defaults: Dict[str, str],
    provided: Dict[str, str],
    show_tree: bool = False,
    live: bool = False,
) -> Dict[str, str]:
    primary = lremove(provided.keys(),
                      ["cmd", "code", "data", "models", "params"])
    secondary = lremove(provided.keys(),
                        ["live"] if live else ["metrics", "plots"])

    if not (primary or secondary):
        return {}

    message = ui.rich_text.assemble(
        "This command will guide you to set up a ",
        (name, "bright_blue"),
        " stage in ",
        ("dvc.yaml", "green"),
        ".",
    )
    doc_link = ui.rich_text.assemble("See ", (PIPELINE_FILE_LINK, "repr.url"),
                                     ".")
    ui.error_write(message, doc_link, "", sep="\n", styled=True)

    if show_tree:
        from rich.tree import Tree

        tree = Tree(
            "DVC assumes the following workspace structure:",
            highlight=True,
        )
        workspace = {**defaults, **provided}
        workspace.pop("cmd", None)
        if not live and "live" not in provided:
            workspace.pop("live", None)
        for value in sorted(workspace.values()):
            tree.add(f"[green]{value}[/green]")
        ui.error_write(tree, styled=True)
        ui.error_write()

    return compact({
        **_prompts(primary, defaults),
        **_prompts(secondary, defaults),
    })
Пример #20
0
Файл: plots.py Проект: jear/dvc
    def run(self):
        from pathlib import Path

        from dvc.render.match import match_renderers
        from dvc_render import render_html

        if self.args.show_vega:
            if not self.args.targets:
                logger.error("please specify a target for `--show-vega`")
                return 1
            if len(self.args.targets) > 1:
                logger.error(
                    "you can only specify one target for `--show-vega`")
                return 1
            if self.args.json:
                logger.error(
                    "'--show-vega' and '--json' are mutually exclusive "
                    "options.")
                return 1

        try:

            plots_data = self._func(targets=self.args.targets,
                                    props=self._props())

            if not plots_data:
                ui.error_write("No plots were loaded, "
                               "visualization file will not be created.")

            out: str = self.args.out or self.repo.config.get("plots", {}).get(
                "out_dir", "dvc_plots")

            renderers_out = (out if self.args.json else os.path.join(
                out, "static"))
            renderers = match_renderers(
                plots_data=plots_data,
                out=renderers_out,
                templates_dir=self.repo.plots.templates_dir,
            )

            if self.args.show_vega:
                renderer = first(filter(lambda r: r.TYPE == "vega", renderers))
                if renderer:
                    ui.write_json(json.loads(renderer.get_filled_template()))
                return 0
            if self.args.json:
                _show_json(renderers, self.args.split)
                return 0

            html_template_path = self.args.html_template
            if not html_template_path:
                html_template_path = self.repo.config.get("plots", {}).get(
                    "html_template", None)
                if html_template_path and not os.path.isabs(
                        html_template_path):
                    html_template_path = os.path.join(self.repo.dvc_dir,
                                                      html_template_path)

            output_file: Path = (Path.cwd() / out).resolve() / "index.html"

            render_html(
                renderers=renderers,
                output_file=output_file,
                template_path=html_template_path,
            )

            ui.write(output_file.as_uri())
            auto_open = self.repo.config["plots"].get("auto_open", False)
            if self.args.open or auto_open:
                if not auto_open:
                    ui.write("To enable auto opening, you can run:\n"
                             "\n"
                             "\tdvc config plots.auto_open true")
                return ui.open_browser(output_file)

            return 0

        except DvcException:
            logger.exception("")
            return 1
Пример #21
0
    def run(self):
        from dvc.command.stage import parse_cmd

        cmd = parse_cmd(self.args.cmd)
        if not self.args.interactive and not cmd:
            raise InvalidArgumentError("command is not specified")

        from dvc.dvcfile import make_dvcfile

        global_defaults = {
            "code": self.CODE,
            "data": self.DATA,
            "models": self.MODELS,
            "metrics": self.DEFAULT_METRICS,
            "params": self.DEFAULT_PARAMS,
            "plots": self.PLOTS,
            "live": self.DVCLIVE,
        }

        dvcfile = make_dvcfile(self.repo, "dvc.yaml")
        name = self.args.name or self.args.type

        dvcfile_exists = dvcfile.exists()
        if not self.args.force and dvcfile_exists and name in dvcfile.stages:
            from dvc.stage.exceptions import DuplicateStageName

            hint = "Use '--force' to overwrite."
            raise DuplicateStageName(
                f"Stage '{name}' already exists in 'dvc.yaml'. {hint}"
            )

        context = ChainMap()
        if not self.args.explicit:
            config = {}  # TODO
            context.maps.extend([config, global_defaults])

        with_live = self.args.type == "live"
        if self.args.interactive:
            try:
                context = self.init_interactive(
                    defaults=context,
                    show_heading=not dvcfile_exists,
                    live=with_live,
                )
            except (KeyboardInterrupt, EOFError):
                ui.error_write()
                raise
        elif with_live:
            # suppress `metrics`/`params` if live is selected, unless
            # also provided via cli, also make output to be a checkpoint.
            context = context.new_child({"metrics": None, "params": None})
        else:
            # suppress live otherwise
            context = context.new_child({"live": None})

        if not self.args.interactive:
            d = compact(
                {
                    "cmd": cmd,
                    "code": self.args.code,
                    "data": self.args.data,
                    "models": self.args.models,
                    "metrics": self.args.metrics,
                    "params": self.args.params,
                    "plots": self.args.plots,
                    "live": self.args.live,
                }
            )
            context = context.new_child(d)

        assert "cmd" in context
        command = context["cmd"]
        code = context.get("code")
        data = context.get("data")
        models = context.get("models")
        metrics = context.get("metrics")
        plots = context.get("plots")
        live = context.get("live")

        params_kv = []
        if context.get("params"):
            from dvc.utils.serialize import LOADERS

            path = context["params"]
            _, ext = os.path.splitext(path)
            params_kv = [{path: list(LOADERS[ext](path))}]

        checkpoint_out = bool(context.get("live"))
        stage = self.repo.stage.add(
            name=name,
            cmd=command,
            deps=compact([code, data]),
            params=params_kv,
            metrics_no_cache=compact([metrics]),
            plots_no_cache=compact([plots]),
            live=live,
            force=self.args.force,
            **{"checkpoints" if checkpoint_out else "outs": compact([models])},
        )

        if self.args.run:
            return self.repo.experiments.run(targets=[stage.addressing])
        return 0
Пример #22
0
def init(
    repo: "Repo",
    name: str = None,
    type: str = "default",  # pylint: disable=redefined-builtin
    defaults: Dict[str, str] = None,
    overrides: Dict[str, str] = None,
    interactive: bool = False,
    force: bool = False,
) -> "Stage":
    from dvc.dvcfile import make_dvcfile

    dvcfile = make_dvcfile(repo, "dvc.yaml")
    name = name or type

    _check_stage_exists(dvcfile, name, force=force)

    defaults = defaults or {}
    overrides = overrides or {}

    with_live = type == "live"
    if interactive:
        defaults = init_interactive(
            name,
            defaults=defaults,
            live=with_live,
            provided=overrides,
            show_tree=True,
        )
    else:
        if with_live:
            # suppress `metrics`/`params` if live is selected, unless
            # it is also provided via overrides/cli.
            # This makes output to be a checkpoint as well.
            defaults.pop("metrics")
            defaults.pop("params")
        else:
            defaults.pop("live")  # suppress live otherwise

    context: Dict[str, str] = {**defaults, **overrides}
    assert "cmd" in context

    params_kv = []
    if context.get("params"):
        from dvc.utils.serialize import LOADERS

        path = context["params"]
        assert isinstance(path, str)
        _, ext = os.path.splitext(path)
        params_kv = [{path: list(LOADERS[ext](path))}]

    checkpoint_out = bool(context.get("live"))
    models = context.get("models")
    stage = repo.stage.create(
        name=name,
        cmd=context["cmd"],
        deps=compact([context.get("code"),
                      context.get("data")]),
        params=params_kv,
        metrics_no_cache=compact([context.get("metrics")]),
        plots_no_cache=compact([context.get("plots")]),
        live=context.get("live"),
        force=force,
        **{"checkpoints" if checkpoint_out else "outs": compact([models])},
    )

    if interactive:
        ui.write(Rule(style="green"), styled=True)
        _yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage)))
        syn = Syntax(_yaml, "yaml", theme="ansi_dark")
        ui.error_write(syn, styled=True)

    if not interactive or ui.confirm(
            "Do you want to add the above contents to dvc.yaml?"):
        with _disable_logging():
            stage.dump(update_lock=False)
        stage.ignore_outs()
    else:
        raise DvcException("Aborting ...")
    return stage
Пример #23
0
 def get_input(cls, *args, **kwargs) -> str:
     try:
         return super().get_input(*args, **kwargs)
     except (KeyboardInterrupt, EOFError):
         ui.error_write()
         raise
Пример #24
0
def init(
    repo: "Repo",
    name: str = None,
    type: str = "default",  # pylint: disable=redefined-builtin
    defaults: Dict[str, str] = None,
    overrides: Dict[str, str] = None,
    interactive: bool = False,
    force: bool = False,
    stream: Optional[TextIO] = None,
) -> "Stage":
    from dvc.dvcfile import make_dvcfile

    dvcfile = make_dvcfile(repo, "dvc.yaml")
    name = name or type

    _check_stage_exists(dvcfile, name, force=force)

    defaults = defaults.copy() if defaults else {}
    overrides = overrides.copy() if overrides else {}

    with_live = type == "live"

    if interactive:
        defaults = init_interactive(
            name,
            validator=validate_prompts,
            defaults=defaults,
            live=with_live,
            provided=overrides,
            stream=stream,
        )
    else:
        if with_live:
            # suppress `metrics`/`plots` if live is selected, unless
            # it is also provided via overrides/cli.
            # This makes output to be a checkpoint as well.
            defaults.pop("metrics", None)
            defaults.pop("plots", None)
        else:
            defaults.pop("live", None)  # suppress live otherwise

    context: Dict[str, str] = {**defaults, **overrides}
    assert "cmd" in context

    params_kv = []
    params = context.get("params")
    if params:
        params_kv.append(loadd_params(params))

    checkpoint_out = bool(context.get("live"))
    models = context.get("models")
    stage = repo.stage.create(
        name=name,
        cmd=context["cmd"],
        deps=compact([context.get("code"),
                      context.get("data")]),
        params=params_kv,
        metrics_no_cache=compact([context.get("metrics")]),
        plots_no_cache=compact([context.get("plots")]),
        live=context.get("live"),
        force=force,
        **{"checkpoints" if checkpoint_out else "outs": compact([models])},
    )

    if interactive:
        ui.error_write(Rule(style="green"), styled=True)
        _yaml = dumps_yaml(to_pipeline_file(cast(PipelineStage, stage)))
        syn = Syntax(_yaml, "yaml", theme="ansi_dark")
        ui.error_write(syn, styled=True)

    from dvc.ui.prompt import Confirm

    if not interactive or Confirm.ask(
            "Do you want to add the above contents to dvc.yaml?",
            console=ui.error_console,
            default=True,
            stream=stream,
    ):
        with _disable_logging(), repo.scm_context(autostage=True, quiet=True):
            stage.dump(update_lock=False)
            stage.ignore_outs()
            if params:
                repo.scm_context.track_file(params)
    else:
        raise DvcException("Aborting ...")
    return stage
Пример #25
0
    def run(self):
        from pathlib import Path

        from dvc.render.utils import match_renderers, render
        from dvc.render.vega import VegaRenderer

        if self.args.show_vega:
            if not self.args.targets:
                logger.error("please specify a target for `--show-vega`")
                return 1
            if len(self.args.targets) > 1:
                logger.error(
                    "you can only specify one target for `--show-vega`")
                return 1
            if self.args.json:
                logger.error(
                    "'--show-vega' and '--json' are mutually exclusive "
                    "options.")
                return 1

        try:

            plots_data = self._func(targets=self.args.targets,
                                    props=self._props())

            if not plots_data:
                ui.error_write("No plots were loaded, "
                               "visualization file will not be created.")

            renderers = match_renderers(plots_data=plots_data,
                                        templates=self.repo.plots.templates)

            if self.args.show_vega:
                renderer = first(
                    filter(lambda r: isinstance(r, VegaRenderer), renderers))
                if renderer:
                    ui.write_json(renderer.asdict())
                return 0
            if self.args.json:
                _show_json(renderers, self.args.out)
                return 0

            rel: str = self.args.out or "dvc_plots"
            path: Path = (Path.cwd() / rel).resolve()
            index_path = render(
                self.repo,
                renderers,
                path=path,
                html_template_path=self.args.html_template,
            )

            ui.write(index_path.as_uri())
            auto_open = self.repo.config["plots"].get("auto_open", False)
            if self.args.open or auto_open:
                if not auto_open:
                    ui.write("To enable auto opening, you can run:\n"
                             "\n"
                             "\tdvc config plots.auto_open true")
                return ui.open_browser(index_path)

            return 0

        except DvcException:
            logger.exception("")
            return 1