def test_failed_exp(tmp_dir, scm, dvc, exp_stage, mocker, caplog):
from dvc.exceptions import ReproductionError
tmp_dir.gen("params.yaml", "foo: 2")
mocker.patch(
"concurrent.futures.Future.exception",
return_value=ReproductionError(exp_stage.relpath),
)
with caplog.at_level(logging.ERROR):
dvc.experiments.run(exp_stage.addressing, tmp_dir=True)
assert "Failed to reproduce experiment" in caplog.text
def _reproduce_stages(G, stages, node, force, dry, interactive,
ignore_build_cache, no_commit):
import networkx as nx
result = []
for n in nx.dfs_postorder_nodes(G, node):
try:
ret = _reproduce_stage(stages, n, force, dry, interactive,
no_commit)
if len(ret) != 0 and ignore_build_cache:
# NOTE: we are walking our pipeline from the top to the
# bottom. If one stage is changed, it will be reproduced,
# which tells us that we should force reproducing all of
# the other stages down below, even if their direct
# dependencies didn't change.
force = True
result += ret
except Exception as ex:
raise ReproductionError(stages[n].relpath, ex)
return result
def _reproduce_stages(G,
stages,
downstream=False,
single_item=False,
**kwargs):
r"""Derive the evaluation of the given node for the given graph.
When you _reproduce a stage_, you want to _evaluate the descendants_
to know if it make sense to _recompute_ it. A post-ordered search
will give us an order list of the nodes we want.
For example, let's say that we have the following pipeline:
E
/ \
D F
/ \ \
B C G
\ /
A
The derived evaluation of D would be: [A, B, C, D]
In case that `downstream` option is specified, the desired effect
is to derive the evaluation starting from the given stage up to the
ancestors. However, the `networkx.ancestors` returns a set, without
any guarantee of any order, so we are going to reverse the graph and
use a reverse post-ordered search using the given stage as a starting
point.
E A
/ \ / \
D F B C G
/ \ \ --- reverse --> \ / /
B C G D F
\ / \ /
A E
The derived evaluation of _downstream_ B would be: [B, D, E]
"""
import networkx as nx
if single_item:
all_pipelines = stages
else:
all_pipelines = []
for stage in stages:
if downstream:
# NOTE (py3 only):
# Python's `deepcopy` defaults to pickle/unpickle the object.
# Stages are complex objects (with references to `repo`,
# `outs`, and `deps`) that cause struggles when you try
# to serialize them. We need to create a copy of the graph
# itself, and then reverse it, instead of using
# graph.reverse() directly because it calls `deepcopy`
# underneath -- unless copy=False is specified.
nodes = nx.dfs_postorder_nodes(G.copy().reverse(copy=False),
stage)
all_pipelines += reversed(list(nodes))
else:
all_pipelines += nx.dfs_postorder_nodes(G, stage)
pipeline = []
for stage in all_pipelines:
if stage not in pipeline:
pipeline.append(stage)
result = []
for stage in pipeline:
try:
ret = _reproduce_stage(stage, **kwargs)
if len(ret) != 0 and kwargs.get("ignore_build_cache", False):
# NOTE: we are walking our pipeline from the top to the
# bottom. If one stage is changed, it will be reproduced,
# which tells us that we should force reproducing all of
# the other stages down below, even if their direct
# dependencies didn't change.
kwargs["force"] = True
result.extend(ret)
except Exception as exc:
raise ReproductionError(stage.relpath) from exc
return result
def _reproduce_stages(
G, stages, downstream=False, single_item=False, on_unchanged=None, **kwargs
):
r"""Derive the evaluation of the given node for the given graph.
When you _reproduce a stage_, you want to _evaluate the descendants_
to know if it make sense to _recompute_ it. A post-ordered search
will give us an order list of the nodes we want.
For example, let's say that we have the following pipeline:
E
/ \
D F
/ \ \
B C G
\ /
A
The derived evaluation of D would be: [A, B, C, D]
In case that `downstream` option is specified, the desired effect
is to derive the evaluation starting from the given stage up to the
ancestors. However, the `networkx.ancestors` returns a set, without
any guarantee of any order, so we are going to reverse the graph and
use a reverse post-ordered search using the given stage as a starting
point.
E A
/ \ / \
D F B C G
/ \ \ --- reverse --> \ / /
B C G D F
\ / \ /
A E
The derived evaluation of _downstream_ B would be: [B, D, E]
"""
pipeline = _get_pipeline(G, stages, downstream, single_item)
force_downstream = kwargs.pop("force_downstream", False)
result = []
unchanged = []
# `ret` is used to add a cosmetic newline.
ret = []
checkpoint_func = kwargs.pop("checkpoint_func", None)
for stage in pipeline:
if ret:
logger.info("")
if checkpoint_func:
kwargs["checkpoint_func"] = partial(
_repro_callback, checkpoint_func, unchanged
)
try:
ret = _reproduce_stage(stage, **kwargs)
if len(ret) == 0:
unchanged.extend([stage])
elif force_downstream:
# NOTE: we are walking our pipeline from the top to the
# bottom. If one stage is changed, it will be reproduced,
# which tells us that we should force reproducing all of
# the other stages down below, even if their direct
# dependencies didn't change.
kwargs["force"] = True
if ret:
result.extend(ret)
except CheckpointKilledError:
raise
except Exception as exc:
raise ReproductionError(stage.relpath) from exc
if on_unchanged is not None:
on_unchanged(unchanged)
return result
def _reproduce_stages(
G,
stages,
node,
force,
dry,
interactive,
ignore_build_cache,
no_commit,
downstream,
):
r"""Derive the evaluation of the given node for the given graph.
When you _reproduce a stage_, you want to _evaluate the descendants_
to know if it make sense to _recompute_ it. A post-ordered search
will give us an order list of the nodes we want.
For example, let's say that we have the following pipeline:
E
/ \
D F
/ \ \
B C G
\ /
A
The derived evaluation of D would be: [A, B, C, D]
In case that `downstream` option is specifed, the desired effect
is to derive the evaluation starting from the given stage up to the
ancestors. However, the `networkx.ancestors` returns a set, without
any guarantee of any order, so we are going to reverse the graph and
use a pre-ordered search using the given stage as a starting point.
E A
/ \ / \
D F B C G
/ \ \ --- reverse --> \ / /
B C G D F
\ / \ /
A E
The derived evaluation of _downstream_ B would be: [B, D, E]
"""
import networkx as nx
if downstream:
# NOTE (py3 only):
# Python's `deepcopy` defaults to pickle/unpickle the object.
# Stages are complex objects (with references to `repo`, `outs`,
# and `deps`) that cause struggles when you try to serialize them.
# We need to create a copy of the graph itself, and then reverse it,
# instead of using graph.reverse() directly because it calls
# `deepcopy` underneath -- unless copy=False is specified.
pipeline = nx.dfs_preorder_nodes(G.copy().reverse(copy=False), node)
else:
pipeline = nx.dfs_postorder_nodes(G, node)
result = []
for n in pipeline:
try:
ret = _reproduce_stage(stages, n, force, dry, interactive,
no_commit)
if len(ret) != 0 and ignore_build_cache:
# NOTE: we are walking our pipeline from the top to the
# bottom. If one stage is changed, it will be reproduced,
# which tells us that we should force reproducing all of
# the other stages down below, even if their direct
# dependencies didn't change.
force = True
result += ret
except Exception as ex:
raise ReproductionError(stages[n].relpath, ex)
return result