def test_failed_exp(tmp_dir, scm, dvc, exp_stage, mocker, caplog): from dvc.exceptions import ReproductionError tmp_dir.gen("params.yaml", "foo: 2") mocker.patch( "concurrent.futures.Future.exception", return_value=ReproductionError(exp_stage.relpath), ) with caplog.at_level(logging.ERROR): dvc.experiments.run(exp_stage.addressing, tmp_dir=True) assert "Failed to reproduce experiment" in caplog.text
def _reproduce_stages(G, stages, node, force, dry, interactive, ignore_build_cache, no_commit): import networkx as nx result = [] for n in nx.dfs_postorder_nodes(G, node): try: ret = _reproduce_stage(stages, n, force, dry, interactive, no_commit) if len(ret) != 0 and ignore_build_cache: # NOTE: we are walking our pipeline from the top to the # bottom. If one stage is changed, it will be reproduced, # which tells us that we should force reproducing all of # the other stages down below, even if their direct # dependencies didn't change. force = True result += ret except Exception as ex: raise ReproductionError(stages[n].relpath, ex) return result
def _reproduce_stages(G, stages, downstream=False, single_item=False, **kwargs): r"""Derive the evaluation of the given node for the given graph. When you _reproduce a stage_, you want to _evaluate the descendants_ to know if it make sense to _recompute_ it. A post-ordered search will give us an order list of the nodes we want. For example, let's say that we have the following pipeline: E / \ D F / \ \ B C G \ / A The derived evaluation of D would be: [A, B, C, D] In case that `downstream` option is specified, the desired effect is to derive the evaluation starting from the given stage up to the ancestors. However, the `networkx.ancestors` returns a set, without any guarantee of any order, so we are going to reverse the graph and use a reverse post-ordered search using the given stage as a starting point. E A / \ / \ D F B C G / \ \ --- reverse --> \ / / B C G D F \ / \ / A E The derived evaluation of _downstream_ B would be: [B, D, E] """ import networkx as nx if single_item: all_pipelines = stages else: all_pipelines = [] for stage in stages: if downstream: # NOTE (py3 only): # Python's `deepcopy` defaults to pickle/unpickle the object. # Stages are complex objects (with references to `repo`, # `outs`, and `deps`) that cause struggles when you try # to serialize them. We need to create a copy of the graph # itself, and then reverse it, instead of using # graph.reverse() directly because it calls `deepcopy` # underneath -- unless copy=False is specified. nodes = nx.dfs_postorder_nodes(G.copy().reverse(copy=False), stage) all_pipelines += reversed(list(nodes)) else: all_pipelines += nx.dfs_postorder_nodes(G, stage) pipeline = [] for stage in all_pipelines: if stage not in pipeline: pipeline.append(stage) result = [] for stage in pipeline: try: ret = _reproduce_stage(stage, **kwargs) if len(ret) != 0 and kwargs.get("ignore_build_cache", False): # NOTE: we are walking our pipeline from the top to the # bottom. If one stage is changed, it will be reproduced, # which tells us that we should force reproducing all of # the other stages down below, even if their direct # dependencies didn't change. kwargs["force"] = True result.extend(ret) except Exception as exc: raise ReproductionError(stage.relpath) from exc return result
def _reproduce_stages( G, stages, downstream=False, single_item=False, on_unchanged=None, **kwargs ): r"""Derive the evaluation of the given node for the given graph. When you _reproduce a stage_, you want to _evaluate the descendants_ to know if it make sense to _recompute_ it. A post-ordered search will give us an order list of the nodes we want. For example, let's say that we have the following pipeline: E / \ D F / \ \ B C G \ / A The derived evaluation of D would be: [A, B, C, D] In case that `downstream` option is specified, the desired effect is to derive the evaluation starting from the given stage up to the ancestors. However, the `networkx.ancestors` returns a set, without any guarantee of any order, so we are going to reverse the graph and use a reverse post-ordered search using the given stage as a starting point. E A / \ / \ D F B C G / \ \ --- reverse --> \ / / B C G D F \ / \ / A E The derived evaluation of _downstream_ B would be: [B, D, E] """ pipeline = _get_pipeline(G, stages, downstream, single_item) force_downstream = kwargs.pop("force_downstream", False) result = [] unchanged = [] # `ret` is used to add a cosmetic newline. ret = [] checkpoint_func = kwargs.pop("checkpoint_func", None) for stage in pipeline: if ret: logger.info("") if checkpoint_func: kwargs["checkpoint_func"] = partial( _repro_callback, checkpoint_func, unchanged ) try: ret = _reproduce_stage(stage, **kwargs) if len(ret) == 0: unchanged.extend([stage]) elif force_downstream: # NOTE: we are walking our pipeline from the top to the # bottom. If one stage is changed, it will be reproduced, # which tells us that we should force reproducing all of # the other stages down below, even if their direct # dependencies didn't change. kwargs["force"] = True if ret: result.extend(ret) except CheckpointKilledError: raise except Exception as exc: raise ReproductionError(stage.relpath) from exc if on_unchanged is not None: on_unchanged(unchanged) return result
def _reproduce_stages( G, stages, node, force, dry, interactive, ignore_build_cache, no_commit, downstream, ): r"""Derive the evaluation of the given node for the given graph. When you _reproduce a stage_, you want to _evaluate the descendants_ to know if it make sense to _recompute_ it. A post-ordered search will give us an order list of the nodes we want. For example, let's say that we have the following pipeline: E / \ D F / \ \ B C G \ / A The derived evaluation of D would be: [A, B, C, D] In case that `downstream` option is specifed, the desired effect is to derive the evaluation starting from the given stage up to the ancestors. However, the `networkx.ancestors` returns a set, without any guarantee of any order, so we are going to reverse the graph and use a pre-ordered search using the given stage as a starting point. E A / \ / \ D F B C G / \ \ --- reverse --> \ / / B C G D F \ / \ / A E The derived evaluation of _downstream_ B would be: [B, D, E] """ import networkx as nx if downstream: # NOTE (py3 only): # Python's `deepcopy` defaults to pickle/unpickle the object. # Stages are complex objects (with references to `repo`, `outs`, # and `deps`) that cause struggles when you try to serialize them. # We need to create a copy of the graph itself, and then reverse it, # instead of using graph.reverse() directly because it calls # `deepcopy` underneath -- unless copy=False is specified. pipeline = nx.dfs_preorder_nodes(G.copy().reverse(copy=False), node) else: pipeline = nx.dfs_postorder_nodes(G, node) result = [] for n in pipeline: try: ret = _reproduce_stage(stages, n, force, dry, interactive, no_commit) if len(ret) != 0 and ignore_build_cache: # NOTE: we are walking our pipeline from the top to the # bottom. If one stage is changed, it will be reproduced, # which tells us that we should force reproducing all of # the other stages down below, even if their direct # dependencies didn't change. force = True result += ret except Exception as ex: raise ReproductionError(stages[n].relpath, ex) return result