def test_sideffect_not_canceled_if_not_resched(exemethod): # Check op without any provides # an_sfx = sfx("b") op1 = operation(lambda: {an_sfx: False}, name="op1", provides=an_sfx, returns_dict=True) op2 = operation(lambda: 1, name="op2", needs=an_sfx, provides="b") pipeline = compose("t", op1, op2, parallel=exemethod) # sol = pipeline.compute() # assert sol == {an_sfx: False, "b": 1} sol = pipeline.compute(outputs="b") assert sol == {"b": 1} # Check also op with some provides # an_sfx = sfx("b") op1 = operation( lambda: { "a": 1, an_sfx: False }, name="op1", provides=["a", an_sfx], returns_dict=True, ) op2 = operation(lambda: 1, name="op2", needs=an_sfx, provides="b") pipeline = compose("t", op1, op2, parallel=exemethod) sol = pipeline.compute() assert sol == {"a": 1, an_sfx: False, "b": 1} sol = pipeline.compute(outputs="b") assert sol == {"b": 1}
def test_sideffect_real_input(reverse, exemethod): sidefx_fail = is_marshal_tasks() and not isinstance( get_execution_pool(), types.FunctionType # mp_dummy.Pool ) ops = [ operation(name="extend", needs=["box", "a"], provides=[sfx("b")])(_box_extend), operation(name="increment", needs=["box", sfx("b")], provides="c")(_box_increment), ] if reverse: ops = reversed(ops) # Designate `a`, `b` as sideffect inp/out arguments. graph = compose("mygraph", *ops, parallel=exemethod) box_orig = [0] assert graph(**{ "box": [0], "a": True }) == { "a": True, "box": box_orig if sidefx_fail else [1, 2, 3], "c": None, } assert graph.compute({ "box": [0], "a": True }, ["box", "c"]) == { "box": box_orig if sidefx_fail else [1, 2, 3], "c": None, }
def pipeline_sideffect1(request, exemethod) -> Pipeline: ops = [ operation(name="extend", needs=["box", sfx("a")], provides=[sfx("b")])(_box_extend), operation(name="increment", needs=["box", sfx("b")], provides=sfx("c"))(_box_increment), ] if request.param: ops = reversed(ops) # Designate `a`, `b` as sideffect inp/out arguments. graph = compose("sideffect1", *ops, parallel=exemethod) return graph
def test_compose_nest_dict(caplog): pipe = compose( "t", compose( "p1", operation( str, name="op1", needs=[sfx("a"), "aa"], provides=[sfxed("S1", "g"), sfxed("S2", "h")], ), ), compose( "p2", operation( str, name="op2", needs=sfx("a"), provides=["a", sfx("b")], aliases=[("a", "b")], ), ), nest={ "op1": True, "op2": lambda n: "p2.op2", "aa": False, sfx("a"): True, "b": lambda n: f"PP.{n}", sfxed("S1", "g"): True, sfxed("S2", "h"): lambda n: dep_renamed(n, "ss2"), sfx("b"): True, }, ) got = str(pipe.ops) print(got) assert got == re.sub( r"[\n ]{2,}", # collapse all space-chars into a single space " ", """ [FnOp(name='p1.op1', needs=[sfx('p1.a'), 'aa'], provides=[sfxed('p1.S1', 'g'), sfxed('ss2', 'h')], fn='str'), FnOp(name='p2.op2', needs=[sfx('p2.a')], provides=['a', sfx('p2.b'), 'PP.b'], aliases=[('a', 'PP.b')], fn='str')] """.strip(), ) for record in caplog.records: assert record.levelname != "WARNING"
def test_compose_rename_dict(caplog): pip = compose( "t", operation(str, "op1", provides=["a", "aa"]), operation( str, "op2", needs="a", provides=["b", sfx("c")], aliases=[("b", "B"), ("b", "p")], ), nest={ "op1": "OP1", "op2": lambda n: "OP2", "a": "A", "b": "bb" }, ) print(str(pip)) assert str(pip) == ( "Pipeline('t', needs=['A'], " "provides=['A', 'aa', 'bb', sfx('c'), 'B', 'p'], x2 ops: OP1, OP2)") print(str(pip.ops)) assert (str(pip.ops) == dedent(""" [FnOp(name='OP1', provides=['A', 'aa'], fn='str'), FnOp(name='OP2', needs=['A'], provides=['bb', sfx('c'), 'B', 'p'], aliases=[('bb', 'B'), ('bb', 'p')], fn='str')] """).replace("\n", ""))
def test_sideffect_steps(exemethod, pipeline_sideffect1: Pipeline): sidefx_fail = is_marshal_tasks() and not isinstance( get_execution_pool(), types.FunctionType # mp_dummy.Pool ) pipeline = pipeline_sideffect1.withset(parallel=exemethod) box_orig = [0] sol = pipeline.compute({"box": [0], sfx("a"): True}, ["box", sfx("c")]) assert sol == {"box": box_orig if sidefx_fail else [1, 2, 3]} assert len(sol.plan.steps) == 4 ## Check sideffect links plotted as blue # (assumes color used only for this!). dot = pipeline.net.plot() assert "blue" in str(dot)
def test_pipe_rename(): pipe = compose( "t", operation(str, name="op1", needs=sfx("a")), operation( str, name="op2", needs=sfx("a"), provides=["a", sfx("b")], aliases=[("a", "b")], ), ) def renamer(na): assert na.op assert not na.parent return dep_renamed(na.name, lambda name: f"PP.{name}") ren = pipe.withset(renamer=renamer) got = str(ren) assert got == (""" Pipeline('t', needs=[sfx('PP.a')], provides=['PP.a', sfx('PP.b'), 'PP.b'], x2 ops: PP.op1, PP.op2) """.strip()) got = str(ren.ops) assert got == oneliner(""" [FnOp(name='PP.op1', needs=[sfx('PP.a')], fn='str'), FnOp(name='PP.op2', needs=[sfx('PP.a')], provides=['PP.a', sfx('PP.b'), 'PP.b'], aliases=[('PP.a', 'PP.b')], fn='str')] """) ## Check dictionary with callables # ren = pipe.withset(renamer={ "op1": lambda n: "OP1", "op2": False, "a": optional("a"), "b": "B", }) got = str(ren.ops) assert got == oneliner(""" [FnOp(name='OP1', needs=[sfx('a')], fn='str'), FnOp(name='op2', needs=[sfx('a')], provides=['a'(?), sfx('b'), 'B'], aliases=[('a'(?), 'B')], fn='str')] """)
def test_op_rename(): op = operation( str, name="op1", needs=sfx("a"), provides=["a", sfx("b")], aliases=[("a", "b")], ) def renamer(na): assert na.op assert not na.parent return dep_renamed(na.name, lambda name: f"PP.{name}") ren = op.withset(renamer=renamer) got = str(ren) assert got == (""" FnOp(name='PP.op1', needs=[sfx('PP.a')], provides=['PP.a', sfx('PP.b'), 'PP.b'], aliases=[('PP.a', 'PP.b')], fn='str') """.strip())
def test_sideffect_no_real_data(pipeline_sideffect1: Pipeline): sidefx_fail = is_marshal_tasks() and not isinstance( get_execution_pool(), types.FunctionType # mp_dummy.Pool ) graph = pipeline_sideffect1 inp = {"box": [0], "a": True} ## Normal data must not match sideffects. # # with pytest.raises(ValueError, match="Unknown output node"): # graph.compute(inp, ["a"]) # with pytest.raises(ValueError, match="Unknown output node"): # graph.compute(inp, ["b"]) ## Cannot compile due to missing inputs/outputs # with pytest.raises(ValueError, match="Unsolvable graph"): graph(**inp) with pytest.raises(ValueError, match="Unsolvable graph"): graph.compute(inp) with pytest.raises(ValueError, match="Unreachable outputs"): graph.compute(inp, ["box", sfx("b")]) with pytest.raises(ValueError, match="Unsolvable graph"): # Cannot run, since no sideffect inputs given. graph.compute(inp) box_orig = [0] ## OK INPUT SIDEFFECTS # # ok, no asked out sol = graph.compute({"box": [0], sfx("a"): True}) assert sol == { "box": box_orig if sidefx_fail else [1, 2, 3], sfx("a"): True } # # Although no out-sideffect asked (like regular data). assert graph.compute({"box": [0], sfx("a"): True}, "box") == {"box": [0]} # # ok, asked the 1st out-sideffect sol = graph.compute({"box": [0], sfx("a"): True}, ["box", sfx("b")]) assert sol == {"box": box_orig if sidefx_fail else [0, 1, 2]} # # ok, asked the 2nd out-sideffect sol = graph.compute({"box": [0], sfx("a"): True}, ["box", sfx("c")]) assert sol == {"box": box_orig if sidefx_fail else [1, 2, 3]}
def test_sideffect_cancel_sfx_only_operation(exemethod): an_sfx = sfx("b") op1 = operation( lambda: {an_sfx: False}, name="op1", provides=an_sfx, returns_dict=True, rescheduled=True, ) op2 = operation(lambda: 1, name="op2", needs=an_sfx, provides="a") pipeline = compose("t", op1, op2, parallel=exemethod) sol = pipeline.compute({}) assert sol == {an_sfx: False} sol = pipeline.compute(outputs=an_sfx) assert sol == {an_sfx: False}
def test_cwd_fnop(): op = operation( str, None, needs=[ "a", "a/b", "/r/b", optional("o"), keyword("k"), implicit("i"), vararg("v1"), varargs("v2"), sfx("s1"), sfxed("s2", "s22"), vcat("vc"), ], provides=["A/B", "C", "/R"], aliases=[("A/B", "aa"), ("C", "CC"), ("/R", "RR")], cwd="root", ) exp = """ FnOp(name='str', needs=['root/a'($), 'root/a/b'($), '/r/b'($), 'root/o'($?'o'), 'root/k'($>'k'), 'root/i'($), 'root/v1'($*), 'root/v2'($+), sfx('s1'), sfxed('root/s2'($), 's22'), 'root/vc'($)], provides=['root/A/B'($), 'root/C'($), '/R'($), 'root/aa'($), 'root/CC'($), 'root/RR'($)], aliases=[('root/A/B'($), 'root/aa'($)), ('root/C'($), 'root/CC'($)), ('/R'($), 'root/RR'($))], fn='str') """ assert oneliner(op) == oneliner(exp)
def test_jetsam_n_plot_with_DEBUG(): pipe = compose( "mix", operation( str, "FUNC", needs=[ "a", sfxed("b", "foo", keyword="bb"), implicit("c"), sfxed("d", "bar"), vararg("e"), varargs("f"), ], provides=[ "A", sfxed("b", "FOO", keyword="bb"), implicit("C"), sfxed("d", "BAR", optional=True), sfx("FOOBAR"), ], aliases={ "A": "aaa", "b": "bbb", "d": "ddd" }, # FIXME: "D" is implicit! ), ) with debug_enabled(True), pytest.raises(ValueError, match="^Unsolvable"): pipe.compute() with debug_enabled(True), pytest.raises( ValueError, match="^Failed matching inputs <=> needs") as exc: pipe.compute({ "a": 1, sfxed("b", "foo"): 2, "c": 3, sfxed("d", "bar"): 4, "e": 5, "f": [6, 7], }) exc.value.jetsam.plot_fpath.unlink()
def test_sideffect_cancel(exemethod): an_sfx = sfx("b") op1 = operation( lambda: { "a": 1, an_sfx: False }, name="op1", provides=["a", an_sfx], returns_dict=True, rescheduled=True, ) op2 = operation(lambda: 1, name="op2", needs=an_sfx, provides="b") pipeline = compose("t", op1, op2, parallel=exemethod) sol = pipeline.compute() assert sol == {"a": 1, an_sfx: False} sol = pipeline.compute(outputs="a") assert sol == {"a": 1} # an_sfx evicted ## SFX both pruned & evicted # assert an_sfx not in sol.dag.nodes assert an_sfx in sol.plan.steps
def test_op_rename_parts(): op = operation( str, name="op1", needs=[sfx("a/b"), "/a/b"], provides=["b/c", sfxed("d/e/f", "k/l")], aliases=[("b/c", "/b/t")], ) def renamer(na): if na.name and na.typ.endswith(".jsonpart"): return f"PP.{na.name}" ren = op.withset(renamer=renamer) got = str(ren) print(got) assert got == oneliner( """ FnOp(name='op1', needs=[sfx('a/b'), '/PP.a/PP.b'($)], provides=['PP.b/PP.c'($), sfxed('PP.d/PP.e/PP.f'($), 'k/l'), '/PP.b/PP.t'($)], aliases=[('PP.b/PP.c'($), '/PP.b/PP.t'($))], fn='str') """, )
def yield_wrapped_ops( self, fn: Union[Callable, Tuple[Union[str, Collection[str]], Union[Callable, Collection[Callable]]], ], exclude=(), domain: Union[str, int, Collection] = None, ) -> Iterable[FnOp]: """ Convert a (possibly **@autographed**) function into an graphtik **FnOperations**, respecting any configured overrides :param fn: either a callable, or a 2-tuple(`name-path`, `fn-path`) for:: [module[, class, ...]] callable - If `fn` is an operation, yielded as is (found also in 2-tuple). - Both tuple elements may be singulars, and are auto-tuple-zed. - The `name-path` may (or may not) correspond to the given `fn-path`, and is used to derrive the operation-name; If not given, the function name is inspected. - The last elements of the `name-path` are overridden by names in decorations; if the decor-name is the "default" (`None`), the `name-path` becomes the op-name. - The `name-path` is not used when matching overrides. :param exclude: a list of decor-names to exclude, as stored in decors. Ignored if `fn` already an operation. :param domain: if given, overrides :attr:`domain` for :func:`.autographed` decorators to search. List-ified if a single str, :func:`autographed` decors for the 1st one matching are used. :return: one or more :class:`FnOp` instances (if more than one name is defined when the given function was :func:`autographed`). Overriddes order: my-args, self.overrides, autograph-decorator, inspection See also: David Brubeck Quartet, "40 days" """ if isinstance(fn, tuple): name_path, fn_path = fn else: name_path, fn_path = (), fn fun_path = cast(Tuple[Callable, ...], astuple(fn_path, None)) fun = fun_path[-1] if isinstance(fun, Operation): ## pass-through operations yield fun return def param_to_modifier(name: str, param: inspect.Parameter) -> str: return (optional(name) # is optional? if param.default is not inspect._empty # type: ignore else keyword(name) if param.kind == Parameter.KEYWORD_ONLY else name) given_name_path = astuple(name_path, None) decors_by_name = get_autograph_decors(fun, {}, domain or self.domain) for decor_name, decors in decors_by_name.items() or ((None, {}), ): if given_name_path and not decor_name: name_path = decor_path = given_name_path else: # Name in decors was "default"(None). name_path = decor_path = astuple( (decor_name if decor_name else func_name(fun, fqdn=1)).split("."), None, ) assert decor_path, locals() if given_name_path: # Overlay `decor_path` over `named_path`, right-aligned. name_path = tuple(*name_path[:-len(decor_path)], *decor_path) fn_name = str(name_path[-1]) if fn_name in exclude: continue overrides = self._from_overrides(decor_path) op_data = (ChainMap(overrides, decors) if (overrides and decors) else overrides if overrides else decors) if op_data: log.debug("Autograph overrides for %r: %s", name_path, op_data) op_props = "needs provides renames, inp_sideffects out_sideffects".split( ) needs, provides, override_renames, inp_sideffects, out_sideffects = ( op_data.get(a, _unset) for a in op_props) sig = None if needs is _unset: sig = inspect.signature(fun) needs = [ param_to_modifier(name, param) for name, param in sig.parameters.items() if name != "self" and param.kind is not Parameter.VAR_KEYWORD ] ## Insert object as 1st need for object-methods. # if len(fun_path) > 1: clazz = fun_path[-2] # TODO: respect autograph decorator for object-names. class_name = name_path[-2] if len( name_path) > 1 else clazz.__name__ if is_regular_class(class_name, clazz): log.debug("Object-method %s.%s", class_name, fn_name) needs.insert(0, camel_2_snake_case(class_name)) needs = aslist(needs, "needs") if ... in needs: if sig is None: sig = inspect.signature(fun) needs = [ arg_name if n is ... else n for n, arg_name in zip(needs, sig.parameters) ] if provides is _unset: if is_regular_class(fn_name, fun): ## Convert class-name into object variable. provides = camel_2_snake_case(fn_name) elif self.out_patterns: provides = self._deduce_provides_from_fn_name( fn_name) or _unset if provides is _unset: provides = () provides = aslist(provides, "provides") needs, provides = self._apply_renames( (override_renames, self.renames), (needs, provides)) if inp_sideffects is not _unset: needs.extend((i if is_sfx(i) else sfxed( *i) if isinstance(i, tuple) else sfx(i)) for i in aslist(inp_sideffects, "inp_sideffects")) if out_sideffects is not _unset: provides.extend( (i if is_sfx(i) else sfxed( *i) if isinstance(i, tuple) else sfx(i)) for i in aslist(out_sideffects, "out_sideffects")) if self.full_path_names: fn_name = self._join_path_names(*name_path) op_kws = self._collect_rest_op_args(decors) yield FnOp(fn=fun, name=fn_name, needs=needs, provides=provides, **op_kws)
def test_sideffect_NO_RESULT(caplog, exemethod): # NO_RESULT does not cancel sideffects unless op-rescheduled # an_sfx = sfx("b") op1 = operation(lambda: NO_RESULT, name="do-SFX", provides=an_sfx) op2 = operation(lambda: 1, name="ask-SFX", needs=an_sfx, provides="a") pipeline = compose("t", op1, op2, parallel=exemethod) sol = pipeline.compute({}, outputs=an_sfx) assert op1 in sol.executed assert op2 not in sol.executed assert sol == {} sol = pipeline.compute({}) assert op1 in sol.executed assert op2 in sol.executed assert sol == {"a": 1} sol = pipeline.compute({}, outputs="a") assert op1 in sol.executed assert op2 in sol.executed assert sol == {"a": 1} # NO_RESULT cancels sideffects of rescheduled ops. # pipeline = compose("t", op1, op2, rescheduled=True, parallel=exemethod) sol = pipeline.compute({}) assert op1 in sol.executed assert op2 not in sol.executed assert sol == {an_sfx: False} sol = pipeline.compute({}, outputs="a") assert op2 not in sol.executed assert op1 in sol.executed assert sol == {} # an_sfx evicted # NO_RESULT_BUT_SFX cancels sideffects of rescheduled ops. # op11 = operation(lambda: NO_RESULT_BUT_SFX, name="do-SFX", provides=an_sfx) pipeline = compose("t", op11, op2, rescheduled=True, parallel=exemethod) sol = pipeline.compute({}, outputs=an_sfx) assert op11 in sol.executed assert op2 not in sol.executed assert sol == {} sol = pipeline.compute({}) assert op11 in sol.executed assert op2 in sol.executed assert sol == {"a": 1} sol = pipeline.compute({}, outputs="a") assert op11 in sol.executed assert op2 in sol.executed assert sol == {"a": 1} ## If NO_RESULT were not translated, # a warning of unknown out might have emerged. caplog.clear() pipeline = compose("t", operation(lambda: 1, provides=an_sfx), parallel=exemethod) pipeline.compute({}, outputs=an_sfx) for record in caplog.records: if record.levelname == "WARNING": assert "Ignoring result(1) because no `provides`" in record.message caplog.clear() pipeline = compose("t", operation(lambda: NO_RESULT, provides=an_sfx), parallel=exemethod) pipeline.compute({}, outputs=an_sfx) for record in caplog.records: assert record.levelname != "WARNING"
return n2v_g_vmax * V.max() def calc_n_max_vehicle(n2v_g_vmax, v_max): """Calc `n_max3` of Annex 2-2.g from `v_max` (Annex 2-2.i). """ return n2v_g_vmax * v_max def calc_n_max(n95_high, n_max_cycle, n_max_vehicle): n_max = max(n95_high, n_max_cycle, n_max_vehicle) assert np.isfinite(n_max), ( "All `n_max` are NANs?", n95_high, n_max_cycle, n_max_vehicle, n_max, ) return n_max @autog.autographed(needs=["wot", ...], provides=sfx("valid_n_max")) def validate_n_max(wot: pd.DataFrame, n_max: int) -> None: """Simply check the last N wot point is >= `n_max`. """ w = wio.pstep_factory.get().wot if wot[w.n].iloc[-1] < n_max: raise ValueError( f"Last wot point ({wot[w.n].iloc[-1]}min⁻¹, {wot[w.n].iloc[-1]:.02f}kW) is below n_max({n_max})!" )
}), ValueError( r"The `aliases` \['a'-->'A'\] rename non-existent provides in \[\]" ), ), ( ("a", { "a": "A", "b": "B" }), ValueError( r"The `aliases` \['b'-->'B'\] rename non-existent provides in \['a'\]" ), ), ( (sfx("a"), { sfx("a"): "a" }), ValueError("must not contain `sideffects"), ), ( ("a", { "a": sfx("AA") }), ValueError("must not contain `sideffects"), ), ( (["a", "b"], { "a": "b" }), ValueError(r"clash with existing provides in \['a', 'b'\]"),